@marinoscar/kvox-eval 0.0.0-bootstrap → 0.1.0-next.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (245) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/LICENSE +21 -0
  3. package/README.md +166 -0
  4. package/dist/ask/index.d.ts +6 -0
  5. package/dist/ask/index.js +28 -0
  6. package/dist/ask/question-schema.d.ts +121 -0
  7. package/dist/ask/question-schema.js +167 -0
  8. package/dist/ask/report.d.ts +88 -0
  9. package/dist/ask/report.js +170 -0
  10. package/dist/ask/scorer.d.ts +104 -0
  11. package/dist/ask/scorer.js +164 -0
  12. package/dist/ask-run/ask-command.d.ts +62 -0
  13. package/dist/ask-run/ask-command.js +254 -0
  14. package/dist/ask-run/host/app.d.ts +65 -0
  15. package/dist/ask-run/host/app.js +146 -0
  16. package/dist/ask-run/host/db.d.ts +38 -0
  17. package/dist/ask-run/host/db.js +126 -0
  18. package/dist/ask-run/host/ports.d.ts +35 -0
  19. package/dist/ask-run/host/ports.js +113 -0
  20. package/dist/ask-run/index.d.ts +13 -0
  21. package/dist/ask-run/index.js +35 -0
  22. package/dist/ask-run/replay.d.ts +128 -0
  23. package/dist/ask-run/replay.js +212 -0
  24. package/dist/ask-run/run-ask-eval.d.ts +125 -0
  25. package/dist/ask-run/run-ask-eval.js +193 -0
  26. package/dist/ask-run/seed-golden-graph.d.ts +64 -0
  27. package/dist/ask-run/seed-golden-graph.js +495 -0
  28. package/dist/cli/index.d.ts +3 -0
  29. package/dist/cli/index.js +12 -0
  30. package/dist/cli/kg-command.d.ts +66 -0
  31. package/dist/cli/kg-command.js +238 -0
  32. package/dist/cli/main.d.ts +18 -0
  33. package/dist/cli/main.js +69 -0
  34. package/dist/fixtures/domains.d.ts +16 -0
  35. package/dist/fixtures/domains.js +33 -0
  36. package/dist/fixtures/fixture-schema.d.ts +323 -0
  37. package/dist/fixtures/fixture-schema.js +199 -0
  38. package/dist/fixtures/index.d.ts +16 -0
  39. package/dist/fixtures/index.js +40 -0
  40. package/dist/fixtures/load.d.ts +73 -0
  41. package/dist/fixtures/load.js +137 -0
  42. package/dist/fixtures/questions.d.ts +15 -0
  43. package/dist/fixtures/questions.js +26 -0
  44. package/dist/index.d.ts +20 -0
  45. package/dist/index.js +36 -0
  46. package/dist/kg/gold-runner.d.ts +13 -0
  47. package/dist/kg/gold-runner.js +57 -0
  48. package/dist/kg/index.d.ts +17 -0
  49. package/dist/kg/index.js +65 -0
  50. package/dist/kg/internals.d.ts +7 -0
  51. package/dist/kg/internals.js +17 -0
  52. package/dist/kg/prediction-schema.d.ts +245 -0
  53. package/dist/kg/prediction-schema.js +102 -0
  54. package/dist/kg/predictions.d.ts +11 -0
  55. package/dist/kg/predictions.js +49 -0
  56. package/dist/kg/report.d.ts +84 -0
  57. package/dist/kg/report.js +125 -0
  58. package/dist/kg/run-kg-eval.d.ts +77 -0
  59. package/dist/kg/run-kg-eval.js +137 -0
  60. package/dist/kg/runner.d.ts +72 -0
  61. package/dist/kg/runner.js +51 -0
  62. package/dist/kg/runners/extract-resolve-runner.d.ts +40 -0
  63. package/dist/kg/runners/extract-resolve-runner.js +199 -0
  64. package/dist/kg/runners/extract-runner.d.ts +65 -0
  65. package/dist/kg/runners/extract-runner.js +243 -0
  66. package/dist/kg/scorer.d.ts +226 -0
  67. package/dist/kg/scorer.js +463 -0
  68. package/dist/kg/targets.d.ts +17 -0
  69. package/dist/kg/targets.js +16 -0
  70. package/dist/kg/text.d.ts +45 -0
  71. package/dist/kg/text.js +100 -0
  72. package/dist/live/adapters.d.ts +16 -0
  73. package/dist/live/adapters.js +58 -0
  74. package/dist/live/env.d.ts +39 -0
  75. package/dist/live/env.js +56 -0
  76. package/dist/live/index.d.ts +6 -0
  77. package/dist/live/index.js +16 -0
  78. package/dist/live/live-ai.d.ts +104 -0
  79. package/dist/live/live-ai.js +157 -0
  80. package/dist/parity/constants.d.ts +79 -0
  81. package/dist/parity/constants.js +104 -0
  82. package/dist/parity/index.d.ts +20 -0
  83. package/dist/parity/index.js +38 -0
  84. package/dist/parity/internals.d.ts +8 -0
  85. package/dist/parity/internals.js +16 -0
  86. package/dist/parity/scenario.d.ts +45 -0
  87. package/dist/parity/scenario.js +245 -0
  88. package/dist/parity/script.d.ts +44 -0
  89. package/dist/parity/script.js +136 -0
  90. package/dist/parity/snapshot.d.ts +43 -0
  91. package/dist/parity/snapshot.js +71 -0
  92. package/dist/real-data/index.d.ts +2 -0
  93. package/dist/real-data/index.js +12 -0
  94. package/dist/real-data/real-data.d.ts +76 -0
  95. package/dist/real-data/real-data.js +156 -0
  96. package/fixtures/kg-golden/README.md +136 -0
  97. package/fixtures/kg-golden/ask-questions.json +458 -0
  98. package/fixtures/kg-golden/meetings/m01-northwind-pilot-kickoff.json +792 -0
  99. package/fixtures/kg-golden/meetings/m02-halden-migration-planning.json +814 -0
  100. package/fixtures/kg-golden/meetings/m03-larkspur-weekly-sync.json +601 -0
  101. package/fixtures/kg-golden/meetings/m04-corvid-intake-discovery.json +690 -0
  102. package/fixtures/kg-golden/meetings/m05-hosting-vendor-selection.json +631 -0
  103. package/fixtures/kg-golden/meetings/m06-northwind-floor-data-review.json +649 -0
  104. package/fixtures/kg-golden/meetings/m07-gullwing-intro-notes.json +428 -0
  105. package/fixtures/kg-golden/meetings/m08-halden-migration-checkpoint.json +672 -0
  106. package/fixtures/kg-golden/meetings/m09-northwind-platform-org-update.json +711 -0
  107. package/fixtures/kg-golden/meetings/m10-corvid-prototype-review.json +623 -0
  108. package/fixtures/kg-golden/meetings/m11-solvane-grid-dashboard-scoping.json +689 -0
  109. package/fixtures/kg-golden/meetings/m12-northwind-pilot-replan.json +681 -0
  110. package/fixtures/kg-golden/meetings/m13-halden-cutover-go-no-go.json +721 -0
  111. package/fixtures/kg-golden/meetings/m14-gullwing-discovery-readout.json +719 -0
  112. package/fixtures/kg-golden/meetings/m15-halden-handover-ravi-leaving.json +716 -0
  113. package/fixtures/kg-golden/meetings/m16-varden-claims-portal-intro.json +704 -0
  114. package/fixtures/kg-golden/meetings/m17-corvid-usability-debrief.json +572 -0
  115. package/fixtures/kg-golden/meetings/m18-hosting-decision-revisited.json +655 -0
  116. package/fixtures/kg-golden/meetings/m19-solvane-integration-kickoff.json +726 -0
  117. package/fixtures/kg-golden/meetings/m20-northwind-go-live-retro.json +673 -0
  118. package/fixtures/kg-golden/meetings/m21-quarterly-account-review.json +449 -0
  119. package/fixtures/kg-golden/meetings/m22-halden-intro-call-2020.json +558 -0
  120. package/fixtures/kg-golden/meetings/m23-gullwing-model-review.json +702 -0
  121. package/fixtures/kg-golden/meetings/m24-varden-discovery-midpoint.json +643 -0
  122. package/fixtures/kg-golden/meetings/m25-corvid-rollout-planning.json +642 -0
  123. package/fixtures/kg-golden/meetings/m26-larkspur-engineering-prep.json +695 -0
  124. package/fixtures/kg-golden/meetings/m27-solvane-phase-one-demo.json +649 -0
  125. package/fixtures/kg-golden/meetings/m28-northwind-building-5-scoping.json +715 -0
  126. package/fixtures/kg-golden/meetings/m29-halden-southern-cutover-review.json +752 -0
  127. package/fixtures/kg-golden/meetings/m30-varden-portal-build-kickoff.json +697 -0
  128. package/fixtures/kg-golden/meetings/m31-gullwing-rollout-renewal.json +681 -0
  129. package/fixtures/kg-golden/meetings/m32-brambleway-weekly-sync.json +472 -0
  130. package/fixtures/kg-golden/meetings/m33-nina-oliver-one-on-one.json +342 -0
  131. package/fixtures/kg-golden/meetings/m34-coffee-with-dana.json +249 -0
  132. package/fixtures/kg-golden/meetings/m35-brambleway-june-sync.json +553 -0
  133. package/fixtures/kg-golden/meetings/m36-copenhagen-conference-planning.json +296 -0
  134. package/host/generated/client.d.ts +1 -0
  135. package/host/generated/client.js +5 -0
  136. package/host/generated/default.d.ts +1 -0
  137. package/host/generated/default.js +5 -0
  138. package/host/generated/edge.d.ts +1 -0
  139. package/host/generated/edge.js +1658 -0
  140. package/host/generated/index-browser.js +1684 -0
  141. package/host/generated/index.d.ts +192335 -0
  142. package/host/generated/index.js +1658 -0
  143. package/host/generated/package.json +139 -0
  144. package/host/generated/query_compiler_fast_bg.js +2 -0
  145. package/host/generated/query_compiler_fast_bg.wasm +0 -0
  146. package/host/generated/query_compiler_fast_bg.wasm-base64.js +2 -0
  147. package/host/generated/runtime/client.d.ts +3433 -0
  148. package/host/generated/runtime/client.js +110 -0
  149. package/host/generated/runtime/index-browser.d.ts +90 -0
  150. package/host/generated/runtime/index-browser.js +6 -0
  151. package/host/generated/runtime/wasm-compiler-edge.js +100 -0
  152. package/host/generated/schema.prisma +6312 -0
  153. package/host/generated/wasm-edge-light-loader.mjs +5 -0
  154. package/host/generated/wasm-worker-loader.mjs +5 -0
  155. package/host/prisma/fragments/kvox.ask.prisma +126 -0
  156. package/host/prisma/fragments/kvox.graph.prisma +1073 -0
  157. package/host/prisma/fragments/kvox.notes.prisma +787 -0
  158. package/host/prisma/fragments/kvox.search.prisma +211 -0
  159. package/host/prisma/fragments/kvox.transcripts.prisma +790 -0
  160. package/host/prisma/kvox.lock +101 -0
  161. package/host/prisma/migrations/20240101000000_initial/migration.sql +297 -0
  162. package/host/prisma/migrations/20240101000001_add_personal_access_tokens/migration.sql +31 -0
  163. package/host/prisma/migrations/20240101000002_add_credentials/migration.sql +20 -0
  164. package/host/prisma/migrations/20240101000003_add_notification_deliveries/migration.sql +27 -0
  165. package/host/prisma/migrations/20240101000004_drop_stale_uuid_defaults/migration.sql +35 -0
  166. package/host/prisma/migrations/20240101000005_add_notifications/migration.sql +22 -0
  167. package/host/prisma/migrations/20240101000006_add_push_subscriptions/migration.sql +25 -0
  168. package/host/prisma/migrations/20240101000007_add_jobs/migration.sql +110 -0
  169. package/host/prisma/migrations/20240101000008_add_worker_nodes/migration.sql +90 -0
  170. package/host/prisma/migrations/20240101000009_add_database_backup_runs/migration.sql +104 -0
  171. package/host/prisma/migrations/20240101000010_add_notification_broadcasts/migration.sql +57 -0
  172. package/host/prisma/migrations/20240101000011_add_backup_run_job_link/migration.sql +87 -0
  173. package/host/prisma/migrations/20240101000012_add_job_node_secrets/migration.sql +61 -0
  174. package/host/prisma/migrations/20240101000013_add_backup_run_pg_dump_version/migration.sql +26 -0
  175. package/host/prisma/migrations/20240101000014_add_job_claim_token/migration.sql +92 -0
  176. package/host/prisma/migrations/20240101000015_add_ai_platform/migration.sql +108 -0
  177. package/host/prisma/migrations/20240101000016_revoke_viewer_ai_use/migration.sql +32 -0
  178. package/host/prisma/migrations/20240101000017_add_user_credentials/migration.sql +29 -0
  179. package/host/prisma/migrations/20240101000018_add_device_session_credential_link/migration.sql +52 -0
  180. package/host/prisma/migrations/20240101000019_add_worker_node_vitals/migration.sql +3 -0
  181. package/host/prisma/migrations/20240101000020_add_job_trace_context/migration.sql +6 -0
  182. package/host/prisma/migrations/20240101000021_add_retention_created_at_indexes/migration.sql +13 -0
  183. package/host/prisma/migrations/20240101000022_add_organizations/migration.sql +142 -0
  184. package/host/prisma/migrations/20240101000023_split_system_org_roles/migration.sql +131 -0
  185. package/host/prisma/migrations/20240101000024_org_scoped_rls/migration.sql +215 -0
  186. package/host/prisma/migrations/20240101000025_add_groups/migration.sql +165 -0
  187. package/host/prisma/migrations/20240101000026_add_grants/migration.sql +117 -0
  188. package/host/prisma/migrations/20240101000027_add_org_settings/migration.sql +49 -0
  189. package/host/prisma/migrations/20240101000028_add_org_credentials/migration.sql +54 -0
  190. package/host/prisma/migrations/20240101000029_add_jobs_org_id/migration.sql +18 -0
  191. package/host/prisma/migrations/20240101000030_add_jobs_org_id_status_index/migration.sql +6 -0
  192. package/host/prisma/migrations/20240101000031_add_broadcast_target_org/migration.sql +21 -0
  193. package/host/prisma/migrations/20240101000032_add_android_app/migration.sql +60 -0
  194. package/host/prisma/migrations/20240101000033_open_job_reason/migration.sql +16 -0
  195. package/host/prisma/migrations/20240101000034_ai_usage_feature/migration.sql +37 -0
  196. package/host/prisma/migrations/20240101000035_add_app_metadata/migration.sql +49 -0
  197. package/host/prisma/migrations/20240101000036_open_sharing_vocabularies/migration.sql +97 -0
  198. package/host/prisma/migrations/20260914130000_kvox_add_transcripts/migration.sql +322 -0
  199. package/host/prisma/migrations/20260914140000_kvox_add_notes/migration.sql +311 -0
  200. package/host/prisma/migrations/20260915120000_kvox_add_search_vectors/migration.sql +150 -0
  201. package/host/prisma/migrations/20260915130000_kvox_add_search_embeddings/migration.sql +362 -0
  202. package/host/prisma/migrations/20260915210000_kvox_add_note_title_source/migration.sql +30 -0
  203. package/host/prisma/migrations/20260923120000_kvox_add_note_generation_context_snapshot/migration.sql +24 -0
  204. package/host/prisma/migrations/20260923130000_kvox_add_user_hidden_note_templates/migration.sql +35 -0
  205. package/host/prisma/migrations/20260923140000_kvox_add_transcript_speaker_identities/migration.sql +6 -0
  206. package/host/prisma/migrations/20260923150000_kvox_transcript_name_checks/migration.sql +95 -0
  207. package/host/prisma/migrations/20260923160000_kvox_add_note_body_format/migration.sql +27 -0
  208. package/host/prisma/migrations/20260923170000_kvox_add_note_version_body_format/migration.sql +23 -0
  209. package/host/prisma/migrations/20260926012548_kvox_add_knowledge_graph/migration.sql +680 -0
  210. package/host/prisma/migrations/20260926030000_kvox_fix_kg_relations_valid_precision_chk/migration.sql +29 -0
  211. package/host/prisma/migrations/20260926030100_kvox_add_transcript_recorded_at/migration.sql +9 -0
  212. package/host/prisma/migrations/20260926030200_kvox_kg_evidence_invariant/migration.sql +60 -0
  213. package/host/prisma/migrations/20260926040000_kvox_add_kg_graph_layouts/migration.sql +30 -0
  214. package/host/prisma/migrations/20260926050000_kvox_add_ask_conversations/migration.sql +99 -0
  215. package/host/prisma/migrations/20260926231512_kvox_add_kg_exports/migration.sql +52 -0
  216. package/host/prisma/migrations/20261016120000_kvox_transcripts_org_id/migration.sql +49 -0
  217. package/host/prisma/platform.lock +319 -0
  218. package/host/prisma/schema/app.kvox.ask.prisma +110 -0
  219. package/host/prisma/schema/app.kvox.graph.prisma +1030 -0
  220. package/host/prisma/schema/app.kvox.notes.prisma +746 -0
  221. package/host/prisma/schema/app.kvox.search.prisma +208 -0
  222. package/host/prisma/schema/app.kvox.transcripts.prisma +726 -0
  223. package/host/prisma/schema/platform.ai.prisma +419 -0
  224. package/host/prisma/schema/platform.android-app.prisma +46 -0
  225. package/host/prisma/schema/platform.base.prisma +15 -0
  226. package/host/prisma/schema/platform.credentials.prisma +156 -0
  227. package/host/prisma/schema/platform.db-backup.prisma +359 -0
  228. package/host/prisma/schema/platform.identity.prisma +638 -0
  229. package/host/prisma/schema/platform.jobs.prisma +824 -0
  230. package/host/prisma/schema/platform.notifications.prisma +641 -0
  231. package/host/prisma/schema/platform.settings.prisma +53 -0
  232. package/host/prisma/schema/platform.sharing.prisma +208 -0
  233. package/host/prisma/schema/platform.storage.prisma +118 -0
  234. package/package.json +112 -1
  235. package/parity/requests.ask.json +2681 -0
  236. package/parity/requests.m01.json +1554 -0
  237. package/parity/requests.m02.json +1554 -0
  238. package/parity/requests.m03.json +1554 -0
  239. package/parity/requests.m04.json +1554 -0
  240. package/parity/requests.m05.json +1554 -0
  241. package/parity/requests.m06.json +1554 -0
  242. package/parity/requests.m07.json +1554 -0
  243. package/parity/requests.m08.json +1554 -0
  244. package/parity/requests.m09.json +1554 -0
  245. package/parity/requests.m10.json +1554 -0
@@ -0,0 +1,463 @@
1
+ "use strict";
2
+ // =============================================================================
3
+ // The kg:eval scorer (issue #362) — pure, deterministic, no I/O.
4
+ //
5
+ // Matching rules (the issue's contract, restated only where a choice was made):
6
+ // - Entities: a candidate pair needs equal types and either the normalized
7
+ // predicted label ∈ normalized {gold label ∪ gold aliases} (similarity 1)
8
+ // or a token-set Jaccard ≥ 0.8 against one of those names. Assignment is
9
+ // one-to-one, greedy by descending similarity — never double-counted.
10
+ // - Entity coverage uses the same matcher with the type check dropped.
11
+ // - A relation/item endpoint is compared by CANONICAL identity: a gold
12
+ // entity is its `existingId` when it has one, else `key:<key>`; a
13
+ // `knownEntities` id is itself; a predicted ref maps to the gold entity it
14
+ // matched (or, unmatched, to the id its resolution linked, else nothing).
15
+ // - Items: equal kind, subject matched, statement token-set F1 ≥ 0.5, and a
16
+ // commitment's owner matched when the label names one.
17
+ // - Temporal accuracy: over matched relations of a temporal type (from the
18
+ // ontology registry, never a hand list): equal precision and both bounds
19
+ // equal after truncation to it (`unknown` matches only `unknown`); over
20
+ // matched items that carry a date: `occurredAt`/`dueAt` equal to the day.
21
+ // =============================================================================
22
+ Object.defineProperty(exports, "__esModule", { value: true });
23
+ exports.ratio = exports.BASE_REPORT_TYPES = exports.ITEM_REPORT_TYPES = exports.ENTITY_REPORT_TYPES = exports.ITEM_STATEMENT_F1 = exports.ENTITY_MATCH_JACCARD = exports.normalizeName = void 0;
24
+ exports.nameSimilarity = nameSimilarity;
25
+ exports.matchEntities = matchEntities;
26
+ exports.temporalEqual = temporalEqual;
27
+ exports.scoreFixture = scoreFixture;
28
+ exports.scoreRun = scoreRun;
29
+ exports.precisionOf = precisionOf;
30
+ exports.recallOf = recallOf;
31
+ exports.f1Of = f1Of;
32
+ exports.entityCoverage = entityCoverage;
33
+ exports.commitmentRecall = commitmentRecall;
34
+ exports.autoLinkPrecision = autoLinkPrecision;
35
+ const ontology_1 = require("@marinoscar/kvox-contract/ontology");
36
+ const text_js_1 = require("./text.js");
37
+ var text_js_2 = require("./text.js");
38
+ Object.defineProperty(exports, "normalizeName", { enumerable: true, get: function () { return text_js_2.normalizeName; } });
39
+ /**
40
+ * The token-set Jaccard a predicted label needs against a gold name to be a match candidate.
41
+ *
42
+ * @stability experimental
43
+ */
44
+ exports.ENTITY_MATCH_JACCARD = 0.8;
45
+ /**
46
+ * The token-set F1 a predicted item statement needs against the gold statement.
47
+ *
48
+ * @stability experimental
49
+ */
50
+ exports.ITEM_STATEMENT_F1 = 0.5;
51
+ /**
52
+ * Report rows, fixed order. Relation rows (`relation:<TYPE>`) follow, sorted.
53
+ *
54
+ * @stability experimental
55
+ */
56
+ exports.ENTITY_REPORT_TYPES = [
57
+ 'Person',
58
+ 'Organization',
59
+ 'Project',
60
+ 'Meeting',
61
+ // #383: the `personal` domain's entity types (only `personal`-tagged fixtures label them).
62
+ 'Interest',
63
+ 'Trip',
64
+ 'Milestone',
65
+ ];
66
+ /**
67
+ * The report row of each item kind.
68
+ *
69
+ * @stability experimental
70
+ */
71
+ exports.ITEM_REPORT_TYPES = {
72
+ commitment: 'Commitment',
73
+ decision: 'Decision',
74
+ claim: 'Claim',
75
+ person_fact: 'PersonFact',
76
+ };
77
+ /**
78
+ * Every fixed report row: the entity types, then the item kinds.
79
+ *
80
+ * @stability experimental
81
+ */
82
+ exports.BASE_REPORT_TYPES = [...exports.ENTITY_REPORT_TYPES, ...Object.values(exports.ITEM_REPORT_TYPES)];
83
+ function bump(types, type, key, by = 1) {
84
+ const c = types[type] ?? (types[type] = { tp: 0, fp: 0, fn: 0 });
85
+ c[key] += by;
86
+ }
87
+ function goldNames(label, aliases) {
88
+ return new Set([label, ...aliases].map(text_js_1.normalizeName).filter((n) => n.length > 0));
89
+ }
90
+ /**
91
+ * Similarity of one predicted label to one gold name set; 0 when not a candidate.
92
+ *
93
+ * @stability experimental
94
+ */
95
+ function nameSimilarity(predLabel, gold) {
96
+ const n = (0, text_js_1.normalizeName)(predLabel);
97
+ if (gold.names.has(n))
98
+ return 1;
99
+ const toks = (0, text_js_1.tokenSet)(predLabel);
100
+ let best = 0;
101
+ for (const g of gold.tokenSets)
102
+ best = Math.max(best, (0, text_js_1.jaccard)(toks, g));
103
+ return best >= exports.ENTITY_MATCH_JACCARD ? best : 0;
104
+ }
105
+ /** Greedy one-to-one assignment; returns predIdx → goldIdx. */
106
+ function greedyAssign(candidates) {
107
+ const sorted = [...candidates].sort((a, b) => b.sim - a.sim || a.p - b.p || a.g - b.g);
108
+ const usedP = new Set();
109
+ const usedG = new Set();
110
+ const out = new Map();
111
+ for (const c of sorted) {
112
+ if (usedP.has(c.p) || usedG.has(c.g))
113
+ continue;
114
+ usedP.add(c.p);
115
+ usedG.add(c.g);
116
+ out.set(c.p, c.g);
117
+ }
118
+ return out;
119
+ }
120
+ /**
121
+ * The one-to-one assignment of predicted to gold entities (greedy by descending similarity).
122
+ *
123
+ * @param gold - the gold entities (label and aliases).
124
+ * @param preds - the predicted entities.
125
+ * @param opts - `typed`: candidates must share a type (entity coverage drops the check).
126
+ * @returns predicted index to gold index.
127
+ *
128
+ * @stability experimental
129
+ */
130
+ function matchEntities(gold, preds, opts) {
131
+ const g = gold.map((e) => {
132
+ const names = goldNames(e.label, e.aliases);
133
+ return { type: e.type, names, tokenSets: [...names].map((n) => (0, text_js_1.tokenSet)(n)) };
134
+ });
135
+ const candidates = [];
136
+ preds.forEach((p, pi) => {
137
+ g.forEach((ge, gi) => {
138
+ if (opts.typed && ge.type !== p.type)
139
+ return;
140
+ const sim = nameSimilarity(p.label, ge);
141
+ if (sim > 0)
142
+ candidates.push({ p: pi, g: gi, sim });
143
+ });
144
+ });
145
+ return greedyAssign(candidates);
146
+ }
147
+ const TRUNC = { day: 10, month: 7, year: 4 };
148
+ /**
149
+ * Whether two temporal ranges are equal at the gold precision (`unknown` matches only `unknown`).
150
+ *
151
+ * @param gold - the gold bounds and precision.
152
+ * @param pred - the predicted bounds and precision.
153
+ * @returns `true` when equal.
154
+ *
155
+ * @stability experimental
156
+ */
157
+ function temporalEqual(gold, pred) {
158
+ if (gold.precision !== pred.precision)
159
+ return false;
160
+ if (gold.precision === 'unknown')
161
+ return true;
162
+ const n = TRUNC[gold.precision];
163
+ const t = (d) => (d === null ? null : d.slice(0, n));
164
+ return t(gold.validFrom) === t(pred.validFrom) && t(gold.validTo) === t(pred.validTo);
165
+ }
166
+ const day = (d) => (d ? d.slice(0, 10) : null);
167
+ function evidenceValid(fixture, ev) {
168
+ // A note citation may quote the body (`N`) or the note's Context (`C`, #440).
169
+ if (ev.source === 'note') {
170
+ return (0, text_js_1.quoteFoundIn)(ev.quote, fixture.note.body) || (fixture.contextText !== null && (0, text_js_1.quoteFoundIn)(ev.quote, fixture.contextText));
171
+ }
172
+ const seg = fixture.segments.find((s) => s.id === ev.segmentId);
173
+ return seg !== undefined && (0, text_js_1.quoteFoundIn)(ev.quote, seg.text);
174
+ }
175
+ /**
176
+ * Scores one fixture's prediction. Pure and deterministic.
177
+ *
178
+ * @param fixture - the fixture and its labels.
179
+ * @param prediction - what a run predicted for it.
180
+ * @returns the fixture's score.
181
+ *
182
+ * @stability experimental
183
+ */
184
+ function scoreFixture(fixture, prediction) {
185
+ const types = {};
186
+ for (const t of exports.BASE_REPORT_TYPES)
187
+ types[t] = { tp: 0, fp: 0, fn: 0 };
188
+ const goldEntities = fixture.labels.entities;
189
+ const knownIds = new Set(fixture.knownEntities.map((k) => k.id));
190
+ // --- entities -------------------------------------------------------------
191
+ const typedMatch = matchEntities(goldEntities, prediction.entities, { typed: true });
192
+ const matchedGold = new Set(typedMatch.values());
193
+ prediction.entities.forEach((p, pi) => {
194
+ if (typedMatch.has(pi))
195
+ bump(types, p.type, 'tp');
196
+ else
197
+ bump(types, p.type, 'fp');
198
+ });
199
+ goldEntities.forEach((g, gi) => {
200
+ if (!matchedGold.has(gi))
201
+ bump(types, g.type, 'fn');
202
+ });
203
+ const anyTypeMatch = matchEntities(goldEntities, prediction.entities, { typed: false });
204
+ const coverage = { matched: new Set(anyTypeMatch.values()).size, total: goldEntities.length };
205
+ // --- canonical identities -------------------------------------------------
206
+ const goldCanon = new Map();
207
+ for (const g of goldEntities)
208
+ goldCanon.set(g.key, g.existingId ?? `key:${g.key}`);
209
+ const canonGold = (s) => {
210
+ if (s === null)
211
+ return null;
212
+ return goldCanon.get(s) ?? (knownIds.has(s) ? s : `unresolved:${s}`);
213
+ };
214
+ const predCanon = new Map();
215
+ prediction.entities.forEach((p, pi) => {
216
+ const gi = typedMatch.get(pi);
217
+ if (gi !== undefined) {
218
+ predCanon.set(p.ref, canonGold(goldEntities[gi].key));
219
+ }
220
+ else if (p.resolution?.outcome === 'linked' && p.resolution.entityId) {
221
+ predCanon.set(p.ref, p.resolution.entityId);
222
+ }
223
+ else {
224
+ predCanon.set(p.ref, `pred:${p.ref}`);
225
+ }
226
+ });
227
+ const canonPred = (s) => {
228
+ if (s === null)
229
+ return null;
230
+ return predCanon.get(s) ?? (knownIds.has(s) ? s : `pred:${s}`);
231
+ };
232
+ // --- auto-link precision --------------------------------------------------
233
+ const autoLink = { correct: 0, total: 0, anyResolution: false };
234
+ prediction.entities.forEach((p, pi) => {
235
+ if (p.resolution)
236
+ autoLink.anyResolution = true;
237
+ if (!p.resolution || !p.resolution.prechecked || p.resolution.outcome !== 'linked')
238
+ return;
239
+ autoLink.total += 1;
240
+ const gi = typedMatch.get(pi);
241
+ const expected = gi === undefined ? undefined : goldEntities[gi].existingId;
242
+ if (expected && p.resolution.entityId === expected)
243
+ autoLink.correct += 1;
244
+ });
245
+ const temporal = { accurate: 0, total: 0 };
246
+ // --- relations ------------------------------------------------------------
247
+ const goldRels = fixture.labels.relations;
248
+ const usedGoldRel = new Set();
249
+ for (const p of prediction.relations) {
250
+ const pf = canonPred(p.from);
251
+ const pt = canonPred(p.to);
252
+ // #383: a symmetric relation (SPOUSE_OF, FRIEND_OF) matches either way round.
253
+ const symmetric = ontology_1.ONTOLOGY.relationType(p.type)?.symmetric === true;
254
+ const gi = goldRels.findIndex((g, i) => !usedGoldRel.has(i) &&
255
+ g.type === p.type &&
256
+ ((canonGold(g.from) === pf && canonGold(g.to) === pt) ||
257
+ (symmetric && canonGold(g.from) === pt && canonGold(g.to) === pf)));
258
+ const row = `relation:${p.type}`;
259
+ if (gi === -1) {
260
+ bump(types, row, 'fp');
261
+ continue;
262
+ }
263
+ usedGoldRel.add(gi);
264
+ bump(types, row, 'tp');
265
+ const g = goldRels[gi];
266
+ if (ontology_1.ONTOLOGY.relationType(g.type)?.temporal) {
267
+ temporal.total += 1;
268
+ if (temporalEqual(g, p))
269
+ temporal.accurate += 1;
270
+ }
271
+ }
272
+ goldRels.forEach((g, i) => {
273
+ if (!usedGoldRel.has(i))
274
+ bump(types, `relation:${g.type}`, 'fn');
275
+ });
276
+ // --- items ----------------------------------------------------------------
277
+ const goldItems = fixture.labels.items;
278
+ const itemCandidates = [];
279
+ prediction.items.forEach((p, pi) => {
280
+ goldItems.forEach((g, gi) => {
281
+ if (g.kind !== p.kind)
282
+ return;
283
+ if (canonGold(g.subject) !== canonPred(p.subject))
284
+ return;
285
+ if (g.kind === 'commitment' && g.owner !== null && canonGold(g.owner) !== canonPred(p.owner))
286
+ return;
287
+ const f1 = (0, text_js_1.tokenF1)(g.statement, p.statement);
288
+ if (f1 >= exports.ITEM_STATEMENT_F1)
289
+ itemCandidates.push({ p: pi, g: gi, sim: f1 });
290
+ });
291
+ });
292
+ const itemMatch = greedyAssign(itemCandidates);
293
+ const matchedGoldItems = new Set(itemMatch.values());
294
+ prediction.items.forEach((p, pi) => {
295
+ const row = exports.ITEM_REPORT_TYPES[p.kind];
296
+ const gi = itemMatch.get(pi);
297
+ if (gi === undefined) {
298
+ bump(types, row, 'fp');
299
+ return;
300
+ }
301
+ bump(types, row, 'tp');
302
+ const g = goldItems[gi];
303
+ if (g.occurredAt !== null || g.dueAt !== null) {
304
+ temporal.total += 1;
305
+ if (day(p.occurredAt) === g.occurredAt && day(p.dueAt) === g.dueAt)
306
+ temporal.accurate += 1;
307
+ }
308
+ });
309
+ goldItems.forEach((g, gi) => {
310
+ if (!matchedGoldItems.has(gi))
311
+ bump(types, exports.ITEM_REPORT_TYPES[g.kind], 'fn');
312
+ });
313
+ // --- evidence validity ----------------------------------------------------
314
+ const evidence = { valid: 0, total: 0 };
315
+ const allEvidence = [
316
+ ...prediction.entities.flatMap((e) => e.evidence),
317
+ ...prediction.relations.flatMap((r) => r.evidence),
318
+ ...prediction.items.flatMap((i) => i.evidence),
319
+ ];
320
+ for (const ev of allEvidence) {
321
+ evidence.total += 1;
322
+ if (evidenceValid(fixture, ev))
323
+ evidence.valid += 1;
324
+ }
325
+ return {
326
+ fixtureId: fixture.id,
327
+ types,
328
+ coverage,
329
+ autoLink,
330
+ evidence,
331
+ temporal,
332
+ stats: { ...prediction.stats },
333
+ };
334
+ }
335
+ /**
336
+ * Scores every fixture of a run (a fixture with no prediction scores as empty).
337
+ *
338
+ * @param fixtures - the fixtures.
339
+ * @param predictions - the predictions by fixture id.
340
+ * @returns the run's score.
341
+ *
342
+ * @stability experimental
343
+ */
344
+ function scoreRun(fixtures, predictions) {
345
+ const run = {
346
+ fixtures: fixtures.length,
347
+ types: {},
348
+ coverage: { matched: 0, total: 0 },
349
+ autoLink: { correct: 0, total: 0, anyResolution: false },
350
+ evidence: { valid: 0, total: 0 },
351
+ temporal: { accurate: 0, total: 0 },
352
+ stats: {},
353
+ perFixture: [],
354
+ };
355
+ for (const t of exports.BASE_REPORT_TYPES)
356
+ run.types[t] = { tp: 0, fp: 0, fn: 0 };
357
+ for (const f of fixtures) {
358
+ const pred = predictions.get(f.id) ?? {
359
+ fixtureId: f.id,
360
+ model: null,
361
+ entities: [],
362
+ relations: [],
363
+ items: [],
364
+ stats: {},
365
+ };
366
+ const s = scoreFixture(f, pred);
367
+ run.perFixture.push(s);
368
+ for (const [type, c] of Object.entries(s.types)) {
369
+ bump(run.types, type, 'tp', c.tp);
370
+ bump(run.types, type, 'fp', c.fp);
371
+ bump(run.types, type, 'fn', c.fn);
372
+ }
373
+ run.coverage.matched += s.coverage.matched;
374
+ run.coverage.total += s.coverage.total;
375
+ run.autoLink.correct += s.autoLink.correct;
376
+ run.autoLink.total += s.autoLink.total;
377
+ run.autoLink.anyResolution ||= s.autoLink.anyResolution;
378
+ run.evidence.valid += s.evidence.valid;
379
+ run.evidence.total += s.evidence.total;
380
+ run.temporal.accurate += s.temporal.accurate;
381
+ run.temporal.total += s.temporal.total;
382
+ for (const [k, v] of Object.entries(s.stats))
383
+ run.stats[k] = (run.stats[k] ?? 0) + v;
384
+ }
385
+ return run;
386
+ }
387
+ // --- ratios -----------------------------------------------------------------
388
+ /**
389
+ * `num / den`, or `null` (printed `n/a`) when `den` is 0.
390
+ *
391
+ * @stability experimental
392
+ */
393
+ const ratio = (num, den) => (den === 0 ? null : num / den);
394
+ exports.ratio = ratio;
395
+ /**
396
+ * Precision of one row, or `null` with nothing predicted.
397
+ *
398
+ * @param c - the row's counts.
399
+ * @returns the ratio or `null`.
400
+ *
401
+ * @stability experimental
402
+ */
403
+ function precisionOf(c) {
404
+ return (0, exports.ratio)(c.tp, c.tp + c.fp);
405
+ }
406
+ /**
407
+ * Recall of one row, or `null` with nothing labelled.
408
+ *
409
+ * @param c - the row's counts.
410
+ * @returns the ratio or `null`.
411
+ *
412
+ * @stability experimental
413
+ */
414
+ function recallOf(c) {
415
+ return (0, exports.ratio)(c.tp, c.tp + c.fn);
416
+ }
417
+ /**
418
+ * F1 of one row, or `null` when precision or recall is.
419
+ *
420
+ * @param c - the row's counts.
421
+ * @returns the ratio or `null`.
422
+ *
423
+ * @stability experimental
424
+ */
425
+ function f1Of(c) {
426
+ const p = precisionOf(c);
427
+ const r = recallOf(c);
428
+ if (p === null || r === null)
429
+ return null;
430
+ return p + r === 0 ? 0 : (2 * p * r) / (p + r);
431
+ }
432
+ /**
433
+ * Gold entities matched by any prediction, type ignored, over all gold entities (the §6 entity-coverage target).
434
+ *
435
+ * @param run - the run's coverage counts.
436
+ * @returns the ratio or `null`.
437
+ *
438
+ * @stability experimental
439
+ */
440
+ function entityCoverage(run) {
441
+ return (0, exports.ratio)(run.coverage.matched, run.coverage.total);
442
+ }
443
+ /**
444
+ * Recall of the Commitment row (the §6 commitment-recall target).
445
+ *
446
+ * @param run - the run's per-type counts.
447
+ * @returns the ratio or `null`.
448
+ *
449
+ * @stability experimental
450
+ */
451
+ function commitmentRecall(run) {
452
+ return recallOf(run.types.Commitment ?? { tp: 0, fp: 0, fn: 0 });
453
+ }
454
+ /**
455
+ * `null` (printed `n/a`) when no prediction carries a resolution, or none was a prechecked link.
456
+ *
457
+ * @stability experimental
458
+ */
459
+ function autoLinkPrecision(run) {
460
+ if (!run.autoLink.anyResolution)
461
+ return null;
462
+ return (0, exports.ratio)(run.autoLink.correct, run.autoLink.total);
463
+ }
@@ -0,0 +1,17 @@
1
+ /**
2
+ * The §6 quality bar (docs/specs/ontology.md). Gates on relaxing §7's resolution
3
+ * thresholds, not on shipping: CI reports, `--enforce` fails.
4
+ *
5
+ * @stability experimental
6
+ */
7
+ export declare const KG_EVAL_TARGETS: {
8
+ readonly autoLinkPrecision: 0.95;
9
+ readonly commitmentRecall: 0.85;
10
+ readonly entityCoverage: 0.9;
11
+ };
12
+ /**
13
+ * One of the {@link KG_EVAL_TARGETS} keys.
14
+ *
15
+ * @stability experimental
16
+ */
17
+ export type KgEvalTargetKey = keyof typeof KG_EVAL_TARGETS;
@@ -0,0 +1,16 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.KG_EVAL_TARGETS = void 0;
4
+ // The §6 quality bar (docs/specs/ontology.md). Gates on relaxing §7's
5
+ // resolution thresholds — NOT gates on shipping; CI reports, `--enforce` fails.
6
+ /**
7
+ * The §6 quality bar (docs/specs/ontology.md). Gates on relaxing §7's resolution
8
+ * thresholds, not on shipping: CI reports, `--enforce` fails.
9
+ *
10
+ * @stability experimental
11
+ */
12
+ exports.KG_EVAL_TARGETS = {
13
+ autoLinkPrecision: 0.95,
14
+ commitmentRecall: 0.85,
15
+ entityCoverage: 0.9,
16
+ };
@@ -0,0 +1,45 @@
1
+ /**
2
+ * Collapse every whitespace run to one space and trim.
3
+ *
4
+ * @stability experimental
5
+ */
6
+ export declare function collapseWhitespace(s: string): string;
7
+ /**
8
+ * Whether `quote` occurs verbatim in `haystack`, after whitespace collapse on both.
9
+ *
10
+ * @stability experimental
11
+ */
12
+ export declare function quoteFoundIn(quote: string, haystack: string): boolean;
13
+ /**
14
+ * NFKC → lower-case → strip punctuation except an inner `'`/`-` (one joining
15
+ * two letters/digits) → collapse whitespace. "Sarah Chen," → "sarah chen";
16
+ * "O'Neil-Smith" keeps both joiners.
17
+ *
18
+ * @stability experimental
19
+ */
20
+ export declare function normalizeName(s: string): string;
21
+ /**
22
+ * The set of normalized tokens of a string.
23
+ *
24
+ * @param s - the string.
25
+ * @returns its tokens.
26
+ *
27
+ * @stability experimental
28
+ */
29
+ export declare function tokenSet(s: string): Set<string>;
30
+ /**
31
+ * Jaccard similarity of two token sets.
32
+ *
33
+ * @param a - one set.
34
+ * @param b - the other.
35
+ * @returns |A∩B| / |A∪B| (1 for two empty sets).
36
+ *
37
+ * @stability experimental
38
+ */
39
+ export declare function jaccard(a: Set<string>, b: Set<string>): number;
40
+ /**
41
+ * Token-set F1: 2·|A∩B| / (|A|+|B|).
42
+ *
43
+ * @stability experimental
44
+ */
45
+ export declare function tokenF1(a: string, b: string): number;
@@ -0,0 +1,100 @@
1
+ "use strict";
2
+ // Pure text helpers shared by the scorer, the loader and the fixture test.
3
+ Object.defineProperty(exports, "__esModule", { value: true });
4
+ exports.collapseWhitespace = collapseWhitespace;
5
+ exports.quoteFoundIn = quoteFoundIn;
6
+ exports.normalizeName = normalizeName;
7
+ exports.tokenSet = tokenSet;
8
+ exports.jaccard = jaccard;
9
+ exports.tokenF1 = tokenF1;
10
+ /**
11
+ * Collapse every whitespace run to one space and trim.
12
+ *
13
+ * @stability experimental
14
+ */
15
+ function collapseWhitespace(s) {
16
+ return s.replace(/\s+/g, ' ').trim();
17
+ }
18
+ /**
19
+ * Whether `quote` occurs verbatim in `haystack`, after whitespace collapse on both.
20
+ *
21
+ * @stability experimental
22
+ */
23
+ function quoteFoundIn(quote, haystack) {
24
+ const q = collapseWhitespace(quote);
25
+ return q.length > 0 && collapseWhitespace(haystack).includes(q);
26
+ }
27
+ /**
28
+ * NFKC → lower-case → strip punctuation except an inner `'`/`-` (one joining
29
+ * two letters/digits) → collapse whitespace. "Sarah Chen," → "sarah chen";
30
+ * "O'Neil-Smith" keeps both joiners.
31
+ *
32
+ * @stability experimental
33
+ */
34
+ function normalizeName(s) {
35
+ const lowered = s.normalize('NFKC').toLowerCase().replace(/[‘’]/g, "'");
36
+ let out = '';
37
+ for (let i = 0; i < lowered.length; i += 1) {
38
+ const ch = lowered[i];
39
+ if (/[\p{L}\p{N}\s]/u.test(ch)) {
40
+ out += ch;
41
+ continue;
42
+ }
43
+ if ((ch === "'" || ch === '-') && i > 0 && i < lowered.length - 1) {
44
+ const prev = lowered[i - 1];
45
+ const next = lowered[i + 1];
46
+ if (/[\p{L}\p{N}]/u.test(prev) && /[\p{L}\p{N}]/u.test(next)) {
47
+ out += ch;
48
+ continue;
49
+ }
50
+ }
51
+ out += ' ';
52
+ }
53
+ return out.replace(/\s+/g, ' ').trim();
54
+ }
55
+ /**
56
+ * The set of normalized tokens of a string.
57
+ *
58
+ * @param s - the string.
59
+ * @returns its tokens.
60
+ *
61
+ * @stability experimental
62
+ */
63
+ function tokenSet(s) {
64
+ const n = normalizeName(s);
65
+ return new Set(n.length === 0 ? [] : n.split(' '));
66
+ }
67
+ /**
68
+ * Jaccard similarity of two token sets.
69
+ *
70
+ * @param a - one set.
71
+ * @param b - the other.
72
+ * @returns |A∩B| / |A∪B| (1 for two empty sets).
73
+ *
74
+ * @stability experimental
75
+ */
76
+ function jaccard(a, b) {
77
+ if (a.size === 0 && b.size === 0)
78
+ return 1;
79
+ let inter = 0;
80
+ for (const t of a)
81
+ if (b.has(t))
82
+ inter += 1;
83
+ return inter / (a.size + b.size - inter);
84
+ }
85
+ /**
86
+ * Token-set F1: 2·|A∩B| / (|A|+|B|).
87
+ *
88
+ * @stability experimental
89
+ */
90
+ function tokenF1(a, b) {
91
+ const A = tokenSet(a);
92
+ const B = tokenSet(b);
93
+ if (A.size === 0 && B.size === 0)
94
+ return 1;
95
+ let inter = 0;
96
+ for (const t of A)
97
+ if (B.has(t))
98
+ inter += 1;
99
+ return (2 * inter) / (A.size + B.size);
100
+ }
@@ -0,0 +1,16 @@
1
+ import type { AiProviderAdapter } from '@marinoscar/platform-api/ai';
2
+ /**
3
+ * A provider id {@link platformAdapter} can build.
4
+ *
5
+ * @stability experimental
6
+ */
7
+ export type PlatformAdapterProvider = 'openai' | 'openai-compatible';
8
+ /**
9
+ * The platform's adapter for a provider, constructed outside Nest.
10
+ *
11
+ * @param provider - `openai` or `openai-compatible`.
12
+ * @returns the adapter (`responses`, `embeddings`, ...).
13
+ * @throws Error when the installed platform no longer carries the adapter where this shim looks.
14
+ * @stability experimental
15
+ */
16
+ export declare function platformAdapter(provider: PlatformAdapterProvider): AiProviderAdapter;