@marinoscar/kvox-eval 0.0.0-bootstrap → 0.1.0-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/LICENSE +21 -0
- package/README.md +166 -0
- package/dist/ask/index.d.ts +6 -0
- package/dist/ask/index.js +28 -0
- package/dist/ask/question-schema.d.ts +121 -0
- package/dist/ask/question-schema.js +167 -0
- package/dist/ask/report.d.ts +88 -0
- package/dist/ask/report.js +170 -0
- package/dist/ask/scorer.d.ts +104 -0
- package/dist/ask/scorer.js +164 -0
- package/dist/ask-run/ask-command.d.ts +62 -0
- package/dist/ask-run/ask-command.js +254 -0
- package/dist/ask-run/host/app.d.ts +65 -0
- package/dist/ask-run/host/app.js +146 -0
- package/dist/ask-run/host/db.d.ts +38 -0
- package/dist/ask-run/host/db.js +126 -0
- package/dist/ask-run/host/ports.d.ts +35 -0
- package/dist/ask-run/host/ports.js +113 -0
- package/dist/ask-run/index.d.ts +13 -0
- package/dist/ask-run/index.js +35 -0
- package/dist/ask-run/replay.d.ts +128 -0
- package/dist/ask-run/replay.js +212 -0
- package/dist/ask-run/run-ask-eval.d.ts +125 -0
- package/dist/ask-run/run-ask-eval.js +193 -0
- package/dist/ask-run/seed-golden-graph.d.ts +64 -0
- package/dist/ask-run/seed-golden-graph.js +495 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.js +12 -0
- package/dist/cli/kg-command.d.ts +66 -0
- package/dist/cli/kg-command.js +238 -0
- package/dist/cli/main.d.ts +18 -0
- package/dist/cli/main.js +69 -0
- package/dist/fixtures/domains.d.ts +16 -0
- package/dist/fixtures/domains.js +33 -0
- package/dist/fixtures/fixture-schema.d.ts +323 -0
- package/dist/fixtures/fixture-schema.js +199 -0
- package/dist/fixtures/index.d.ts +16 -0
- package/dist/fixtures/index.js +40 -0
- package/dist/fixtures/load.d.ts +73 -0
- package/dist/fixtures/load.js +137 -0
- package/dist/fixtures/questions.d.ts +15 -0
- package/dist/fixtures/questions.js +26 -0
- package/dist/index.d.ts +20 -0
- package/dist/index.js +36 -0
- package/dist/kg/gold-runner.d.ts +13 -0
- package/dist/kg/gold-runner.js +57 -0
- package/dist/kg/index.d.ts +17 -0
- package/dist/kg/index.js +65 -0
- package/dist/kg/internals.d.ts +7 -0
- package/dist/kg/internals.js +17 -0
- package/dist/kg/prediction-schema.d.ts +245 -0
- package/dist/kg/prediction-schema.js +102 -0
- package/dist/kg/predictions.d.ts +11 -0
- package/dist/kg/predictions.js +49 -0
- package/dist/kg/report.d.ts +84 -0
- package/dist/kg/report.js +125 -0
- package/dist/kg/run-kg-eval.d.ts +77 -0
- package/dist/kg/run-kg-eval.js +137 -0
- package/dist/kg/runner.d.ts +72 -0
- package/dist/kg/runner.js +51 -0
- package/dist/kg/runners/extract-resolve-runner.d.ts +40 -0
- package/dist/kg/runners/extract-resolve-runner.js +199 -0
- package/dist/kg/runners/extract-runner.d.ts +65 -0
- package/dist/kg/runners/extract-runner.js +243 -0
- package/dist/kg/scorer.d.ts +226 -0
- package/dist/kg/scorer.js +463 -0
- package/dist/kg/targets.d.ts +17 -0
- package/dist/kg/targets.js +16 -0
- package/dist/kg/text.d.ts +45 -0
- package/dist/kg/text.js +100 -0
- package/dist/live/adapters.d.ts +16 -0
- package/dist/live/adapters.js +58 -0
- package/dist/live/env.d.ts +39 -0
- package/dist/live/env.js +56 -0
- package/dist/live/index.d.ts +6 -0
- package/dist/live/index.js +16 -0
- package/dist/live/live-ai.d.ts +104 -0
- package/dist/live/live-ai.js +157 -0
- package/dist/parity/constants.d.ts +79 -0
- package/dist/parity/constants.js +104 -0
- package/dist/parity/index.d.ts +20 -0
- package/dist/parity/index.js +38 -0
- package/dist/parity/internals.d.ts +8 -0
- package/dist/parity/internals.js +16 -0
- package/dist/parity/scenario.d.ts +45 -0
- package/dist/parity/scenario.js +245 -0
- package/dist/parity/script.d.ts +44 -0
- package/dist/parity/script.js +136 -0
- package/dist/parity/snapshot.d.ts +43 -0
- package/dist/parity/snapshot.js +71 -0
- package/dist/real-data/index.d.ts +2 -0
- package/dist/real-data/index.js +12 -0
- package/dist/real-data/real-data.d.ts +76 -0
- package/dist/real-data/real-data.js +156 -0
- package/fixtures/kg-golden/README.md +136 -0
- package/fixtures/kg-golden/ask-questions.json +458 -0
- package/fixtures/kg-golden/meetings/m01-northwind-pilot-kickoff.json +792 -0
- package/fixtures/kg-golden/meetings/m02-halden-migration-planning.json +814 -0
- package/fixtures/kg-golden/meetings/m03-larkspur-weekly-sync.json +601 -0
- package/fixtures/kg-golden/meetings/m04-corvid-intake-discovery.json +690 -0
- package/fixtures/kg-golden/meetings/m05-hosting-vendor-selection.json +631 -0
- package/fixtures/kg-golden/meetings/m06-northwind-floor-data-review.json +649 -0
- package/fixtures/kg-golden/meetings/m07-gullwing-intro-notes.json +428 -0
- package/fixtures/kg-golden/meetings/m08-halden-migration-checkpoint.json +672 -0
- package/fixtures/kg-golden/meetings/m09-northwind-platform-org-update.json +711 -0
- package/fixtures/kg-golden/meetings/m10-corvid-prototype-review.json +623 -0
- package/fixtures/kg-golden/meetings/m11-solvane-grid-dashboard-scoping.json +689 -0
- package/fixtures/kg-golden/meetings/m12-northwind-pilot-replan.json +681 -0
- package/fixtures/kg-golden/meetings/m13-halden-cutover-go-no-go.json +721 -0
- package/fixtures/kg-golden/meetings/m14-gullwing-discovery-readout.json +719 -0
- package/fixtures/kg-golden/meetings/m15-halden-handover-ravi-leaving.json +716 -0
- package/fixtures/kg-golden/meetings/m16-varden-claims-portal-intro.json +704 -0
- package/fixtures/kg-golden/meetings/m17-corvid-usability-debrief.json +572 -0
- package/fixtures/kg-golden/meetings/m18-hosting-decision-revisited.json +655 -0
- package/fixtures/kg-golden/meetings/m19-solvane-integration-kickoff.json +726 -0
- package/fixtures/kg-golden/meetings/m20-northwind-go-live-retro.json +673 -0
- package/fixtures/kg-golden/meetings/m21-quarterly-account-review.json +449 -0
- package/fixtures/kg-golden/meetings/m22-halden-intro-call-2020.json +558 -0
- package/fixtures/kg-golden/meetings/m23-gullwing-model-review.json +702 -0
- package/fixtures/kg-golden/meetings/m24-varden-discovery-midpoint.json +643 -0
- package/fixtures/kg-golden/meetings/m25-corvid-rollout-planning.json +642 -0
- package/fixtures/kg-golden/meetings/m26-larkspur-engineering-prep.json +695 -0
- package/fixtures/kg-golden/meetings/m27-solvane-phase-one-demo.json +649 -0
- package/fixtures/kg-golden/meetings/m28-northwind-building-5-scoping.json +715 -0
- package/fixtures/kg-golden/meetings/m29-halden-southern-cutover-review.json +752 -0
- package/fixtures/kg-golden/meetings/m30-varden-portal-build-kickoff.json +697 -0
- package/fixtures/kg-golden/meetings/m31-gullwing-rollout-renewal.json +681 -0
- package/fixtures/kg-golden/meetings/m32-brambleway-weekly-sync.json +472 -0
- package/fixtures/kg-golden/meetings/m33-nina-oliver-one-on-one.json +342 -0
- package/fixtures/kg-golden/meetings/m34-coffee-with-dana.json +249 -0
- package/fixtures/kg-golden/meetings/m35-brambleway-june-sync.json +553 -0
- package/fixtures/kg-golden/meetings/m36-copenhagen-conference-planning.json +296 -0
- package/host/generated/client.d.ts +1 -0
- package/host/generated/client.js +5 -0
- package/host/generated/default.d.ts +1 -0
- package/host/generated/default.js +5 -0
- package/host/generated/edge.d.ts +1 -0
- package/host/generated/edge.js +1658 -0
- package/host/generated/index-browser.js +1684 -0
- package/host/generated/index.d.ts +192335 -0
- package/host/generated/index.js +1658 -0
- package/host/generated/package.json +139 -0
- package/host/generated/query_compiler_fast_bg.js +2 -0
- package/host/generated/query_compiler_fast_bg.wasm +0 -0
- package/host/generated/query_compiler_fast_bg.wasm-base64.js +2 -0
- package/host/generated/runtime/client.d.ts +3433 -0
- package/host/generated/runtime/client.js +110 -0
- package/host/generated/runtime/index-browser.d.ts +90 -0
- package/host/generated/runtime/index-browser.js +6 -0
- package/host/generated/runtime/wasm-compiler-edge.js +100 -0
- package/host/generated/schema.prisma +6312 -0
- package/host/generated/wasm-edge-light-loader.mjs +5 -0
- package/host/generated/wasm-worker-loader.mjs +5 -0
- package/host/prisma/fragments/kvox.ask.prisma +126 -0
- package/host/prisma/fragments/kvox.graph.prisma +1073 -0
- package/host/prisma/fragments/kvox.notes.prisma +787 -0
- package/host/prisma/fragments/kvox.search.prisma +211 -0
- package/host/prisma/fragments/kvox.transcripts.prisma +790 -0
- package/host/prisma/kvox.lock +101 -0
- package/host/prisma/migrations/20240101000000_initial/migration.sql +297 -0
- package/host/prisma/migrations/20240101000001_add_personal_access_tokens/migration.sql +31 -0
- package/host/prisma/migrations/20240101000002_add_credentials/migration.sql +20 -0
- package/host/prisma/migrations/20240101000003_add_notification_deliveries/migration.sql +27 -0
- package/host/prisma/migrations/20240101000004_drop_stale_uuid_defaults/migration.sql +35 -0
- package/host/prisma/migrations/20240101000005_add_notifications/migration.sql +22 -0
- package/host/prisma/migrations/20240101000006_add_push_subscriptions/migration.sql +25 -0
- package/host/prisma/migrations/20240101000007_add_jobs/migration.sql +110 -0
- package/host/prisma/migrations/20240101000008_add_worker_nodes/migration.sql +90 -0
- package/host/prisma/migrations/20240101000009_add_database_backup_runs/migration.sql +104 -0
- package/host/prisma/migrations/20240101000010_add_notification_broadcasts/migration.sql +57 -0
- package/host/prisma/migrations/20240101000011_add_backup_run_job_link/migration.sql +87 -0
- package/host/prisma/migrations/20240101000012_add_job_node_secrets/migration.sql +61 -0
- package/host/prisma/migrations/20240101000013_add_backup_run_pg_dump_version/migration.sql +26 -0
- package/host/prisma/migrations/20240101000014_add_job_claim_token/migration.sql +92 -0
- package/host/prisma/migrations/20240101000015_add_ai_platform/migration.sql +108 -0
- package/host/prisma/migrations/20240101000016_revoke_viewer_ai_use/migration.sql +32 -0
- package/host/prisma/migrations/20240101000017_add_user_credentials/migration.sql +29 -0
- package/host/prisma/migrations/20240101000018_add_device_session_credential_link/migration.sql +52 -0
- package/host/prisma/migrations/20240101000019_add_worker_node_vitals/migration.sql +3 -0
- package/host/prisma/migrations/20240101000020_add_job_trace_context/migration.sql +6 -0
- package/host/prisma/migrations/20240101000021_add_retention_created_at_indexes/migration.sql +13 -0
- package/host/prisma/migrations/20240101000022_add_organizations/migration.sql +142 -0
- package/host/prisma/migrations/20240101000023_split_system_org_roles/migration.sql +131 -0
- package/host/prisma/migrations/20240101000024_org_scoped_rls/migration.sql +215 -0
- package/host/prisma/migrations/20240101000025_add_groups/migration.sql +165 -0
- package/host/prisma/migrations/20240101000026_add_grants/migration.sql +117 -0
- package/host/prisma/migrations/20240101000027_add_org_settings/migration.sql +49 -0
- package/host/prisma/migrations/20240101000028_add_org_credentials/migration.sql +54 -0
- package/host/prisma/migrations/20240101000029_add_jobs_org_id/migration.sql +18 -0
- package/host/prisma/migrations/20240101000030_add_jobs_org_id_status_index/migration.sql +6 -0
- package/host/prisma/migrations/20240101000031_add_broadcast_target_org/migration.sql +21 -0
- package/host/prisma/migrations/20240101000032_add_android_app/migration.sql +60 -0
- package/host/prisma/migrations/20240101000033_open_job_reason/migration.sql +16 -0
- package/host/prisma/migrations/20240101000034_ai_usage_feature/migration.sql +37 -0
- package/host/prisma/migrations/20240101000035_add_app_metadata/migration.sql +49 -0
- package/host/prisma/migrations/20240101000036_open_sharing_vocabularies/migration.sql +97 -0
- package/host/prisma/migrations/20260914130000_kvox_add_transcripts/migration.sql +322 -0
- package/host/prisma/migrations/20260914140000_kvox_add_notes/migration.sql +311 -0
- package/host/prisma/migrations/20260915120000_kvox_add_search_vectors/migration.sql +150 -0
- package/host/prisma/migrations/20260915130000_kvox_add_search_embeddings/migration.sql +362 -0
- package/host/prisma/migrations/20260915210000_kvox_add_note_title_source/migration.sql +30 -0
- package/host/prisma/migrations/20260923120000_kvox_add_note_generation_context_snapshot/migration.sql +24 -0
- package/host/prisma/migrations/20260923130000_kvox_add_user_hidden_note_templates/migration.sql +35 -0
- package/host/prisma/migrations/20260923140000_kvox_add_transcript_speaker_identities/migration.sql +6 -0
- package/host/prisma/migrations/20260923150000_kvox_transcript_name_checks/migration.sql +95 -0
- package/host/prisma/migrations/20260923160000_kvox_add_note_body_format/migration.sql +27 -0
- package/host/prisma/migrations/20260923170000_kvox_add_note_version_body_format/migration.sql +23 -0
- package/host/prisma/migrations/20260926012548_kvox_add_knowledge_graph/migration.sql +680 -0
- package/host/prisma/migrations/20260926030000_kvox_fix_kg_relations_valid_precision_chk/migration.sql +29 -0
- package/host/prisma/migrations/20260926030100_kvox_add_transcript_recorded_at/migration.sql +9 -0
- package/host/prisma/migrations/20260926030200_kvox_kg_evidence_invariant/migration.sql +60 -0
- package/host/prisma/migrations/20260926040000_kvox_add_kg_graph_layouts/migration.sql +30 -0
- package/host/prisma/migrations/20260926050000_kvox_add_ask_conversations/migration.sql +99 -0
- package/host/prisma/migrations/20260926231512_kvox_add_kg_exports/migration.sql +52 -0
- package/host/prisma/migrations/20261016120000_kvox_transcripts_org_id/migration.sql +49 -0
- package/host/prisma/platform.lock +319 -0
- package/host/prisma/schema/app.kvox.ask.prisma +110 -0
- package/host/prisma/schema/app.kvox.graph.prisma +1030 -0
- package/host/prisma/schema/app.kvox.notes.prisma +746 -0
- package/host/prisma/schema/app.kvox.search.prisma +208 -0
- package/host/prisma/schema/app.kvox.transcripts.prisma +726 -0
- package/host/prisma/schema/platform.ai.prisma +419 -0
- package/host/prisma/schema/platform.android-app.prisma +46 -0
- package/host/prisma/schema/platform.base.prisma +15 -0
- package/host/prisma/schema/platform.credentials.prisma +156 -0
- package/host/prisma/schema/platform.db-backup.prisma +359 -0
- package/host/prisma/schema/platform.identity.prisma +638 -0
- package/host/prisma/schema/platform.jobs.prisma +824 -0
- package/host/prisma/schema/platform.notifications.prisma +641 -0
- package/host/prisma/schema/platform.settings.prisma +53 -0
- package/host/prisma/schema/platform.sharing.prisma +208 -0
- package/host/prisma/schema/platform.storage.prisma +118 -0
- package/package.json +112 -1
- package/parity/requests.ask.json +2681 -0
- package/parity/requests.m01.json +1554 -0
- package/parity/requests.m02.json +1554 -0
- package/parity/requests.m03.json +1554 -0
- package/parity/requests.m04.json +1554 -0
- package/parity/requests.m05.json +1554 -0
- package/parity/requests.m06.json +1554 -0
- package/parity/requests.m07.json +1554 -0
- package/parity/requests.m08.json +1554 -0
- package/parity/requests.m09.json +1554 -0
- package/parity/requests.m10.json +1554 -0
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// =============================================================================
|
|
3
|
+
// The kg:eval scorer (issue #362) — pure, deterministic, no I/O.
|
|
4
|
+
//
|
|
5
|
+
// Matching rules (the issue's contract, restated only where a choice was made):
|
|
6
|
+
// - Entities: a candidate pair needs equal types and either the normalized
|
|
7
|
+
// predicted label ∈ normalized {gold label ∪ gold aliases} (similarity 1)
|
|
8
|
+
// or a token-set Jaccard ≥ 0.8 against one of those names. Assignment is
|
|
9
|
+
// one-to-one, greedy by descending similarity — never double-counted.
|
|
10
|
+
// - Entity coverage uses the same matcher with the type check dropped.
|
|
11
|
+
// - A relation/item endpoint is compared by CANONICAL identity: a gold
|
|
12
|
+
// entity is its `existingId` when it has one, else `key:<key>`; a
|
|
13
|
+
// `knownEntities` id is itself; a predicted ref maps to the gold entity it
|
|
14
|
+
// matched (or, unmatched, to the id its resolution linked, else nothing).
|
|
15
|
+
// - Items: equal kind, subject matched, statement token-set F1 ≥ 0.5, and a
|
|
16
|
+
// commitment's owner matched when the label names one.
|
|
17
|
+
// - Temporal accuracy: over matched relations of a temporal type (from the
|
|
18
|
+
// ontology registry, never a hand list): equal precision and both bounds
|
|
19
|
+
// equal after truncation to it (`unknown` matches only `unknown`); over
|
|
20
|
+
// matched items that carry a date: `occurredAt`/`dueAt` equal to the day.
|
|
21
|
+
// =============================================================================
|
|
22
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
|
+
exports.ratio = exports.BASE_REPORT_TYPES = exports.ITEM_REPORT_TYPES = exports.ENTITY_REPORT_TYPES = exports.ITEM_STATEMENT_F1 = exports.ENTITY_MATCH_JACCARD = exports.normalizeName = void 0;
|
|
24
|
+
exports.nameSimilarity = nameSimilarity;
|
|
25
|
+
exports.matchEntities = matchEntities;
|
|
26
|
+
exports.temporalEqual = temporalEqual;
|
|
27
|
+
exports.scoreFixture = scoreFixture;
|
|
28
|
+
exports.scoreRun = scoreRun;
|
|
29
|
+
exports.precisionOf = precisionOf;
|
|
30
|
+
exports.recallOf = recallOf;
|
|
31
|
+
exports.f1Of = f1Of;
|
|
32
|
+
exports.entityCoverage = entityCoverage;
|
|
33
|
+
exports.commitmentRecall = commitmentRecall;
|
|
34
|
+
exports.autoLinkPrecision = autoLinkPrecision;
|
|
35
|
+
const ontology_1 = require("@marinoscar/kvox-contract/ontology");
|
|
36
|
+
const text_js_1 = require("./text.js");
|
|
37
|
+
var text_js_2 = require("./text.js");
|
|
38
|
+
Object.defineProperty(exports, "normalizeName", { enumerable: true, get: function () { return text_js_2.normalizeName; } });
|
|
39
|
+
/**
|
|
40
|
+
* The token-set Jaccard a predicted label needs against a gold name to be a match candidate.
|
|
41
|
+
*
|
|
42
|
+
* @stability experimental
|
|
43
|
+
*/
|
|
44
|
+
exports.ENTITY_MATCH_JACCARD = 0.8;
|
|
45
|
+
/**
|
|
46
|
+
* The token-set F1 a predicted item statement needs against the gold statement.
|
|
47
|
+
*
|
|
48
|
+
* @stability experimental
|
|
49
|
+
*/
|
|
50
|
+
exports.ITEM_STATEMENT_F1 = 0.5;
|
|
51
|
+
/**
|
|
52
|
+
* Report rows, fixed order. Relation rows (`relation:<TYPE>`) follow, sorted.
|
|
53
|
+
*
|
|
54
|
+
* @stability experimental
|
|
55
|
+
*/
|
|
56
|
+
exports.ENTITY_REPORT_TYPES = [
|
|
57
|
+
'Person',
|
|
58
|
+
'Organization',
|
|
59
|
+
'Project',
|
|
60
|
+
'Meeting',
|
|
61
|
+
// #383: the `personal` domain's entity types (only `personal`-tagged fixtures label them).
|
|
62
|
+
'Interest',
|
|
63
|
+
'Trip',
|
|
64
|
+
'Milestone',
|
|
65
|
+
];
|
|
66
|
+
/**
|
|
67
|
+
* The report row of each item kind.
|
|
68
|
+
*
|
|
69
|
+
* @stability experimental
|
|
70
|
+
*/
|
|
71
|
+
exports.ITEM_REPORT_TYPES = {
|
|
72
|
+
commitment: 'Commitment',
|
|
73
|
+
decision: 'Decision',
|
|
74
|
+
claim: 'Claim',
|
|
75
|
+
person_fact: 'PersonFact',
|
|
76
|
+
};
|
|
77
|
+
/**
|
|
78
|
+
* Every fixed report row: the entity types, then the item kinds.
|
|
79
|
+
*
|
|
80
|
+
* @stability experimental
|
|
81
|
+
*/
|
|
82
|
+
exports.BASE_REPORT_TYPES = [...exports.ENTITY_REPORT_TYPES, ...Object.values(exports.ITEM_REPORT_TYPES)];
|
|
83
|
+
function bump(types, type, key, by = 1) {
|
|
84
|
+
const c = types[type] ?? (types[type] = { tp: 0, fp: 0, fn: 0 });
|
|
85
|
+
c[key] += by;
|
|
86
|
+
}
|
|
87
|
+
function goldNames(label, aliases) {
|
|
88
|
+
return new Set([label, ...aliases].map(text_js_1.normalizeName).filter((n) => n.length > 0));
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Similarity of one predicted label to one gold name set; 0 when not a candidate.
|
|
92
|
+
*
|
|
93
|
+
* @stability experimental
|
|
94
|
+
*/
|
|
95
|
+
function nameSimilarity(predLabel, gold) {
|
|
96
|
+
const n = (0, text_js_1.normalizeName)(predLabel);
|
|
97
|
+
if (gold.names.has(n))
|
|
98
|
+
return 1;
|
|
99
|
+
const toks = (0, text_js_1.tokenSet)(predLabel);
|
|
100
|
+
let best = 0;
|
|
101
|
+
for (const g of gold.tokenSets)
|
|
102
|
+
best = Math.max(best, (0, text_js_1.jaccard)(toks, g));
|
|
103
|
+
return best >= exports.ENTITY_MATCH_JACCARD ? best : 0;
|
|
104
|
+
}
|
|
105
|
+
/** Greedy one-to-one assignment; returns predIdx → goldIdx. */
|
|
106
|
+
function greedyAssign(candidates) {
|
|
107
|
+
const sorted = [...candidates].sort((a, b) => b.sim - a.sim || a.p - b.p || a.g - b.g);
|
|
108
|
+
const usedP = new Set();
|
|
109
|
+
const usedG = new Set();
|
|
110
|
+
const out = new Map();
|
|
111
|
+
for (const c of sorted) {
|
|
112
|
+
if (usedP.has(c.p) || usedG.has(c.g))
|
|
113
|
+
continue;
|
|
114
|
+
usedP.add(c.p);
|
|
115
|
+
usedG.add(c.g);
|
|
116
|
+
out.set(c.p, c.g);
|
|
117
|
+
}
|
|
118
|
+
return out;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* The one-to-one assignment of predicted to gold entities (greedy by descending similarity).
|
|
122
|
+
*
|
|
123
|
+
* @param gold - the gold entities (label and aliases).
|
|
124
|
+
* @param preds - the predicted entities.
|
|
125
|
+
* @param opts - `typed`: candidates must share a type (entity coverage drops the check).
|
|
126
|
+
* @returns predicted index to gold index.
|
|
127
|
+
*
|
|
128
|
+
* @stability experimental
|
|
129
|
+
*/
|
|
130
|
+
function matchEntities(gold, preds, opts) {
|
|
131
|
+
const g = gold.map((e) => {
|
|
132
|
+
const names = goldNames(e.label, e.aliases);
|
|
133
|
+
return { type: e.type, names, tokenSets: [...names].map((n) => (0, text_js_1.tokenSet)(n)) };
|
|
134
|
+
});
|
|
135
|
+
const candidates = [];
|
|
136
|
+
preds.forEach((p, pi) => {
|
|
137
|
+
g.forEach((ge, gi) => {
|
|
138
|
+
if (opts.typed && ge.type !== p.type)
|
|
139
|
+
return;
|
|
140
|
+
const sim = nameSimilarity(p.label, ge);
|
|
141
|
+
if (sim > 0)
|
|
142
|
+
candidates.push({ p: pi, g: gi, sim });
|
|
143
|
+
});
|
|
144
|
+
});
|
|
145
|
+
return greedyAssign(candidates);
|
|
146
|
+
}
|
|
147
|
+
const TRUNC = { day: 10, month: 7, year: 4 };
|
|
148
|
+
/**
|
|
149
|
+
* Whether two temporal ranges are equal at the gold precision (`unknown` matches only `unknown`).
|
|
150
|
+
*
|
|
151
|
+
* @param gold - the gold bounds and precision.
|
|
152
|
+
* @param pred - the predicted bounds and precision.
|
|
153
|
+
* @returns `true` when equal.
|
|
154
|
+
*
|
|
155
|
+
* @stability experimental
|
|
156
|
+
*/
|
|
157
|
+
function temporalEqual(gold, pred) {
|
|
158
|
+
if (gold.precision !== pred.precision)
|
|
159
|
+
return false;
|
|
160
|
+
if (gold.precision === 'unknown')
|
|
161
|
+
return true;
|
|
162
|
+
const n = TRUNC[gold.precision];
|
|
163
|
+
const t = (d) => (d === null ? null : d.slice(0, n));
|
|
164
|
+
return t(gold.validFrom) === t(pred.validFrom) && t(gold.validTo) === t(pred.validTo);
|
|
165
|
+
}
|
|
166
|
+
const day = (d) => (d ? d.slice(0, 10) : null);
|
|
167
|
+
function evidenceValid(fixture, ev) {
|
|
168
|
+
// A note citation may quote the body (`N`) or the note's Context (`C`, #440).
|
|
169
|
+
if (ev.source === 'note') {
|
|
170
|
+
return (0, text_js_1.quoteFoundIn)(ev.quote, fixture.note.body) || (fixture.contextText !== null && (0, text_js_1.quoteFoundIn)(ev.quote, fixture.contextText));
|
|
171
|
+
}
|
|
172
|
+
const seg = fixture.segments.find((s) => s.id === ev.segmentId);
|
|
173
|
+
return seg !== undefined && (0, text_js_1.quoteFoundIn)(ev.quote, seg.text);
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Scores one fixture's prediction. Pure and deterministic.
|
|
177
|
+
*
|
|
178
|
+
* @param fixture - the fixture and its labels.
|
|
179
|
+
* @param prediction - what a run predicted for it.
|
|
180
|
+
* @returns the fixture's score.
|
|
181
|
+
*
|
|
182
|
+
* @stability experimental
|
|
183
|
+
*/
|
|
184
|
+
function scoreFixture(fixture, prediction) {
|
|
185
|
+
const types = {};
|
|
186
|
+
for (const t of exports.BASE_REPORT_TYPES)
|
|
187
|
+
types[t] = { tp: 0, fp: 0, fn: 0 };
|
|
188
|
+
const goldEntities = fixture.labels.entities;
|
|
189
|
+
const knownIds = new Set(fixture.knownEntities.map((k) => k.id));
|
|
190
|
+
// --- entities -------------------------------------------------------------
|
|
191
|
+
const typedMatch = matchEntities(goldEntities, prediction.entities, { typed: true });
|
|
192
|
+
const matchedGold = new Set(typedMatch.values());
|
|
193
|
+
prediction.entities.forEach((p, pi) => {
|
|
194
|
+
if (typedMatch.has(pi))
|
|
195
|
+
bump(types, p.type, 'tp');
|
|
196
|
+
else
|
|
197
|
+
bump(types, p.type, 'fp');
|
|
198
|
+
});
|
|
199
|
+
goldEntities.forEach((g, gi) => {
|
|
200
|
+
if (!matchedGold.has(gi))
|
|
201
|
+
bump(types, g.type, 'fn');
|
|
202
|
+
});
|
|
203
|
+
const anyTypeMatch = matchEntities(goldEntities, prediction.entities, { typed: false });
|
|
204
|
+
const coverage = { matched: new Set(anyTypeMatch.values()).size, total: goldEntities.length };
|
|
205
|
+
// --- canonical identities -------------------------------------------------
|
|
206
|
+
const goldCanon = new Map();
|
|
207
|
+
for (const g of goldEntities)
|
|
208
|
+
goldCanon.set(g.key, g.existingId ?? `key:${g.key}`);
|
|
209
|
+
const canonGold = (s) => {
|
|
210
|
+
if (s === null)
|
|
211
|
+
return null;
|
|
212
|
+
return goldCanon.get(s) ?? (knownIds.has(s) ? s : `unresolved:${s}`);
|
|
213
|
+
};
|
|
214
|
+
const predCanon = new Map();
|
|
215
|
+
prediction.entities.forEach((p, pi) => {
|
|
216
|
+
const gi = typedMatch.get(pi);
|
|
217
|
+
if (gi !== undefined) {
|
|
218
|
+
predCanon.set(p.ref, canonGold(goldEntities[gi].key));
|
|
219
|
+
}
|
|
220
|
+
else if (p.resolution?.outcome === 'linked' && p.resolution.entityId) {
|
|
221
|
+
predCanon.set(p.ref, p.resolution.entityId);
|
|
222
|
+
}
|
|
223
|
+
else {
|
|
224
|
+
predCanon.set(p.ref, `pred:${p.ref}`);
|
|
225
|
+
}
|
|
226
|
+
});
|
|
227
|
+
const canonPred = (s) => {
|
|
228
|
+
if (s === null)
|
|
229
|
+
return null;
|
|
230
|
+
return predCanon.get(s) ?? (knownIds.has(s) ? s : `pred:${s}`);
|
|
231
|
+
};
|
|
232
|
+
// --- auto-link precision --------------------------------------------------
|
|
233
|
+
const autoLink = { correct: 0, total: 0, anyResolution: false };
|
|
234
|
+
prediction.entities.forEach((p, pi) => {
|
|
235
|
+
if (p.resolution)
|
|
236
|
+
autoLink.anyResolution = true;
|
|
237
|
+
if (!p.resolution || !p.resolution.prechecked || p.resolution.outcome !== 'linked')
|
|
238
|
+
return;
|
|
239
|
+
autoLink.total += 1;
|
|
240
|
+
const gi = typedMatch.get(pi);
|
|
241
|
+
const expected = gi === undefined ? undefined : goldEntities[gi].existingId;
|
|
242
|
+
if (expected && p.resolution.entityId === expected)
|
|
243
|
+
autoLink.correct += 1;
|
|
244
|
+
});
|
|
245
|
+
const temporal = { accurate: 0, total: 0 };
|
|
246
|
+
// --- relations ------------------------------------------------------------
|
|
247
|
+
const goldRels = fixture.labels.relations;
|
|
248
|
+
const usedGoldRel = new Set();
|
|
249
|
+
for (const p of prediction.relations) {
|
|
250
|
+
const pf = canonPred(p.from);
|
|
251
|
+
const pt = canonPred(p.to);
|
|
252
|
+
// #383: a symmetric relation (SPOUSE_OF, FRIEND_OF) matches either way round.
|
|
253
|
+
const symmetric = ontology_1.ONTOLOGY.relationType(p.type)?.symmetric === true;
|
|
254
|
+
const gi = goldRels.findIndex((g, i) => !usedGoldRel.has(i) &&
|
|
255
|
+
g.type === p.type &&
|
|
256
|
+
((canonGold(g.from) === pf && canonGold(g.to) === pt) ||
|
|
257
|
+
(symmetric && canonGold(g.from) === pt && canonGold(g.to) === pf)));
|
|
258
|
+
const row = `relation:${p.type}`;
|
|
259
|
+
if (gi === -1) {
|
|
260
|
+
bump(types, row, 'fp');
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
usedGoldRel.add(gi);
|
|
264
|
+
bump(types, row, 'tp');
|
|
265
|
+
const g = goldRels[gi];
|
|
266
|
+
if (ontology_1.ONTOLOGY.relationType(g.type)?.temporal) {
|
|
267
|
+
temporal.total += 1;
|
|
268
|
+
if (temporalEqual(g, p))
|
|
269
|
+
temporal.accurate += 1;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
goldRels.forEach((g, i) => {
|
|
273
|
+
if (!usedGoldRel.has(i))
|
|
274
|
+
bump(types, `relation:${g.type}`, 'fn');
|
|
275
|
+
});
|
|
276
|
+
// --- items ----------------------------------------------------------------
|
|
277
|
+
const goldItems = fixture.labels.items;
|
|
278
|
+
const itemCandidates = [];
|
|
279
|
+
prediction.items.forEach((p, pi) => {
|
|
280
|
+
goldItems.forEach((g, gi) => {
|
|
281
|
+
if (g.kind !== p.kind)
|
|
282
|
+
return;
|
|
283
|
+
if (canonGold(g.subject) !== canonPred(p.subject))
|
|
284
|
+
return;
|
|
285
|
+
if (g.kind === 'commitment' && g.owner !== null && canonGold(g.owner) !== canonPred(p.owner))
|
|
286
|
+
return;
|
|
287
|
+
const f1 = (0, text_js_1.tokenF1)(g.statement, p.statement);
|
|
288
|
+
if (f1 >= exports.ITEM_STATEMENT_F1)
|
|
289
|
+
itemCandidates.push({ p: pi, g: gi, sim: f1 });
|
|
290
|
+
});
|
|
291
|
+
});
|
|
292
|
+
const itemMatch = greedyAssign(itemCandidates);
|
|
293
|
+
const matchedGoldItems = new Set(itemMatch.values());
|
|
294
|
+
prediction.items.forEach((p, pi) => {
|
|
295
|
+
const row = exports.ITEM_REPORT_TYPES[p.kind];
|
|
296
|
+
const gi = itemMatch.get(pi);
|
|
297
|
+
if (gi === undefined) {
|
|
298
|
+
bump(types, row, 'fp');
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
301
|
+
bump(types, row, 'tp');
|
|
302
|
+
const g = goldItems[gi];
|
|
303
|
+
if (g.occurredAt !== null || g.dueAt !== null) {
|
|
304
|
+
temporal.total += 1;
|
|
305
|
+
if (day(p.occurredAt) === g.occurredAt && day(p.dueAt) === g.dueAt)
|
|
306
|
+
temporal.accurate += 1;
|
|
307
|
+
}
|
|
308
|
+
});
|
|
309
|
+
goldItems.forEach((g, gi) => {
|
|
310
|
+
if (!matchedGoldItems.has(gi))
|
|
311
|
+
bump(types, exports.ITEM_REPORT_TYPES[g.kind], 'fn');
|
|
312
|
+
});
|
|
313
|
+
// --- evidence validity ----------------------------------------------------
|
|
314
|
+
const evidence = { valid: 0, total: 0 };
|
|
315
|
+
const allEvidence = [
|
|
316
|
+
...prediction.entities.flatMap((e) => e.evidence),
|
|
317
|
+
...prediction.relations.flatMap((r) => r.evidence),
|
|
318
|
+
...prediction.items.flatMap((i) => i.evidence),
|
|
319
|
+
];
|
|
320
|
+
for (const ev of allEvidence) {
|
|
321
|
+
evidence.total += 1;
|
|
322
|
+
if (evidenceValid(fixture, ev))
|
|
323
|
+
evidence.valid += 1;
|
|
324
|
+
}
|
|
325
|
+
return {
|
|
326
|
+
fixtureId: fixture.id,
|
|
327
|
+
types,
|
|
328
|
+
coverage,
|
|
329
|
+
autoLink,
|
|
330
|
+
evidence,
|
|
331
|
+
temporal,
|
|
332
|
+
stats: { ...prediction.stats },
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Scores every fixture of a run (a fixture with no prediction scores as empty).
|
|
337
|
+
*
|
|
338
|
+
* @param fixtures - the fixtures.
|
|
339
|
+
* @param predictions - the predictions by fixture id.
|
|
340
|
+
* @returns the run's score.
|
|
341
|
+
*
|
|
342
|
+
* @stability experimental
|
|
343
|
+
*/
|
|
344
|
+
function scoreRun(fixtures, predictions) {
|
|
345
|
+
const run = {
|
|
346
|
+
fixtures: fixtures.length,
|
|
347
|
+
types: {},
|
|
348
|
+
coverage: { matched: 0, total: 0 },
|
|
349
|
+
autoLink: { correct: 0, total: 0, anyResolution: false },
|
|
350
|
+
evidence: { valid: 0, total: 0 },
|
|
351
|
+
temporal: { accurate: 0, total: 0 },
|
|
352
|
+
stats: {},
|
|
353
|
+
perFixture: [],
|
|
354
|
+
};
|
|
355
|
+
for (const t of exports.BASE_REPORT_TYPES)
|
|
356
|
+
run.types[t] = { tp: 0, fp: 0, fn: 0 };
|
|
357
|
+
for (const f of fixtures) {
|
|
358
|
+
const pred = predictions.get(f.id) ?? {
|
|
359
|
+
fixtureId: f.id,
|
|
360
|
+
model: null,
|
|
361
|
+
entities: [],
|
|
362
|
+
relations: [],
|
|
363
|
+
items: [],
|
|
364
|
+
stats: {},
|
|
365
|
+
};
|
|
366
|
+
const s = scoreFixture(f, pred);
|
|
367
|
+
run.perFixture.push(s);
|
|
368
|
+
for (const [type, c] of Object.entries(s.types)) {
|
|
369
|
+
bump(run.types, type, 'tp', c.tp);
|
|
370
|
+
bump(run.types, type, 'fp', c.fp);
|
|
371
|
+
bump(run.types, type, 'fn', c.fn);
|
|
372
|
+
}
|
|
373
|
+
run.coverage.matched += s.coverage.matched;
|
|
374
|
+
run.coverage.total += s.coverage.total;
|
|
375
|
+
run.autoLink.correct += s.autoLink.correct;
|
|
376
|
+
run.autoLink.total += s.autoLink.total;
|
|
377
|
+
run.autoLink.anyResolution ||= s.autoLink.anyResolution;
|
|
378
|
+
run.evidence.valid += s.evidence.valid;
|
|
379
|
+
run.evidence.total += s.evidence.total;
|
|
380
|
+
run.temporal.accurate += s.temporal.accurate;
|
|
381
|
+
run.temporal.total += s.temporal.total;
|
|
382
|
+
for (const [k, v] of Object.entries(s.stats))
|
|
383
|
+
run.stats[k] = (run.stats[k] ?? 0) + v;
|
|
384
|
+
}
|
|
385
|
+
return run;
|
|
386
|
+
}
|
|
387
|
+
// --- ratios -----------------------------------------------------------------
|
|
388
|
+
/**
|
|
389
|
+
* `num / den`, or `null` (printed `n/a`) when `den` is 0.
|
|
390
|
+
*
|
|
391
|
+
* @stability experimental
|
|
392
|
+
*/
|
|
393
|
+
const ratio = (num, den) => (den === 0 ? null : num / den);
|
|
394
|
+
exports.ratio = ratio;
|
|
395
|
+
/**
|
|
396
|
+
* Precision of one row, or `null` with nothing predicted.
|
|
397
|
+
*
|
|
398
|
+
* @param c - the row's counts.
|
|
399
|
+
* @returns the ratio or `null`.
|
|
400
|
+
*
|
|
401
|
+
* @stability experimental
|
|
402
|
+
*/
|
|
403
|
+
function precisionOf(c) {
|
|
404
|
+
return (0, exports.ratio)(c.tp, c.tp + c.fp);
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Recall of one row, or `null` with nothing labelled.
|
|
408
|
+
*
|
|
409
|
+
* @param c - the row's counts.
|
|
410
|
+
* @returns the ratio or `null`.
|
|
411
|
+
*
|
|
412
|
+
* @stability experimental
|
|
413
|
+
*/
|
|
414
|
+
function recallOf(c) {
|
|
415
|
+
return (0, exports.ratio)(c.tp, c.tp + c.fn);
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* F1 of one row, or `null` when precision or recall is.
|
|
419
|
+
*
|
|
420
|
+
* @param c - the row's counts.
|
|
421
|
+
* @returns the ratio or `null`.
|
|
422
|
+
*
|
|
423
|
+
* @stability experimental
|
|
424
|
+
*/
|
|
425
|
+
function f1Of(c) {
|
|
426
|
+
const p = precisionOf(c);
|
|
427
|
+
const r = recallOf(c);
|
|
428
|
+
if (p === null || r === null)
|
|
429
|
+
return null;
|
|
430
|
+
return p + r === 0 ? 0 : (2 * p * r) / (p + r);
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* Gold entities matched by any prediction, type ignored, over all gold entities (the §6 entity-coverage target).
|
|
434
|
+
*
|
|
435
|
+
* @param run - the run's coverage counts.
|
|
436
|
+
* @returns the ratio or `null`.
|
|
437
|
+
*
|
|
438
|
+
* @stability experimental
|
|
439
|
+
*/
|
|
440
|
+
function entityCoverage(run) {
|
|
441
|
+
return (0, exports.ratio)(run.coverage.matched, run.coverage.total);
|
|
442
|
+
}
|
|
443
|
+
/**
|
|
444
|
+
* Recall of the Commitment row (the §6 commitment-recall target).
|
|
445
|
+
*
|
|
446
|
+
* @param run - the run's per-type counts.
|
|
447
|
+
* @returns the ratio or `null`.
|
|
448
|
+
*
|
|
449
|
+
* @stability experimental
|
|
450
|
+
*/
|
|
451
|
+
function commitmentRecall(run) {
|
|
452
|
+
return recallOf(run.types.Commitment ?? { tp: 0, fp: 0, fn: 0 });
|
|
453
|
+
}
|
|
454
|
+
/**
|
|
455
|
+
* `null` (printed `n/a`) when no prediction carries a resolution, or none was a prechecked link.
|
|
456
|
+
*
|
|
457
|
+
* @stability experimental
|
|
458
|
+
*/
|
|
459
|
+
function autoLinkPrecision(run) {
|
|
460
|
+
if (!run.autoLink.anyResolution)
|
|
461
|
+
return null;
|
|
462
|
+
return (0, exports.ratio)(run.autoLink.correct, run.autoLink.total);
|
|
463
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The §6 quality bar (docs/specs/ontology.md). Gates on relaxing §7's resolution
|
|
3
|
+
* thresholds, not on shipping: CI reports, `--enforce` fails.
|
|
4
|
+
*
|
|
5
|
+
* @stability experimental
|
|
6
|
+
*/
|
|
7
|
+
export declare const KG_EVAL_TARGETS: {
|
|
8
|
+
readonly autoLinkPrecision: 0.95;
|
|
9
|
+
readonly commitmentRecall: 0.85;
|
|
10
|
+
readonly entityCoverage: 0.9;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* One of the {@link KG_EVAL_TARGETS} keys.
|
|
14
|
+
*
|
|
15
|
+
* @stability experimental
|
|
16
|
+
*/
|
|
17
|
+
export type KgEvalTargetKey = keyof typeof KG_EVAL_TARGETS;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.KG_EVAL_TARGETS = void 0;
|
|
4
|
+
// The §6 quality bar (docs/specs/ontology.md). Gates on relaxing §7's
|
|
5
|
+
// resolution thresholds — NOT gates on shipping; CI reports, `--enforce` fails.
|
|
6
|
+
/**
|
|
7
|
+
* The §6 quality bar (docs/specs/ontology.md). Gates on relaxing §7's resolution
|
|
8
|
+
* thresholds, not on shipping: CI reports, `--enforce` fails.
|
|
9
|
+
*
|
|
10
|
+
* @stability experimental
|
|
11
|
+
*/
|
|
12
|
+
exports.KG_EVAL_TARGETS = {
|
|
13
|
+
autoLinkPrecision: 0.95,
|
|
14
|
+
commitmentRecall: 0.85,
|
|
15
|
+
entityCoverage: 0.9,
|
|
16
|
+
};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Collapse every whitespace run to one space and trim.
|
|
3
|
+
*
|
|
4
|
+
* @stability experimental
|
|
5
|
+
*/
|
|
6
|
+
export declare function collapseWhitespace(s: string): string;
|
|
7
|
+
/**
|
|
8
|
+
* Whether `quote` occurs verbatim in `haystack`, after whitespace collapse on both.
|
|
9
|
+
*
|
|
10
|
+
* @stability experimental
|
|
11
|
+
*/
|
|
12
|
+
export declare function quoteFoundIn(quote: string, haystack: string): boolean;
|
|
13
|
+
/**
|
|
14
|
+
* NFKC → lower-case → strip punctuation except an inner `'`/`-` (one joining
|
|
15
|
+
* two letters/digits) → collapse whitespace. "Sarah Chen," → "sarah chen";
|
|
16
|
+
* "O'Neil-Smith" keeps both joiners.
|
|
17
|
+
*
|
|
18
|
+
* @stability experimental
|
|
19
|
+
*/
|
|
20
|
+
export declare function normalizeName(s: string): string;
|
|
21
|
+
/**
|
|
22
|
+
* The set of normalized tokens of a string.
|
|
23
|
+
*
|
|
24
|
+
* @param s - the string.
|
|
25
|
+
* @returns its tokens.
|
|
26
|
+
*
|
|
27
|
+
* @stability experimental
|
|
28
|
+
*/
|
|
29
|
+
export declare function tokenSet(s: string): Set<string>;
|
|
30
|
+
/**
|
|
31
|
+
* Jaccard similarity of two token sets.
|
|
32
|
+
*
|
|
33
|
+
* @param a - one set.
|
|
34
|
+
* @param b - the other.
|
|
35
|
+
* @returns |A∩B| / |A∪B| (1 for two empty sets).
|
|
36
|
+
*
|
|
37
|
+
* @stability experimental
|
|
38
|
+
*/
|
|
39
|
+
export declare function jaccard(a: Set<string>, b: Set<string>): number;
|
|
40
|
+
/**
|
|
41
|
+
* Token-set F1: 2·|A∩B| / (|A|+|B|).
|
|
42
|
+
*
|
|
43
|
+
* @stability experimental
|
|
44
|
+
*/
|
|
45
|
+
export declare function tokenF1(a: string, b: string): number;
|
package/dist/kg/text.js
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Pure text helpers shared by the scorer, the loader and the fixture test.
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.collapseWhitespace = collapseWhitespace;
|
|
5
|
+
exports.quoteFoundIn = quoteFoundIn;
|
|
6
|
+
exports.normalizeName = normalizeName;
|
|
7
|
+
exports.tokenSet = tokenSet;
|
|
8
|
+
exports.jaccard = jaccard;
|
|
9
|
+
exports.tokenF1 = tokenF1;
|
|
10
|
+
/**
|
|
11
|
+
* Collapse every whitespace run to one space and trim.
|
|
12
|
+
*
|
|
13
|
+
* @stability experimental
|
|
14
|
+
*/
|
|
15
|
+
function collapseWhitespace(s) {
|
|
16
|
+
return s.replace(/\s+/g, ' ').trim();
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Whether `quote` occurs verbatim in `haystack`, after whitespace collapse on both.
|
|
20
|
+
*
|
|
21
|
+
* @stability experimental
|
|
22
|
+
*/
|
|
23
|
+
function quoteFoundIn(quote, haystack) {
|
|
24
|
+
const q = collapseWhitespace(quote);
|
|
25
|
+
return q.length > 0 && collapseWhitespace(haystack).includes(q);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* NFKC → lower-case → strip punctuation except an inner `'`/`-` (one joining
|
|
29
|
+
* two letters/digits) → collapse whitespace. "Sarah Chen," → "sarah chen";
|
|
30
|
+
* "O'Neil-Smith" keeps both joiners.
|
|
31
|
+
*
|
|
32
|
+
* @stability experimental
|
|
33
|
+
*/
|
|
34
|
+
function normalizeName(s) {
|
|
35
|
+
const lowered = s.normalize('NFKC').toLowerCase().replace(/[‘’]/g, "'");
|
|
36
|
+
let out = '';
|
|
37
|
+
for (let i = 0; i < lowered.length; i += 1) {
|
|
38
|
+
const ch = lowered[i];
|
|
39
|
+
if (/[\p{L}\p{N}\s]/u.test(ch)) {
|
|
40
|
+
out += ch;
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
if ((ch === "'" || ch === '-') && i > 0 && i < lowered.length - 1) {
|
|
44
|
+
const prev = lowered[i - 1];
|
|
45
|
+
const next = lowered[i + 1];
|
|
46
|
+
if (/[\p{L}\p{N}]/u.test(prev) && /[\p{L}\p{N}]/u.test(next)) {
|
|
47
|
+
out += ch;
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
out += ' ';
|
|
52
|
+
}
|
|
53
|
+
return out.replace(/\s+/g, ' ').trim();
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* The set of normalized tokens of a string.
|
|
57
|
+
*
|
|
58
|
+
* @param s - the string.
|
|
59
|
+
* @returns its tokens.
|
|
60
|
+
*
|
|
61
|
+
* @stability experimental
|
|
62
|
+
*/
|
|
63
|
+
function tokenSet(s) {
|
|
64
|
+
const n = normalizeName(s);
|
|
65
|
+
return new Set(n.length === 0 ? [] : n.split(' '));
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Jaccard similarity of two token sets.
|
|
69
|
+
*
|
|
70
|
+
* @param a - one set.
|
|
71
|
+
* @param b - the other.
|
|
72
|
+
* @returns |A∩B| / |A∪B| (1 for two empty sets).
|
|
73
|
+
*
|
|
74
|
+
* @stability experimental
|
|
75
|
+
*/
|
|
76
|
+
function jaccard(a, b) {
|
|
77
|
+
if (a.size === 0 && b.size === 0)
|
|
78
|
+
return 1;
|
|
79
|
+
let inter = 0;
|
|
80
|
+
for (const t of a)
|
|
81
|
+
if (b.has(t))
|
|
82
|
+
inter += 1;
|
|
83
|
+
return inter / (a.size + b.size - inter);
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Token-set F1: 2·|A∩B| / (|A|+|B|).
|
|
87
|
+
*
|
|
88
|
+
* @stability experimental
|
|
89
|
+
*/
|
|
90
|
+
function tokenF1(a, b) {
|
|
91
|
+
const A = tokenSet(a);
|
|
92
|
+
const B = tokenSet(b);
|
|
93
|
+
if (A.size === 0 && B.size === 0)
|
|
94
|
+
return 1;
|
|
95
|
+
let inter = 0;
|
|
96
|
+
for (const t of A)
|
|
97
|
+
if (B.has(t))
|
|
98
|
+
inter += 1;
|
|
99
|
+
return (2 * inter) / (A.size + B.size);
|
|
100
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { AiProviderAdapter } from '@marinoscar/platform-api/ai';
|
|
2
|
+
/**
|
|
3
|
+
* A provider id {@link platformAdapter} can build.
|
|
4
|
+
*
|
|
5
|
+
* @stability experimental
|
|
6
|
+
*/
|
|
7
|
+
export type PlatformAdapterProvider = 'openai' | 'openai-compatible';
|
|
8
|
+
/**
|
|
9
|
+
* The platform's adapter for a provider, constructed outside Nest.
|
|
10
|
+
*
|
|
11
|
+
* @param provider - `openai` or `openai-compatible`.
|
|
12
|
+
* @returns the adapter (`responses`, `embeddings`, ...).
|
|
13
|
+
* @throws Error when the installed platform no longer carries the adapter where this shim looks.
|
|
14
|
+
* @stability experimental
|
|
15
|
+
*/
|
|
16
|
+
export declare function platformAdapter(provider: PlatformAdapterProvider): AiProviderAdapter;
|