@jungjaehoon/mama-core 1.4.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/026-create-canonical-entity-tables.sql +99 -0
- package/db/migrations/027-create-entity-review-support.sql +44 -0
- package/db/migrations/028-create-entity-audit-tables.sql +45 -0
- package/db/migrations/029-extend-entity-observations-observation-type.sql +60 -0
- package/db/migrations/030-case-first-memory-substrate.sql +901 -0
- package/db/migrations/031-wiki-page-index-provenance-columns.sql +2 -0
- package/dist/canonicalize.d.ts +38 -0
- package/dist/canonicalize.js +106 -0
- package/dist/cases/case-links.d.ts +116 -0
- package/dist/cases/case-links.js +794 -0
- package/dist/cases/composition-overrides.d.ts +90 -0
- package/dist/cases/composition-overrides.js +724 -0
- package/dist/cases/corrections.d.ts +156 -0
- package/dist/cases/corrections.js +1045 -0
- package/dist/cases/freshness.d.ts +57 -0
- package/dist/cases/freshness.js +283 -0
- package/dist/cases/live-state.d.ts +48 -0
- package/dist/cases/live-state.js +256 -0
- package/dist/cases/membership-explain.d.ts +56 -0
- package/dist/cases/membership-explain.js +140 -0
- package/dist/cases/membership-matcher.d.ts +23 -0
- package/dist/cases/membership-matcher.js +193 -0
- package/dist/cases/merge-split.d.ts +61 -0
- package/dist/cases/merge-split.js +368 -0
- package/dist/cases/role-inference.d.ts +10 -0
- package/dist/cases/role-inference.js +85 -0
- package/dist/cases/search-rollup.d.ts +23 -0
- package/dist/cases/search-rollup.js +225 -0
- package/dist/cases/sqlite-transaction.d.ts +10 -0
- package/dist/cases/sqlite-transaction.js +62 -0
- package/dist/cases/store.d.ts +74 -0
- package/dist/cases/store.js +816 -0
- package/dist/cases/target-ref.d.ts +25 -0
- package/dist/cases/target-ref.js +22 -0
- package/dist/cases/timeline-range.d.ts +6 -0
- package/dist/cases/timeline-range.js +322 -0
- package/dist/cases/tombstone-sweeper.d.ts +13 -0
- package/dist/cases/tombstone-sweeper.js +128 -0
- package/dist/cases/types.d.ts +244 -0
- package/dist/cases/types.js +47 -0
- package/dist/cases/wiki-page-index.d.ts +50 -0
- package/dist/cases/wiki-page-index.js +350 -0
- package/dist/config-loader.d.ts +1 -0
- package/dist/config-loader.js +43 -0
- package/dist/connectors/event-index.d.ts +44 -0
- package/dist/connectors/event-index.js +366 -0
- package/dist/connectors/types.d.ts +69 -0
- package/dist/connectors/types.js +3 -0
- package/dist/db-manager.d.ts +4 -0
- package/dist/db-manager.js +10 -3
- package/dist/decision-formatter.d.ts +2 -0
- package/dist/decision-formatter.js +11 -8
- package/dist/embedding-server/mobile/daemon.d.ts +1 -0
- package/dist/embedding-server/mobile/daemon.js +11 -1
- package/dist/entities/audit-metrics.d.ts +73 -0
- package/dist/entities/audit-metrics.js +128 -0
- package/dist/entities/candidate-generator.d.ts +9 -0
- package/dist/entities/candidate-generator.js +436 -0
- package/dist/entities/entity-impact.d.ts +41 -0
- package/dist/entities/entity-impact.js +140 -0
- package/dist/entities/entity-linked-decision-counts.d.ts +2 -0
- package/dist/entities/entity-linked-decision-counts.js +29 -0
- package/dist/entities/entity-list.d.ts +23 -0
- package/dist/entities/entity-list.js +99 -0
- package/dist/entities/entity-orphan-list.d.ts +26 -0
- package/dist/entities/entity-orphan-list.js +91 -0
- package/dist/entities/entity-search.d.ts +21 -0
- package/dist/entities/entity-search.js +150 -0
- package/dist/entities/errors.d.ts +65 -0
- package/dist/entities/errors.js +124 -0
- package/dist/entities/exact-merge-backfill.d.ts +11 -0
- package/dist/entities/exact-merge-backfill.js +161 -0
- package/dist/entities/lineage-backfill.d.ts +12 -0
- package/dist/entities/lineage-backfill.js +101 -0
- package/dist/entities/lineage-store.d.ts +51 -0
- package/dist/entities/lineage-store.js +308 -0
- package/dist/entities/normalization.d.ts +14 -0
- package/dist/entities/normalization.js +63 -0
- package/dist/entities/policy-store.d.ts +17 -0
- package/dist/entities/policy-store.js +263 -0
- package/dist/entities/policy-types.d.ts +72 -0
- package/dist/entities/policy-types.js +59 -0
- package/dist/entities/projection.d.ts +8 -0
- package/dist/entities/projection.js +82 -0
- package/dist/entities/provenance-query.d.ts +33 -0
- package/dist/entities/provenance-query.js +167 -0
- package/dist/entities/read-identity.d.ts +31 -0
- package/dist/entities/read-identity.js +264 -0
- package/dist/entities/recall-bridge.d.ts +5 -0
- package/dist/entities/recall-bridge.js +171 -0
- package/dist/entities/resolution-engine.d.ts +15 -0
- package/dist/entities/resolution-engine.js +69 -0
- package/dist/entities/rollback-preview.d.ts +39 -0
- package/dist/entities/rollback-preview.js +235 -0
- package/dist/entities/source-locator.d.ts +3 -0
- package/dist/entities/source-locator.js +32 -0
- package/dist/entities/store.d.ts +75 -0
- package/dist/entities/store.js +465 -0
- package/dist/entities/types.d.ts +162 -0
- package/dist/entities/types.js +30 -0
- package/dist/index.d.ts +47 -2
- package/dist/index.js +66 -25
- package/dist/mama-api.d.ts +22 -1
- package/dist/mama-api.js +272 -64
- package/dist/memory/api.d.ts +28 -1
- package/dist/memory/api.js +293 -31
- package/dist/memory/types.d.ts +11 -2
- package/dist/search/feedback-store.d.ts +59 -0
- package/dist/search/feedback-store.js +211 -0
- package/dist/search/question-type.d.ts +5 -0
- package/dist/search/question-type.js +41 -0
- package/dist/search/ranker-features.d.ts +23 -0
- package/dist/search/ranker-features.js +104 -0
- package/dist/search/ranker-rescore.d.ts +31 -0
- package/dist/search/ranker-rescore.js +226 -0
- package/dist/search/ranker-trainer.d.ts +94 -0
- package/dist/search/ranker-trainer.js +860 -0
- package/dist/test-utils.d.ts +14 -0
- package/dist/test-utils.js +52 -0
- package/package.json +21 -3
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ENTITY_EMBEDDING_TOPN = void 0;
|
|
4
|
+
exports.dedupeObservationsBySource = dedupeObservationsBySource;
|
|
5
|
+
exports.generateResolutionCandidates = generateResolutionCandidates;
|
|
6
|
+
const errors_js_1 = require("./errors.js");
|
|
7
|
+
const normalization_js_1 = require("./normalization.js");
|
|
8
|
+
exports.ENTITY_EMBEDDING_TOPN = 50;
|
|
9
|
+
const CROSS_SCOPE_PROBE_MIN_EMBEDDING_SCORE = 0.84;
|
|
10
|
+
const MAX_DETERMINISTIC_SCOPE_BUCKET = 32;
|
|
11
|
+
const MAX_CROSS_SCOPE_SCOPE_BUCKET = 8;
|
|
12
|
+
function tokenize(input) {
|
|
13
|
+
return input
|
|
14
|
+
.toLowerCase()
|
|
15
|
+
.split(/[\s,.!?;:()[\]{}"'/_-]+/u)
|
|
16
|
+
.map((token) => token.trim())
|
|
17
|
+
.filter((token) => token.length > 1);
|
|
18
|
+
}
|
|
19
|
+
function contextScore(left, right) {
|
|
20
|
+
const leftTokens = new Set(tokenize(left.context_summary ?? ''));
|
|
21
|
+
const rightTokens = new Set(tokenize(right.context_summary ?? ''));
|
|
22
|
+
if (leftTokens.size === 0 || rightTokens.size === 0) {
|
|
23
|
+
return 0;
|
|
24
|
+
}
|
|
25
|
+
let overlap = 0;
|
|
26
|
+
for (const token of leftTokens) {
|
|
27
|
+
if (rightTokens.has(token)) {
|
|
28
|
+
overlap += 1;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return overlap / Math.max(Math.min(leftTokens.size, rightTokens.size), 1);
|
|
32
|
+
}
|
|
33
|
+
function structuralScore(left, right) {
|
|
34
|
+
if (left.normalized_form === right.normalized_form) {
|
|
35
|
+
return 1;
|
|
36
|
+
}
|
|
37
|
+
const leftIds = (0, normalization_js_1.extractStructuredIdentifiers)(left.surface_form);
|
|
38
|
+
const rightIds = (0, normalization_js_1.extractStructuredIdentifiers)(right.surface_form);
|
|
39
|
+
const hasSharedEmail = leftIds.emails.some((value) => rightIds.emails.includes(value));
|
|
40
|
+
const hasSharedHandle = leftIds.handles.some((value) => rightIds.handles.includes(value));
|
|
41
|
+
const hasSharedDomain = leftIds.domains.some((value) => rightIds.domains.includes(value));
|
|
42
|
+
if (hasSharedEmail || hasSharedHandle || hasSharedDomain) {
|
|
43
|
+
return 0.75;
|
|
44
|
+
}
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
function stringScore(left, right) {
|
|
48
|
+
if (left.normalized_form === right.normalized_form) {
|
|
49
|
+
return 1;
|
|
50
|
+
}
|
|
51
|
+
const leftText = left.surface_form.toLowerCase();
|
|
52
|
+
const rightText = right.surface_form.toLowerCase();
|
|
53
|
+
const sharedPrefix = leftText.length > 0 && rightText.length > 0 && leftText[0] === rightText[0];
|
|
54
|
+
return sharedPrefix ? 0.1 : 0;
|
|
55
|
+
}
|
|
56
|
+
function scorePair(left, right, probeKind) {
|
|
57
|
+
const score_structural = structuralScore(left, right);
|
|
58
|
+
const score_string = stringScore(left, right);
|
|
59
|
+
const score_context = contextScore(left, right);
|
|
60
|
+
const score_graph = 0;
|
|
61
|
+
const score_total = score_structural + score_string + score_context + score_graph;
|
|
62
|
+
return {
|
|
63
|
+
left,
|
|
64
|
+
right,
|
|
65
|
+
score_structural,
|
|
66
|
+
score_string,
|
|
67
|
+
score_context,
|
|
68
|
+
score_graph,
|
|
69
|
+
score_total,
|
|
70
|
+
probe_kind: probeKind,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
function shouldConsiderPair(left, right) {
|
|
74
|
+
if (left.entity_kind_hint &&
|
|
75
|
+
right.entity_kind_hint &&
|
|
76
|
+
left.entity_kind_hint !== right.entity_kind_hint) {
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
if (left.scope_kind !== right.scope_kind) {
|
|
80
|
+
return false;
|
|
81
|
+
}
|
|
82
|
+
if (left.scope_id !== right.scope_id) {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
function shouldConsiderCrossScopeProbe(left, right) {
|
|
88
|
+
if (!left.entity_kind_hint ||
|
|
89
|
+
!right.entity_kind_hint ||
|
|
90
|
+
left.entity_kind_hint !== right.entity_kind_hint) {
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
if (left.observation_type !== right.observation_type) {
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
if (left.scope_kind !== right.scope_kind) {
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
if (left.scope_id === right.scope_id) {
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
return true;
|
|
103
|
+
}
|
|
104
|
+
function buildBlockKeys(observation) {
|
|
105
|
+
const keys = new Set();
|
|
106
|
+
keys.add(`norm:${observation.normalized_form}`);
|
|
107
|
+
for (const related of observation.related_surface_forms) {
|
|
108
|
+
keys.add(`related:${related}`);
|
|
109
|
+
}
|
|
110
|
+
const ids = (0, normalization_js_1.extractStructuredIdentifiers)(observation.surface_form);
|
|
111
|
+
for (const email of ids.emails) {
|
|
112
|
+
keys.add(`email:${email}`);
|
|
113
|
+
}
|
|
114
|
+
for (const handle of ids.handles) {
|
|
115
|
+
keys.add(`handle:${handle}`);
|
|
116
|
+
}
|
|
117
|
+
for (const domain of ids.domains) {
|
|
118
|
+
keys.add(`domain:${domain}`);
|
|
119
|
+
}
|
|
120
|
+
const contextTokens = tokenize(observation.context_summary ?? '').slice(0, 3);
|
|
121
|
+
for (const token of contextTokens) {
|
|
122
|
+
keys.add(`ctx:${observation.scope_kind}:${observation.scope_id ?? 'global'}:${token}`);
|
|
123
|
+
}
|
|
124
|
+
return Array.from(keys);
|
|
125
|
+
}
|
|
126
|
+
function normalizeSurfaceForm(value) {
|
|
127
|
+
if (value.trim().length === 0) {
|
|
128
|
+
return '';
|
|
129
|
+
}
|
|
130
|
+
return (0, normalization_js_1.normalizeEntityLabel)(value).normalized;
|
|
131
|
+
}
|
|
132
|
+
function sharedNormalizedRelatedCount(left, right) {
|
|
133
|
+
const leftRelated = new Set(left.related_surface_forms.map((value) => normalizeSurfaceForm(value)));
|
|
134
|
+
const rightRelated = new Set(right.related_surface_forms.map((value) => normalizeSurfaceForm(value)));
|
|
135
|
+
if (leftRelated.size === 0 || rightRelated.size === 0) {
|
|
136
|
+
return 0;
|
|
137
|
+
}
|
|
138
|
+
let overlap = 0;
|
|
139
|
+
for (const related of leftRelated) {
|
|
140
|
+
if (rightRelated.has(related)) {
|
|
141
|
+
overlap += 1;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return overlap;
|
|
145
|
+
}
|
|
146
|
+
function getTimestampDistance(left, right) {
|
|
147
|
+
if (left.timestamp_observed === null || right.timestamp_observed === null) {
|
|
148
|
+
return Number.POSITIVE_INFINITY;
|
|
149
|
+
}
|
|
150
|
+
return Math.abs(left.timestamp_observed - right.timestamp_observed);
|
|
151
|
+
}
|
|
152
|
+
function buildPairKey(left, right) {
|
|
153
|
+
return `${left.id}::${right.id}`;
|
|
154
|
+
}
|
|
155
|
+
function compareObservationForSampling(left, right) {
|
|
156
|
+
const leftRelated = left.related_surface_forms.length;
|
|
157
|
+
const rightRelated = right.related_surface_forms.length;
|
|
158
|
+
if (rightRelated !== leftRelated) {
|
|
159
|
+
return rightRelated - leftRelated;
|
|
160
|
+
}
|
|
161
|
+
const leftContext = left.context_summary?.length ?? 0;
|
|
162
|
+
const rightContext = right.context_summary?.length ?? 0;
|
|
163
|
+
if (rightContext !== leftContext) {
|
|
164
|
+
return rightContext - leftContext;
|
|
165
|
+
}
|
|
166
|
+
const leftObserved = left.timestamp_observed ?? Number.NEGATIVE_INFINITY;
|
|
167
|
+
const rightObserved = right.timestamp_observed ?? Number.NEGATIVE_INFINITY;
|
|
168
|
+
if (rightObserved !== leftObserved) {
|
|
169
|
+
return rightObserved - leftObserved;
|
|
170
|
+
}
|
|
171
|
+
return left.id.localeCompare(right.id);
|
|
172
|
+
}
|
|
173
|
+
function capObservationGroup(observations, limit) {
|
|
174
|
+
if (observations.length <= limit) {
|
|
175
|
+
return observations;
|
|
176
|
+
}
|
|
177
|
+
return [...observations].sort(compareObservationForSampling).slice(0, limit);
|
|
178
|
+
}
|
|
179
|
+
function pushTopProbe(probes, candidate, maxPairs) {
|
|
180
|
+
probes.push(candidate);
|
|
181
|
+
probes.sort(compareProbePairs);
|
|
182
|
+
if (probes.length > maxPairs) {
|
|
183
|
+
probes.length = maxPairs;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
function buildCrossLanguageFamilyKey(left, right) {
|
|
187
|
+
if (!left.lang || !right.lang || left.lang === right.lang) {
|
|
188
|
+
return null;
|
|
189
|
+
}
|
|
190
|
+
const forms = [
|
|
191
|
+
`${left.lang}:${left.normalized_form}`,
|
|
192
|
+
`${right.lang}:${right.normalized_form}`,
|
|
193
|
+
].sort();
|
|
194
|
+
const observationTypes = [left.observation_type, right.observation_type].sort();
|
|
195
|
+
return JSON.stringify([
|
|
196
|
+
left.entity_kind_hint ?? right.entity_kind_hint ?? 'unknown',
|
|
197
|
+
observationTypes,
|
|
198
|
+
forms,
|
|
199
|
+
]);
|
|
200
|
+
}
|
|
201
|
+
function compareProbePairs(left, right) {
|
|
202
|
+
const relatedDelta = sharedNormalizedRelatedCount(right.left, right.right) -
|
|
203
|
+
sharedNormalizedRelatedCount(left.left, left.right);
|
|
204
|
+
if (relatedDelta !== 0) {
|
|
205
|
+
return relatedDelta;
|
|
206
|
+
}
|
|
207
|
+
if (right.score_total !== left.score_total) {
|
|
208
|
+
return right.score_total - left.score_total;
|
|
209
|
+
}
|
|
210
|
+
const timestampDelta = getTimestampDistance(left.left, left.right) - getTimestampDistance(right.left, right.right);
|
|
211
|
+
if (timestampDelta !== 0) {
|
|
212
|
+
return timestampDelta;
|
|
213
|
+
}
|
|
214
|
+
return buildPairKey(left.left, left.right).localeCompare(buildPairKey(right.left, right.right));
|
|
215
|
+
}
|
|
216
|
+
function dedupeCrossLanguageFamilies(candidates, observationLookup) {
|
|
217
|
+
const deduped = [];
|
|
218
|
+
const bestByFamily = new Map();
|
|
219
|
+
for (const candidate of candidates) {
|
|
220
|
+
const left = observationLookup.get(candidate.left_ref);
|
|
221
|
+
const right = observationLookup.get(candidate.right_ref);
|
|
222
|
+
if (!left || !right) {
|
|
223
|
+
deduped.push(candidate);
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
const familyKey = buildCrossLanguageFamilyKey(left, right);
|
|
227
|
+
if (!familyKey) {
|
|
228
|
+
deduped.push(candidate);
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
231
|
+
const existing = bestByFamily.get(familyKey);
|
|
232
|
+
if (!existing ||
|
|
233
|
+
candidate.score_total > existing.score_total ||
|
|
234
|
+
(candidate.score_total === existing.score_total &&
|
|
235
|
+
candidate.id.localeCompare(existing.id) < 0)) {
|
|
236
|
+
bestByFamily.set(familyKey, candidate);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return deduped.concat(Array.from(bestByFamily.values()));
|
|
240
|
+
}
|
|
241
|
+
function buildCrossScopeProbePairs(observations, seenPairs, maxPairs) {
|
|
242
|
+
if (maxPairs <= 0) {
|
|
243
|
+
return [];
|
|
244
|
+
}
|
|
245
|
+
const probes = [];
|
|
246
|
+
const bucketMap = new Map();
|
|
247
|
+
for (const observation of observations) {
|
|
248
|
+
if (!observation.entity_kind_hint) {
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
const bucketKey = JSON.stringify([
|
|
252
|
+
observation.entity_kind_hint,
|
|
253
|
+
observation.observation_type,
|
|
254
|
+
observation.scope_kind,
|
|
255
|
+
]);
|
|
256
|
+
const bucket = bucketMap.get(bucketKey) ?? [];
|
|
257
|
+
bucket.push(observation);
|
|
258
|
+
bucketMap.set(bucketKey, bucket);
|
|
259
|
+
}
|
|
260
|
+
for (const bucket of bucketMap.values()) {
|
|
261
|
+
const scopeBuckets = new Map();
|
|
262
|
+
for (const observation of bucket) {
|
|
263
|
+
const scopeKey = observation.scope_id ?? 'global';
|
|
264
|
+
const scopeBucket = scopeBuckets.get(scopeKey) ?? [];
|
|
265
|
+
scopeBucket.push(observation);
|
|
266
|
+
scopeBuckets.set(scopeKey, scopeBucket);
|
|
267
|
+
}
|
|
268
|
+
const limitedScopeBuckets = Array.from(scopeBuckets.values())
|
|
269
|
+
.filter((scopeBucket) => scopeBucket.length > 0)
|
|
270
|
+
.map((scopeBucket) => capObservationGroup(scopeBucket, MAX_CROSS_SCOPE_SCOPE_BUCKET));
|
|
271
|
+
for (let i = 0; i < limitedScopeBuckets.length; i += 1) {
|
|
272
|
+
for (let j = i + 1; j < limitedScopeBuckets.length; j += 1) {
|
|
273
|
+
const leftBucket = limitedScopeBuckets[i];
|
|
274
|
+
const rightBucket = limitedScopeBuckets[j];
|
|
275
|
+
for (const first of leftBucket) {
|
|
276
|
+
for (const second of rightBucket) {
|
|
277
|
+
const [left, right] = first.id.localeCompare(second.id) <= 0 ? [first, second] : [second, first];
|
|
278
|
+
const pairKey = buildPairKey(left, right);
|
|
279
|
+
if (seenPairs.has(pairKey)) {
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
if (!shouldConsiderCrossScopeProbe(left, right)) {
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
seenPairs.add(pairKey);
|
|
286
|
+
pushTopProbe(probes, scorePair(left, right, 'cross_scope'), maxPairs);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
return probes;
|
|
293
|
+
}
|
|
294
|
+
function hasMultipleScopes(observations) {
|
|
295
|
+
const scopes = new Set(observations.map((observation) => `${observation.scope_kind}:${observation.scope_id ?? 'global'}`));
|
|
296
|
+
return scopes.size > 1;
|
|
297
|
+
}
|
|
298
|
+
function dedupeObservationsBySource(observations) {
|
|
299
|
+
const seen = new Set();
|
|
300
|
+
const deduped = [];
|
|
301
|
+
for (const observation of observations) {
|
|
302
|
+
const key = JSON.stringify([
|
|
303
|
+
observation.source_connector,
|
|
304
|
+
observation.source_locator ?? `missing-locator:${observation.id}`,
|
|
305
|
+
observation.source_raw_record_id,
|
|
306
|
+
observation.observation_type,
|
|
307
|
+
]);
|
|
308
|
+
if (seen.has(key)) {
|
|
309
|
+
continue;
|
|
310
|
+
}
|
|
311
|
+
seen.add(key);
|
|
312
|
+
deduped.push(observation);
|
|
313
|
+
}
|
|
314
|
+
return deduped;
|
|
315
|
+
}
|
|
316
|
+
async function generateResolutionCandidates(observations, options = {}) {
|
|
317
|
+
const deduped = dedupeObservationsBySource(observations);
|
|
318
|
+
const observationLookup = new Map(deduped.map((observation) => [observation.id, observation]));
|
|
319
|
+
const blockMap = new Map();
|
|
320
|
+
for (const observation of deduped) {
|
|
321
|
+
const keys = buildBlockKeys(observation);
|
|
322
|
+
for (const key of keys) {
|
|
323
|
+
const existing = blockMap.get(key) ?? [];
|
|
324
|
+
existing.push(observation);
|
|
325
|
+
blockMap.set(key, existing);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
const preliminary = [];
|
|
329
|
+
const seenPairs = new Set();
|
|
330
|
+
for (const bucket of blockMap.values()) {
|
|
331
|
+
if (bucket.length < 2) {
|
|
332
|
+
continue;
|
|
333
|
+
}
|
|
334
|
+
const scopeBuckets = new Map();
|
|
335
|
+
for (const observation of bucket) {
|
|
336
|
+
const scopeKey = `${observation.scope_kind}:${observation.scope_id ?? 'global'}`;
|
|
337
|
+
const scopeBucket = scopeBuckets.get(scopeKey) ?? [];
|
|
338
|
+
scopeBucket.push(observation);
|
|
339
|
+
scopeBuckets.set(scopeKey, scopeBucket);
|
|
340
|
+
}
|
|
341
|
+
for (const scopeBucket of scopeBuckets.values()) {
|
|
342
|
+
const limitedBucket = capObservationGroup(scopeBucket, MAX_DETERMINISTIC_SCOPE_BUCKET);
|
|
343
|
+
if (limitedBucket.length < 2) {
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
for (let i = 0; i < limitedBucket.length; i += 1) {
|
|
347
|
+
for (let j = i + 1; j < limitedBucket.length; j += 1) {
|
|
348
|
+
const first = limitedBucket[i];
|
|
349
|
+
const second = limitedBucket[j];
|
|
350
|
+
const [left, right] = first.id.localeCompare(second.id) <= 0 ? [first, second] : [second, first];
|
|
351
|
+
const pairKey = `${left.id}::${right.id}`;
|
|
352
|
+
if (seenPairs.has(pairKey)) {
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
if (!shouldConsiderPair(left, right)) {
|
|
356
|
+
continue;
|
|
357
|
+
}
|
|
358
|
+
seenPairs.add(pairKey);
|
|
359
|
+
const scored = scorePair(left, right, 'deterministic');
|
|
360
|
+
if (scored.score_total <= 0 && !options.embeddingScorer) {
|
|
361
|
+
continue;
|
|
362
|
+
}
|
|
363
|
+
preliminary.push(scored);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
preliminary.sort((left, right) => right.score_total - left.score_total);
|
|
369
|
+
const topNLimit = options.topN ?? exports.ENTITY_EMBEDDING_TOPN;
|
|
370
|
+
const topN = preliminary.slice(0, topNLimit);
|
|
371
|
+
topN.sort((left, right) => right.score_total - left.score_total);
|
|
372
|
+
const pairBudget = Math.ceil(topNLimit ** 2 / 2);
|
|
373
|
+
const crossScopeBudget = options.embeddingScorer ? Math.max(0, pairBudget - topN.length) : 0;
|
|
374
|
+
const crossScopePairs = options.embeddingScorer && crossScopeBudget > 0 && hasMultipleScopes(deduped)
|
|
375
|
+
? buildCrossScopeProbePairs(deduped, seenPairs, crossScopeBudget)
|
|
376
|
+
: [];
|
|
377
|
+
const candidates = [];
|
|
378
|
+
for (const item of [...topN, ...crossScopePairs]) {
|
|
379
|
+
let score_embedding = 0;
|
|
380
|
+
if (options.embeddingScorer) {
|
|
381
|
+
try {
|
|
382
|
+
score_embedding = await options.embeddingScorer(item.left, item.right);
|
|
383
|
+
}
|
|
384
|
+
catch (error) {
|
|
385
|
+
if (error instanceof errors_js_1.EmbeddingUnavailableError) {
|
|
386
|
+
throw error;
|
|
387
|
+
}
|
|
388
|
+
throw new errors_js_1.EmbeddingUnavailableError({
|
|
389
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
const finalScore = item.score_total + score_embedding;
|
|
394
|
+
if (finalScore <= 0) {
|
|
395
|
+
continue;
|
|
396
|
+
}
|
|
397
|
+
if (item.probe_kind === 'cross_scope' &&
|
|
398
|
+
score_embedding < CROSS_SCOPE_PROBE_MIN_EMBEDDING_SCORE) {
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
const createdAt = Date.now();
|
|
402
|
+
candidates.push({
|
|
403
|
+
id: `candidate_${item.left.id}_${item.right.id}`,
|
|
404
|
+
candidate_kind: 'cluster',
|
|
405
|
+
left_ref: item.left.id,
|
|
406
|
+
right_ref: item.right.id,
|
|
407
|
+
status: 'pending',
|
|
408
|
+
score_total: finalScore,
|
|
409
|
+
score_structural: item.score_structural,
|
|
410
|
+
score_string: item.score_string,
|
|
411
|
+
score_context: item.score_context,
|
|
412
|
+
score_graph: item.score_graph,
|
|
413
|
+
score_embedding,
|
|
414
|
+
rule_trace: JSON.stringify({
|
|
415
|
+
structural: item.score_structural,
|
|
416
|
+
string: item.score_string,
|
|
417
|
+
context: item.score_context,
|
|
418
|
+
}),
|
|
419
|
+
extractor_version: item.left.extractor_version,
|
|
420
|
+
embedding_model_version: item.left.embedding_model_version ?? item.right.embedding_model_version,
|
|
421
|
+
created_at: createdAt,
|
|
422
|
+
updated_at: createdAt,
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
const dedupedFamilies = options.embeddingScorer
|
|
426
|
+
? dedupeCrossLanguageFamilies(candidates, observationLookup)
|
|
427
|
+
: candidates;
|
|
428
|
+
dedupedFamilies.sort((left, right) => {
|
|
429
|
+
if (right.score_total !== left.score_total) {
|
|
430
|
+
return right.score_total - left.score_total;
|
|
431
|
+
}
|
|
432
|
+
return left.id.localeCompare(right.id);
|
|
433
|
+
});
|
|
434
|
+
return dedupedFamilies.slice(0, topNLimit);
|
|
435
|
+
}
|
|
436
|
+
//# sourceMappingURL=candidate-generator.js.map
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { type SourceLocatorKind } from './source-locator.js';
|
|
2
|
+
import type { EntityIngestRun, EntityLineageLink, EntityNode } from './types.js';
|
|
3
|
+
interface InspectorDetail {
|
|
4
|
+
entity: EntityNode;
|
|
5
|
+
history_incomplete: boolean;
|
|
6
|
+
}
|
|
7
|
+
interface InspectorLineageRow extends EntityLineageLink {
|
|
8
|
+
source_connector: string;
|
|
9
|
+
source_raw_record_id: string;
|
|
10
|
+
source_locator: string | null;
|
|
11
|
+
source_locator_kind: SourceLocatorKind;
|
|
12
|
+
surface_form: string;
|
|
13
|
+
}
|
|
14
|
+
interface InspectorLineageResult {
|
|
15
|
+
rows: InspectorLineageRow[];
|
|
16
|
+
history_incomplete: boolean;
|
|
17
|
+
}
|
|
18
|
+
interface ImpactMemoryRow {
|
|
19
|
+
id: string;
|
|
20
|
+
topic: string;
|
|
21
|
+
summary: string;
|
|
22
|
+
created_at: number;
|
|
23
|
+
}
|
|
24
|
+
interface ImpactAuditRunRow {
|
|
25
|
+
id: string;
|
|
26
|
+
classification: string | null;
|
|
27
|
+
status: string;
|
|
28
|
+
created_at: number;
|
|
29
|
+
completed_at: number | null;
|
|
30
|
+
}
|
|
31
|
+
interface EntityImpactResult {
|
|
32
|
+
related_memories: ImpactMemoryRow[];
|
|
33
|
+
ingest_runs: EntityIngestRun[];
|
|
34
|
+
audit_runs: ImpactAuditRunRow[];
|
|
35
|
+
}
|
|
36
|
+
export declare function getEntityInspectorDetail(entityId: string): Promise<InspectorDetail>;
|
|
37
|
+
export declare function listEntityLineageForInspector(entityId: string, limit?: number): Promise<InspectorLineageResult>;
|
|
38
|
+
export declare function getEntityImpact(entityId: string): Promise<EntityImpactResult>;
|
|
39
|
+
export declare function getEntityIngestRun(runId: string): Promise<EntityIngestRun | null>;
|
|
40
|
+
export {};
|
|
41
|
+
//# sourceMappingURL=entity-impact.d.ts.map
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getEntityInspectorDetail = getEntityInspectorDetail;
|
|
4
|
+
exports.listEntityLineageForInspector = listEntityLineageForInspector;
|
|
5
|
+
exports.getEntityImpact = getEntityImpact;
|
|
6
|
+
exports.getEntityIngestRun = getEntityIngestRun;
|
|
7
|
+
const db_manager_js_1 = require("../db-manager.js");
|
|
8
|
+
const source_locator_js_1 = require("./source-locator.js");
|
|
9
|
+
const lineage_store_js_1 = require("./lineage-store.js");
|
|
10
|
+
function mapEntityNode(row) {
|
|
11
|
+
return {
|
|
12
|
+
id: String(row.id),
|
|
13
|
+
kind: row.kind,
|
|
14
|
+
preferred_label: String(row.preferred_label),
|
|
15
|
+
status: row.status,
|
|
16
|
+
scope_kind: typeof row.scope_kind === 'string' ? row.scope_kind : null,
|
|
17
|
+
scope_id: typeof row.scope_id === 'string' ? row.scope_id : null,
|
|
18
|
+
merged_into: typeof row.merged_into === 'string' ? row.merged_into : null,
|
|
19
|
+
created_at: Number(row.created_at),
|
|
20
|
+
updated_at: Number(row.updated_at),
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
async function getEntityInspectorDetail(entityId) {
|
|
24
|
+
await (0, db_manager_js_1.initDB)();
|
|
25
|
+
const adapter = (0, db_manager_js_1.getAdapter)();
|
|
26
|
+
const row = adapter.prepare(`SELECT * FROM entity_nodes WHERE id = ?`).get(entityId);
|
|
27
|
+
if (!row) {
|
|
28
|
+
throw new Error(`Entity not found: ${entityId}`);
|
|
29
|
+
}
|
|
30
|
+
const lineageCount = adapter
|
|
31
|
+
.prepare(`
|
|
32
|
+
SELECT COUNT(*) AS total
|
|
33
|
+
FROM entity_lineage_links
|
|
34
|
+
WHERE canonical_entity_id = ?
|
|
35
|
+
AND status = 'active'
|
|
36
|
+
`)
|
|
37
|
+
.get(entityId);
|
|
38
|
+
return {
|
|
39
|
+
entity: mapEntityNode(row),
|
|
40
|
+
history_incomplete: lineageCount.total === 0,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
async function listEntityLineageForInspector(entityId, limit = 50) {
|
|
44
|
+
await (0, db_manager_js_1.initDB)();
|
|
45
|
+
const adapter = (0, db_manager_js_1.getAdapter)();
|
|
46
|
+
const safeLimit = Math.max(1, Math.min(limit, 500));
|
|
47
|
+
const rows = adapter
|
|
48
|
+
.prepare(`
|
|
49
|
+
SELECT
|
|
50
|
+
l.*,
|
|
51
|
+
o.source_connector,
|
|
52
|
+
o.source_raw_record_id,
|
|
53
|
+
o.source_locator,
|
|
54
|
+
o.surface_form
|
|
55
|
+
FROM entity_lineage_links l
|
|
56
|
+
JOIN entity_observations o ON o.id = l.entity_observation_id
|
|
57
|
+
WHERE l.canonical_entity_id = ?
|
|
58
|
+
AND l.status = 'active'
|
|
59
|
+
ORDER BY l.created_at ASC
|
|
60
|
+
LIMIT ?
|
|
61
|
+
`)
|
|
62
|
+
.all(entityId, safeLimit);
|
|
63
|
+
return {
|
|
64
|
+
rows: rows.map((row) => ({
|
|
65
|
+
id: String(row.id),
|
|
66
|
+
canonical_entity_id: String(row.canonical_entity_id),
|
|
67
|
+
entity_observation_id: String(row.entity_observation_id),
|
|
68
|
+
source_entity_id: typeof row.source_entity_id === 'string' ? row.source_entity_id : null,
|
|
69
|
+
contribution_kind: row.contribution_kind,
|
|
70
|
+
run_id: typeof row.run_id === 'string' ? row.run_id : null,
|
|
71
|
+
candidate_id: typeof row.candidate_id === 'string' ? row.candidate_id : null,
|
|
72
|
+
review_action_id: typeof row.review_action_id === 'string' ? row.review_action_id : null,
|
|
73
|
+
status: row.status,
|
|
74
|
+
capture_mode: row.capture_mode,
|
|
75
|
+
confidence: Number(row.confidence),
|
|
76
|
+
created_at: Number(row.created_at),
|
|
77
|
+
superseded_at: typeof row.superseded_at === 'number' ? row.superseded_at : null,
|
|
78
|
+
source_connector: String(row.source_connector),
|
|
79
|
+
source_raw_record_id: String(row.source_raw_record_id),
|
|
80
|
+
source_locator: typeof row.source_locator === 'string' ? row.source_locator : null,
|
|
81
|
+
source_locator_kind: (0, source_locator_js_1.detectSourceLocatorKind)(typeof row.source_locator === 'string' ? row.source_locator : null),
|
|
82
|
+
surface_form: String(row.surface_form),
|
|
83
|
+
})),
|
|
84
|
+
history_incomplete: rows.length === 0,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
async function getEntityImpact(entityId) {
|
|
88
|
+
await (0, db_manager_js_1.initDB)();
|
|
89
|
+
const adapter = (0, db_manager_js_1.getAdapter)();
|
|
90
|
+
const relatedMemories = adapter
|
|
91
|
+
.prepare(`
|
|
92
|
+
SELECT DISTINCT d.id, d.topic, d.summary, d.created_at
|
|
93
|
+
FROM entity_lineage_links l
|
|
94
|
+
JOIN decision_entity_sources des ON des.entity_observation_id = l.entity_observation_id
|
|
95
|
+
JOIN decisions d ON d.id = des.decision_id
|
|
96
|
+
WHERE l.canonical_entity_id = ?
|
|
97
|
+
AND l.status = 'active'
|
|
98
|
+
ORDER BY d.created_at DESC
|
|
99
|
+
`)
|
|
100
|
+
.all(entityId);
|
|
101
|
+
const ingestRuns = adapter
|
|
102
|
+
.prepare(`
|
|
103
|
+
SELECT DISTINCT ir.*
|
|
104
|
+
FROM entity_lineage_links l
|
|
105
|
+
JOIN entity_ingest_runs ir ON ir.id = l.run_id
|
|
106
|
+
WHERE l.canonical_entity_id = ?
|
|
107
|
+
AND l.status = 'active'
|
|
108
|
+
ORDER BY ir.created_at DESC
|
|
109
|
+
`)
|
|
110
|
+
.all(entityId);
|
|
111
|
+
const auditRuns = adapter
|
|
112
|
+
.prepare(`
|
|
113
|
+
SELECT DISTINCT ar.id, ar.classification, ar.status, ar.created_at, ar.completed_at
|
|
114
|
+
FROM entity_lineage_links l
|
|
115
|
+
JOIN entity_ingest_runs ir ON ir.id = l.run_id
|
|
116
|
+
JOIN entity_audit_runs ar ON ar.id = ir.audit_run_id
|
|
117
|
+
WHERE l.canonical_entity_id = ?
|
|
118
|
+
AND l.status = 'active'
|
|
119
|
+
ORDER BY ar.created_at DESC
|
|
120
|
+
LIMIT 10
|
|
121
|
+
`)
|
|
122
|
+
.all(entityId);
|
|
123
|
+
return {
|
|
124
|
+
related_memories: relatedMemories.map((row) => ({
|
|
125
|
+
id: row.id,
|
|
126
|
+
topic: row.topic,
|
|
127
|
+
summary: row.summary ?? row.topic,
|
|
128
|
+
created_at: row.created_at,
|
|
129
|
+
})),
|
|
130
|
+
ingest_runs: ingestRuns.map(lineage_store_js_1.parseEntityIngestRunRow),
|
|
131
|
+
audit_runs: auditRuns,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
async function getEntityIngestRun(runId) {
|
|
135
|
+
await (0, db_manager_js_1.initDB)();
|
|
136
|
+
const adapter = (0, db_manager_js_1.getAdapter)();
|
|
137
|
+
const row = adapter.prepare(`SELECT * FROM entity_ingest_runs WHERE id = ?`).get(runId);
|
|
138
|
+
return row ? (0, lineage_store_js_1.parseEntityIngestRunRow)(row) : null;
|
|
139
|
+
}
|
|
140
|
+
//# sourceMappingURL=entity-impact.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.loadLinkedDecisionCounts = loadLinkedDecisionCounts;
|
|
4
|
+
function loadLinkedDecisionCounts(adapter, entityIds) {
|
|
5
|
+
const uniqueIds = Array.from(new Set(entityIds.filter((entityId) => typeof entityId === 'string' && entityId.length > 0)));
|
|
6
|
+
if (uniqueIds.length === 0) {
|
|
7
|
+
return new Map();
|
|
8
|
+
}
|
|
9
|
+
const placeholders = uniqueIds.map(() => '?').join(', ');
|
|
10
|
+
const rows = adapter
|
|
11
|
+
.prepare(`
|
|
12
|
+
SELECT
|
|
13
|
+
l.canonical_entity_id AS entity_id,
|
|
14
|
+
COUNT(DISTINCT des.decision_id) AS linked_decision_count
|
|
15
|
+
FROM entity_lineage_links l
|
|
16
|
+
INNER JOIN decision_entity_sources des
|
|
17
|
+
ON des.entity_observation_id = l.entity_observation_id
|
|
18
|
+
WHERE l.status = 'active'
|
|
19
|
+
AND l.canonical_entity_id IN (${placeholders})
|
|
20
|
+
GROUP BY l.canonical_entity_id
|
|
21
|
+
`)
|
|
22
|
+
.all(...uniqueIds);
|
|
23
|
+
const counts = new Map(uniqueIds.map((entityId) => [entityId, 0]));
|
|
24
|
+
for (const row of rows) {
|
|
25
|
+
counts.set(row.entity_id, Number(row.linked_decision_count) || 0);
|
|
26
|
+
}
|
|
27
|
+
return counts;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=entity-linked-decision-counts.js.map
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { EntityNode } from './types.js';
|
|
2
|
+
export interface ListCanonicalEntitiesInput {
|
|
3
|
+
limit: number;
|
|
4
|
+
cursor?: string | null;
|
|
5
|
+
include_noisy?: boolean;
|
|
6
|
+
}
|
|
7
|
+
export interface CanonicalEntityListRow {
|
|
8
|
+
id: string;
|
|
9
|
+
kind: EntityNode['kind'];
|
|
10
|
+
preferred_label: string;
|
|
11
|
+
scope_kind: EntityNode['scope_kind'];
|
|
12
|
+
scope_id: string | null;
|
|
13
|
+
created_at: number;
|
|
14
|
+
linked_decision_count: number;
|
|
15
|
+
}
|
|
16
|
+
export interface ListCanonicalEntitiesResult {
|
|
17
|
+
entities: CanonicalEntityListRow[];
|
|
18
|
+
next_cursor: string | null;
|
|
19
|
+
total_count: number;
|
|
20
|
+
visible_count: number;
|
|
21
|
+
}
|
|
22
|
+
export declare function listCanonicalEntities(input: ListCanonicalEntitiesInput): Promise<ListCanonicalEntitiesResult>;
|
|
23
|
+
//# sourceMappingURL=entity-list.d.ts.map
|