@jungjaehoon/mama-core 1.8.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-graph/alias-write.js +3 -0
- package/dist/agent-graph/entity-resolve.js +3 -7
- package/dist/agent-graph/graph-query.js +5 -4
- package/dist/agent-graph/types.d.ts +30 -0
- package/dist/cases/store.d.ts +0 -4
- package/dist/cases/store.js +0 -66
- package/dist/cases/wiki-page-index.d.ts +0 -13
- package/dist/cases/wiki-page-index.js +0 -153
- package/dist/connectors/event-index.d.ts +1 -22
- package/dist/connectors/event-index.js +0 -157
- package/dist/context-compile/channel-grant.d.ts +46 -0
- package/dist/context-compile/channel-grant.js +66 -0
- package/dist/context-compile/compiler.js +5 -2
- package/dist/context-compile/index.d.ts +1 -0
- package/dist/context-compile/index.js +1 -0
- package/dist/context-compile/packet-store.d.ts +0 -1
- package/dist/context-compile/packet-store.js +0 -16
- package/dist/context-compile/source-readers.d.ts +1 -0
- package/dist/context-compile/source-readers.js +156 -23
- package/dist/context-compile/types.d.ts +17 -0
- package/dist/context-compile/visibility.js +28 -4
- package/dist/db-adapter/node-sqlite-adapter.js +4 -0
- package/dist/db-manager.d.ts +0 -13
- package/dist/db-manager.js +0 -66
- package/dist/decision-formatter.d.ts +0 -11
- package/dist/decision-formatter.js +0 -122
- package/dist/decision-tracker.d.ts +0 -12
- package/dist/decision-tracker.js +0 -50
- package/dist/edges/ref-validation.d.ts +1 -2
- package/dist/edges/ref-validation.js +33 -13
- package/dist/edges/store.d.ts +1 -3
- package/dist/edges/store.js +0 -214
- package/dist/edges/types.d.ts +10 -0
- package/dist/embedding-client.d.ts +0 -17
- package/dist/embedding-client.js +0 -80
- package/dist/embedding-server/mobile/auth.d.ts +1 -10
- package/dist/embedding-server/mobile/auth.js +0 -15
- package/dist/embeddings.d.ts +0 -12
- package/dist/embeddings.js +0 -68
- package/dist/entities/audit-metrics.d.ts +0 -5
- package/dist/entities/audit-metrics.js +0 -4
- package/dist/entities/candidate-generator.d.ts +1 -2
- package/dist/entities/candidate-generator.js +0 -412
- package/dist/entities/entity-list.d.ts +0 -1
- package/dist/entities/entity-list.js +0 -96
- package/dist/entities/entity-orphan-list.d.ts +0 -9
- package/dist/entities/entity-orphan-list.js +0 -88
- package/dist/entities/entity-search.d.ts +0 -1
- package/dist/entities/entity-search.js +0 -147
- package/dist/entities/errors.d.ts +0 -10
- package/dist/entities/errors.js +1 -25
- package/dist/entities/lineage-store.d.ts +0 -23
- package/dist/entities/lineage-store.js +0 -104
- package/dist/entities/policy-store.d.ts +0 -9
- package/dist/entities/policy-store.js +0 -188
- package/dist/entities/provenance-query.d.ts +0 -1
- package/dist/entities/provenance-query.js +0 -164
- package/dist/entities/resolution-engine.d.ts +1 -8
- package/dist/entities/resolution-engine.js +0 -66
- package/dist/entities/rollback-preview.d.ts +0 -4
- package/dist/entities/rollback-preview.js +0 -232
- package/dist/entities/store.d.ts +1 -10
- package/dist/entities/store.js +0 -49
- package/dist/index.d.ts +15 -31
- package/dist/index.js +4 -46
- package/dist/mama-api.d.ts +19 -1
- package/dist/mama-api.js +97 -2
- package/dist/memory/api.d.ts +9 -0
- package/dist/memory/api.js +68 -4
- package/dist/memory/event-store.d.ts +0 -1
- package/dist/memory/event-store.js +0 -15
- package/dist/memory/finding-store.d.ts +0 -1
- package/dist/memory/finding-store.js +0 -12
- package/dist/memory/scope-backfill.d.ts +0 -3
- package/dist/memory/scope-backfill.js +0 -74
- package/dist/memory/scope-store.d.ts +0 -1
- package/dist/memory/scope-store.js +0 -4
- package/dist/memory/truth-store.d.ts +0 -3
- package/dist/memory/truth-store.js +0 -22
- package/dist/memory/types.d.ts +0 -1
- package/dist/memory/types.js +0 -13
- package/dist/memory-store.d.ts +0 -16
- package/dist/memory-store.js +0 -51
- package/dist/ollama-client.d.ts +0 -9
- package/dist/ollama-client.js +0 -125
- package/dist/outcome-tracker.d.ts +0 -14
- package/dist/outcome-tracker.js +0 -44
- package/dist/progress-indicator.d.ts +0 -4
- package/dist/progress-indicator.js +0 -7
- package/dist/provenance/source-ref.d.ts +0 -9
- package/dist/provenance/source-ref.js +0 -145
- package/dist/query-intent.d.ts +0 -4
- package/dist/query-intent.js +0 -30
- package/dist/relevance-scorer.d.ts +0 -9
- package/dist/relevance-scorer.js +0 -81
- package/dist/search/feedback-store.d.ts +0 -1
- package/dist/search/feedback-store.js +0 -4
- package/dist/search/ranker-rescore.d.ts +0 -3
- package/dist/search/ranker-rescore.js +0 -27
- package/dist/search/ranker-trainer.d.ts +0 -8
- package/dist/search/ranker-trainer.js +0 -376
- package/dist/tier-validator.d.ts +0 -12
- package/dist/tier-validator.js +0 -56
- package/package.json +1 -10
- package/dist/cases/case-links.d.ts +0 -116
- package/dist/cases/case-links.js +0 -794
- package/dist/cases/composition-overrides.d.ts +0 -90
- package/dist/cases/composition-overrides.js +0 -724
- package/dist/cases/corrections.d.ts +0 -156
- package/dist/cases/corrections.js +0 -1045
- package/dist/cases/freshness.d.ts +0 -57
- package/dist/cases/freshness.js +0 -283
- package/dist/cases/live-state.d.ts +0 -48
- package/dist/cases/live-state.js +0 -256
- package/dist/cases/membership-explain.d.ts +0 -56
- package/dist/cases/membership-explain.js +0 -140
- package/dist/cases/membership-matcher.d.ts +0 -23
- package/dist/cases/membership-matcher.js +0 -193
- package/dist/cases/merge-split.d.ts +0 -61
- package/dist/cases/merge-split.js +0 -368
- package/dist/cases/role-inference.d.ts +0 -10
- package/dist/cases/role-inference.js +0 -85
- package/dist/cases/sqlite-transaction.d.ts +0 -10
- package/dist/cases/sqlite-transaction.js +0 -62
- package/dist/cases/target-ref.d.ts +0 -25
- package/dist/cases/target-ref.js +0 -22
- package/dist/cases/tombstone-sweeper.d.ts +0 -13
- package/dist/cases/tombstone-sweeper.js +0 -128
- package/dist/entities/entity-impact.d.ts +0 -41
- package/dist/entities/entity-impact.js +0 -140
- package/dist/entities/exact-merge-backfill.d.ts +0 -11
- package/dist/entities/exact-merge-backfill.js +0 -161
- package/dist/entities/lineage-backfill.d.ts +0 -12
- package/dist/entities/lineage-backfill.js +0 -101
- package/dist/memory-inject.d.ts +0 -24
- package/dist/memory-inject.js +0 -116
|
@@ -2,299 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ENTITY_EMBEDDING_TOPN = void 0;
|
|
4
4
|
exports.dedupeObservationsBySource = dedupeObservationsBySource;
|
|
5
|
-
exports.generateResolutionCandidates = generateResolutionCandidates;
|
|
6
|
-
const errors_js_1 = require("./errors.js");
|
|
7
|
-
const normalization_js_1 = require("./normalization.js");
|
|
8
5
|
exports.ENTITY_EMBEDDING_TOPN = 50;
|
|
9
|
-
const CROSS_SCOPE_PROBE_MIN_EMBEDDING_SCORE = 0.84;
|
|
10
|
-
const MAX_DETERMINISTIC_SCOPE_BUCKET = 32;
|
|
11
|
-
const MAX_CROSS_SCOPE_SCOPE_BUCKET = 8;
|
|
12
|
-
function tokenize(input) {
|
|
13
|
-
return input
|
|
14
|
-
.toLowerCase()
|
|
15
|
-
.split(/[\s,.!?;:()[\]{}"'/_-]+/u)
|
|
16
|
-
.map((token) => token.trim())
|
|
17
|
-
.filter((token) => token.length > 1);
|
|
18
|
-
}
|
|
19
|
-
function contextScore(left, right) {
|
|
20
|
-
const leftTokens = new Set(tokenize(left.context_summary ?? ''));
|
|
21
|
-
const rightTokens = new Set(tokenize(right.context_summary ?? ''));
|
|
22
|
-
if (leftTokens.size === 0 || rightTokens.size === 0) {
|
|
23
|
-
return 0;
|
|
24
|
-
}
|
|
25
|
-
let overlap = 0;
|
|
26
|
-
for (const token of leftTokens) {
|
|
27
|
-
if (rightTokens.has(token)) {
|
|
28
|
-
overlap += 1;
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
return overlap / Math.max(Math.min(leftTokens.size, rightTokens.size), 1);
|
|
32
|
-
}
|
|
33
|
-
function structuralScore(left, right) {
|
|
34
|
-
if (left.normalized_form === right.normalized_form) {
|
|
35
|
-
return 1;
|
|
36
|
-
}
|
|
37
|
-
const leftIds = (0, normalization_js_1.extractStructuredIdentifiers)(left.surface_form);
|
|
38
|
-
const rightIds = (0, normalization_js_1.extractStructuredIdentifiers)(right.surface_form);
|
|
39
|
-
const hasSharedEmail = leftIds.emails.some((value) => rightIds.emails.includes(value));
|
|
40
|
-
const hasSharedHandle = leftIds.handles.some((value) => rightIds.handles.includes(value));
|
|
41
|
-
const hasSharedDomain = leftIds.domains.some((value) => rightIds.domains.includes(value));
|
|
42
|
-
if (hasSharedEmail || hasSharedHandle || hasSharedDomain) {
|
|
43
|
-
return 0.75;
|
|
44
|
-
}
|
|
45
|
-
return 0;
|
|
46
|
-
}
|
|
47
|
-
function stringScore(left, right) {
|
|
48
|
-
if (left.normalized_form === right.normalized_form) {
|
|
49
|
-
return 1;
|
|
50
|
-
}
|
|
51
|
-
const leftText = left.surface_form.toLowerCase();
|
|
52
|
-
const rightText = right.surface_form.toLowerCase();
|
|
53
|
-
const sharedPrefix = leftText.length > 0 && rightText.length > 0 && leftText[0] === rightText[0];
|
|
54
|
-
return sharedPrefix ? 0.1 : 0;
|
|
55
|
-
}
|
|
56
|
-
function scorePair(left, right, probeKind) {
|
|
57
|
-
const score_structural = structuralScore(left, right);
|
|
58
|
-
const score_string = stringScore(left, right);
|
|
59
|
-
const score_context = contextScore(left, right);
|
|
60
|
-
const score_graph = 0;
|
|
61
|
-
const score_total = score_structural + score_string + score_context + score_graph;
|
|
62
|
-
return {
|
|
63
|
-
left,
|
|
64
|
-
right,
|
|
65
|
-
score_structural,
|
|
66
|
-
score_string,
|
|
67
|
-
score_context,
|
|
68
|
-
score_graph,
|
|
69
|
-
score_total,
|
|
70
|
-
probe_kind: probeKind,
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
function shouldConsiderPair(left, right) {
|
|
74
|
-
if (left.entity_kind_hint &&
|
|
75
|
-
right.entity_kind_hint &&
|
|
76
|
-
left.entity_kind_hint !== right.entity_kind_hint) {
|
|
77
|
-
return false;
|
|
78
|
-
}
|
|
79
|
-
if (left.scope_kind !== right.scope_kind) {
|
|
80
|
-
return false;
|
|
81
|
-
}
|
|
82
|
-
if (left.scope_id !== right.scope_id) {
|
|
83
|
-
return false;
|
|
84
|
-
}
|
|
85
|
-
return true;
|
|
86
|
-
}
|
|
87
|
-
function shouldConsiderCrossScopeProbe(left, right) {
|
|
88
|
-
if (!left.entity_kind_hint ||
|
|
89
|
-
!right.entity_kind_hint ||
|
|
90
|
-
left.entity_kind_hint !== right.entity_kind_hint) {
|
|
91
|
-
return false;
|
|
92
|
-
}
|
|
93
|
-
if (left.observation_type !== right.observation_type) {
|
|
94
|
-
return false;
|
|
95
|
-
}
|
|
96
|
-
if (left.scope_kind !== right.scope_kind) {
|
|
97
|
-
return false;
|
|
98
|
-
}
|
|
99
|
-
if (left.scope_id === right.scope_id) {
|
|
100
|
-
return false;
|
|
101
|
-
}
|
|
102
|
-
return true;
|
|
103
|
-
}
|
|
104
|
-
function buildBlockKeys(observation) {
|
|
105
|
-
const keys = new Set();
|
|
106
|
-
keys.add(`norm:${observation.normalized_form}`);
|
|
107
|
-
for (const related of observation.related_surface_forms) {
|
|
108
|
-
keys.add(`related:${related}`);
|
|
109
|
-
}
|
|
110
|
-
const ids = (0, normalization_js_1.extractStructuredIdentifiers)(observation.surface_form);
|
|
111
|
-
for (const email of ids.emails) {
|
|
112
|
-
keys.add(`email:${email}`);
|
|
113
|
-
}
|
|
114
|
-
for (const handle of ids.handles) {
|
|
115
|
-
keys.add(`handle:${handle}`);
|
|
116
|
-
}
|
|
117
|
-
for (const domain of ids.domains) {
|
|
118
|
-
keys.add(`domain:${domain}`);
|
|
119
|
-
}
|
|
120
|
-
const contextTokens = tokenize(observation.context_summary ?? '').slice(0, 3);
|
|
121
|
-
for (const token of contextTokens) {
|
|
122
|
-
keys.add(`ctx:${observation.scope_kind}:${observation.scope_id ?? 'global'}:${token}`);
|
|
123
|
-
}
|
|
124
|
-
return Array.from(keys);
|
|
125
|
-
}
|
|
126
|
-
function normalizeSurfaceForm(value) {
|
|
127
|
-
if (value.trim().length === 0) {
|
|
128
|
-
return '';
|
|
129
|
-
}
|
|
130
|
-
return (0, normalization_js_1.normalizeEntityLabel)(value).normalized;
|
|
131
|
-
}
|
|
132
|
-
function sharedNormalizedRelatedCount(left, right) {
|
|
133
|
-
const leftRelated = new Set(left.related_surface_forms.map((value) => normalizeSurfaceForm(value)));
|
|
134
|
-
const rightRelated = new Set(right.related_surface_forms.map((value) => normalizeSurfaceForm(value)));
|
|
135
|
-
if (leftRelated.size === 0 || rightRelated.size === 0) {
|
|
136
|
-
return 0;
|
|
137
|
-
}
|
|
138
|
-
let overlap = 0;
|
|
139
|
-
for (const related of leftRelated) {
|
|
140
|
-
if (rightRelated.has(related)) {
|
|
141
|
-
overlap += 1;
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
return overlap;
|
|
145
|
-
}
|
|
146
|
-
function getTimestampDistance(left, right) {
|
|
147
|
-
if (left.timestamp_observed === null || right.timestamp_observed === null) {
|
|
148
|
-
return Number.POSITIVE_INFINITY;
|
|
149
|
-
}
|
|
150
|
-
return Math.abs(left.timestamp_observed - right.timestamp_observed);
|
|
151
|
-
}
|
|
152
|
-
function buildPairKey(left, right) {
|
|
153
|
-
return `${left.id}::${right.id}`;
|
|
154
|
-
}
|
|
155
|
-
function compareObservationForSampling(left, right) {
|
|
156
|
-
const leftRelated = left.related_surface_forms.length;
|
|
157
|
-
const rightRelated = right.related_surface_forms.length;
|
|
158
|
-
if (rightRelated !== leftRelated) {
|
|
159
|
-
return rightRelated - leftRelated;
|
|
160
|
-
}
|
|
161
|
-
const leftContext = left.context_summary?.length ?? 0;
|
|
162
|
-
const rightContext = right.context_summary?.length ?? 0;
|
|
163
|
-
if (rightContext !== leftContext) {
|
|
164
|
-
return rightContext - leftContext;
|
|
165
|
-
}
|
|
166
|
-
const leftObserved = left.timestamp_observed ?? Number.NEGATIVE_INFINITY;
|
|
167
|
-
const rightObserved = right.timestamp_observed ?? Number.NEGATIVE_INFINITY;
|
|
168
|
-
if (rightObserved !== leftObserved) {
|
|
169
|
-
return rightObserved - leftObserved;
|
|
170
|
-
}
|
|
171
|
-
return left.id.localeCompare(right.id);
|
|
172
|
-
}
|
|
173
|
-
function capObservationGroup(observations, limit) {
|
|
174
|
-
if (observations.length <= limit) {
|
|
175
|
-
return observations;
|
|
176
|
-
}
|
|
177
|
-
return [...observations].sort(compareObservationForSampling).slice(0, limit);
|
|
178
|
-
}
|
|
179
|
-
function pushTopProbe(probes, candidate, maxPairs) {
|
|
180
|
-
probes.push(candidate);
|
|
181
|
-
probes.sort(compareProbePairs);
|
|
182
|
-
if (probes.length > maxPairs) {
|
|
183
|
-
probes.length = maxPairs;
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
function buildCrossLanguageFamilyKey(left, right) {
|
|
187
|
-
if (!left.lang || !right.lang || left.lang === right.lang) {
|
|
188
|
-
return null;
|
|
189
|
-
}
|
|
190
|
-
const forms = [
|
|
191
|
-
`${left.lang}:${left.normalized_form}`,
|
|
192
|
-
`${right.lang}:${right.normalized_form}`,
|
|
193
|
-
].sort();
|
|
194
|
-
const observationTypes = [left.observation_type, right.observation_type].sort();
|
|
195
|
-
return JSON.stringify([
|
|
196
|
-
left.entity_kind_hint ?? right.entity_kind_hint ?? 'unknown',
|
|
197
|
-
observationTypes,
|
|
198
|
-
forms,
|
|
199
|
-
]);
|
|
200
|
-
}
|
|
201
|
-
function compareProbePairs(left, right) {
|
|
202
|
-
const relatedDelta = sharedNormalizedRelatedCount(right.left, right.right) -
|
|
203
|
-
sharedNormalizedRelatedCount(left.left, left.right);
|
|
204
|
-
if (relatedDelta !== 0) {
|
|
205
|
-
return relatedDelta;
|
|
206
|
-
}
|
|
207
|
-
if (right.score_total !== left.score_total) {
|
|
208
|
-
return right.score_total - left.score_total;
|
|
209
|
-
}
|
|
210
|
-
const timestampDelta = getTimestampDistance(left.left, left.right) - getTimestampDistance(right.left, right.right);
|
|
211
|
-
if (timestampDelta !== 0) {
|
|
212
|
-
return timestampDelta;
|
|
213
|
-
}
|
|
214
|
-
return buildPairKey(left.left, left.right).localeCompare(buildPairKey(right.left, right.right));
|
|
215
|
-
}
|
|
216
|
-
function dedupeCrossLanguageFamilies(candidates, observationLookup) {
|
|
217
|
-
const deduped = [];
|
|
218
|
-
const bestByFamily = new Map();
|
|
219
|
-
for (const candidate of candidates) {
|
|
220
|
-
const left = observationLookup.get(candidate.left_ref);
|
|
221
|
-
const right = observationLookup.get(candidate.right_ref);
|
|
222
|
-
if (!left || !right) {
|
|
223
|
-
deduped.push(candidate);
|
|
224
|
-
continue;
|
|
225
|
-
}
|
|
226
|
-
const familyKey = buildCrossLanguageFamilyKey(left, right);
|
|
227
|
-
if (!familyKey) {
|
|
228
|
-
deduped.push(candidate);
|
|
229
|
-
continue;
|
|
230
|
-
}
|
|
231
|
-
const existing = bestByFamily.get(familyKey);
|
|
232
|
-
if (!existing ||
|
|
233
|
-
candidate.score_total > existing.score_total ||
|
|
234
|
-
(candidate.score_total === existing.score_total &&
|
|
235
|
-
candidate.id.localeCompare(existing.id) < 0)) {
|
|
236
|
-
bestByFamily.set(familyKey, candidate);
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
return deduped.concat(Array.from(bestByFamily.values()));
|
|
240
|
-
}
|
|
241
|
-
function buildCrossScopeProbePairs(observations, seenPairs, maxPairs) {
|
|
242
|
-
if (maxPairs <= 0) {
|
|
243
|
-
return [];
|
|
244
|
-
}
|
|
245
|
-
const probes = [];
|
|
246
|
-
const bucketMap = new Map();
|
|
247
|
-
for (const observation of observations) {
|
|
248
|
-
if (!observation.entity_kind_hint) {
|
|
249
|
-
continue;
|
|
250
|
-
}
|
|
251
|
-
const bucketKey = JSON.stringify([
|
|
252
|
-
observation.entity_kind_hint,
|
|
253
|
-
observation.observation_type,
|
|
254
|
-
observation.scope_kind,
|
|
255
|
-
]);
|
|
256
|
-
const bucket = bucketMap.get(bucketKey) ?? [];
|
|
257
|
-
bucket.push(observation);
|
|
258
|
-
bucketMap.set(bucketKey, bucket);
|
|
259
|
-
}
|
|
260
|
-
for (const bucket of bucketMap.values()) {
|
|
261
|
-
const scopeBuckets = new Map();
|
|
262
|
-
for (const observation of bucket) {
|
|
263
|
-
const scopeKey = observation.scope_id ?? 'global';
|
|
264
|
-
const scopeBucket = scopeBuckets.get(scopeKey) ?? [];
|
|
265
|
-
scopeBucket.push(observation);
|
|
266
|
-
scopeBuckets.set(scopeKey, scopeBucket);
|
|
267
|
-
}
|
|
268
|
-
const limitedScopeBuckets = Array.from(scopeBuckets.values())
|
|
269
|
-
.filter((scopeBucket) => scopeBucket.length > 0)
|
|
270
|
-
.map((scopeBucket) => capObservationGroup(scopeBucket, MAX_CROSS_SCOPE_SCOPE_BUCKET));
|
|
271
|
-
for (let i = 0; i < limitedScopeBuckets.length; i += 1) {
|
|
272
|
-
for (let j = i + 1; j < limitedScopeBuckets.length; j += 1) {
|
|
273
|
-
const leftBucket = limitedScopeBuckets[i];
|
|
274
|
-
const rightBucket = limitedScopeBuckets[j];
|
|
275
|
-
for (const first of leftBucket) {
|
|
276
|
-
for (const second of rightBucket) {
|
|
277
|
-
const [left, right] = first.id.localeCompare(second.id) <= 0 ? [first, second] : [second, first];
|
|
278
|
-
const pairKey = buildPairKey(left, right);
|
|
279
|
-
if (seenPairs.has(pairKey)) {
|
|
280
|
-
continue;
|
|
281
|
-
}
|
|
282
|
-
if (!shouldConsiderCrossScopeProbe(left, right)) {
|
|
283
|
-
continue;
|
|
284
|
-
}
|
|
285
|
-
seenPairs.add(pairKey);
|
|
286
|
-
pushTopProbe(probes, scorePair(left, right, 'cross_scope'), maxPairs);
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
return probes;
|
|
293
|
-
}
|
|
294
|
-
function hasMultipleScopes(observations) {
|
|
295
|
-
const scopes = new Set(observations.map((observation) => `${observation.scope_kind}:${observation.scope_id ?? 'global'}`));
|
|
296
|
-
return scopes.size > 1;
|
|
297
|
-
}
|
|
298
6
|
function dedupeObservationsBySource(observations) {
|
|
299
7
|
const seen = new Set();
|
|
300
8
|
const deduped = [];
|
|
@@ -313,124 +21,4 @@ function dedupeObservationsBySource(observations) {
|
|
|
313
21
|
}
|
|
314
22
|
return deduped;
|
|
315
23
|
}
|
|
316
|
-
async function generateResolutionCandidates(observations, options = {}) {
|
|
317
|
-
const deduped = dedupeObservationsBySource(observations);
|
|
318
|
-
const observationLookup = new Map(deduped.map((observation) => [observation.id, observation]));
|
|
319
|
-
const blockMap = new Map();
|
|
320
|
-
for (const observation of deduped) {
|
|
321
|
-
const keys = buildBlockKeys(observation);
|
|
322
|
-
for (const key of keys) {
|
|
323
|
-
const existing = blockMap.get(key) ?? [];
|
|
324
|
-
existing.push(observation);
|
|
325
|
-
blockMap.set(key, existing);
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
const preliminary = [];
|
|
329
|
-
const seenPairs = new Set();
|
|
330
|
-
for (const bucket of blockMap.values()) {
|
|
331
|
-
if (bucket.length < 2) {
|
|
332
|
-
continue;
|
|
333
|
-
}
|
|
334
|
-
const scopeBuckets = new Map();
|
|
335
|
-
for (const observation of bucket) {
|
|
336
|
-
const scopeKey = `${observation.scope_kind}:${observation.scope_id ?? 'global'}`;
|
|
337
|
-
const scopeBucket = scopeBuckets.get(scopeKey) ?? [];
|
|
338
|
-
scopeBucket.push(observation);
|
|
339
|
-
scopeBuckets.set(scopeKey, scopeBucket);
|
|
340
|
-
}
|
|
341
|
-
for (const scopeBucket of scopeBuckets.values()) {
|
|
342
|
-
const limitedBucket = capObservationGroup(scopeBucket, MAX_DETERMINISTIC_SCOPE_BUCKET);
|
|
343
|
-
if (limitedBucket.length < 2) {
|
|
344
|
-
continue;
|
|
345
|
-
}
|
|
346
|
-
for (let i = 0; i < limitedBucket.length; i += 1) {
|
|
347
|
-
for (let j = i + 1; j < limitedBucket.length; j += 1) {
|
|
348
|
-
const first = limitedBucket[i];
|
|
349
|
-
const second = limitedBucket[j];
|
|
350
|
-
const [left, right] = first.id.localeCompare(second.id) <= 0 ? [first, second] : [second, first];
|
|
351
|
-
const pairKey = `${left.id}::${right.id}`;
|
|
352
|
-
if (seenPairs.has(pairKey)) {
|
|
353
|
-
continue;
|
|
354
|
-
}
|
|
355
|
-
if (!shouldConsiderPair(left, right)) {
|
|
356
|
-
continue;
|
|
357
|
-
}
|
|
358
|
-
seenPairs.add(pairKey);
|
|
359
|
-
const scored = scorePair(left, right, 'deterministic');
|
|
360
|
-
if (scored.score_total <= 0 && !options.embeddingScorer) {
|
|
361
|
-
continue;
|
|
362
|
-
}
|
|
363
|
-
preliminary.push(scored);
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
}
|
|
368
|
-
preliminary.sort((left, right) => right.score_total - left.score_total);
|
|
369
|
-
const topNLimit = options.topN ?? exports.ENTITY_EMBEDDING_TOPN;
|
|
370
|
-
const topN = preliminary.slice(0, topNLimit);
|
|
371
|
-
topN.sort((left, right) => right.score_total - left.score_total);
|
|
372
|
-
const pairBudget = Math.ceil(topNLimit ** 2 / 2);
|
|
373
|
-
const crossScopeBudget = options.embeddingScorer ? Math.max(0, pairBudget - topN.length) : 0;
|
|
374
|
-
const crossScopePairs = options.embeddingScorer && crossScopeBudget > 0 && hasMultipleScopes(deduped)
|
|
375
|
-
? buildCrossScopeProbePairs(deduped, seenPairs, crossScopeBudget)
|
|
376
|
-
: [];
|
|
377
|
-
const candidates = [];
|
|
378
|
-
for (const item of [...topN, ...crossScopePairs]) {
|
|
379
|
-
let score_embedding = 0;
|
|
380
|
-
if (options.embeddingScorer) {
|
|
381
|
-
try {
|
|
382
|
-
score_embedding = await options.embeddingScorer(item.left, item.right);
|
|
383
|
-
}
|
|
384
|
-
catch (error) {
|
|
385
|
-
if (error instanceof errors_js_1.EmbeddingUnavailableError) {
|
|
386
|
-
throw error;
|
|
387
|
-
}
|
|
388
|
-
throw new errors_js_1.EmbeddingUnavailableError({
|
|
389
|
-
reason: error instanceof Error ? error.message : String(error),
|
|
390
|
-
});
|
|
391
|
-
}
|
|
392
|
-
}
|
|
393
|
-
const finalScore = item.score_total + score_embedding;
|
|
394
|
-
if (finalScore <= 0) {
|
|
395
|
-
continue;
|
|
396
|
-
}
|
|
397
|
-
if (item.probe_kind === 'cross_scope' &&
|
|
398
|
-
score_embedding < CROSS_SCOPE_PROBE_MIN_EMBEDDING_SCORE) {
|
|
399
|
-
continue;
|
|
400
|
-
}
|
|
401
|
-
const createdAt = Date.now();
|
|
402
|
-
candidates.push({
|
|
403
|
-
id: `candidate_${item.left.id}_${item.right.id}`,
|
|
404
|
-
candidate_kind: 'cluster',
|
|
405
|
-
left_ref: item.left.id,
|
|
406
|
-
right_ref: item.right.id,
|
|
407
|
-
status: 'pending',
|
|
408
|
-
score_total: finalScore,
|
|
409
|
-
score_structural: item.score_structural,
|
|
410
|
-
score_string: item.score_string,
|
|
411
|
-
score_context: item.score_context,
|
|
412
|
-
score_graph: item.score_graph,
|
|
413
|
-
score_embedding,
|
|
414
|
-
rule_trace: JSON.stringify({
|
|
415
|
-
structural: item.score_structural,
|
|
416
|
-
string: item.score_string,
|
|
417
|
-
context: item.score_context,
|
|
418
|
-
}),
|
|
419
|
-
extractor_version: item.left.extractor_version,
|
|
420
|
-
embedding_model_version: item.left.embedding_model_version ?? item.right.embedding_model_version,
|
|
421
|
-
created_at: createdAt,
|
|
422
|
-
updated_at: createdAt,
|
|
423
|
-
});
|
|
424
|
-
}
|
|
425
|
-
const dedupedFamilies = options.embeddingScorer
|
|
426
|
-
? dedupeCrossLanguageFamilies(candidates, observationLookup)
|
|
427
|
-
: candidates;
|
|
428
|
-
dedupedFamilies.sort((left, right) => {
|
|
429
|
-
if (right.score_total !== left.score_total) {
|
|
430
|
-
return right.score_total - left.score_total;
|
|
431
|
-
}
|
|
432
|
-
return left.id.localeCompare(right.id);
|
|
433
|
-
});
|
|
434
|
-
return dedupedFamilies.slice(0, topNLimit);
|
|
435
|
-
}
|
|
436
24
|
//# sourceMappingURL=candidate-generator.js.map
|
|
@@ -19,5 +19,4 @@ export interface ListCanonicalEntitiesResult {
|
|
|
19
19
|
total_count: number;
|
|
20
20
|
visible_count: number;
|
|
21
21
|
}
|
|
22
|
-
export declare function listCanonicalEntities(input: ListCanonicalEntitiesInput): Promise<ListCanonicalEntitiesResult>;
|
|
23
22
|
//# sourceMappingURL=entity-list.d.ts.map
|
|
@@ -1,99 +1,3 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.listCanonicalEntities = listCanonicalEntities;
|
|
4
|
-
const db_manager_js_1 = require("../db-manager.js");
|
|
5
|
-
const entity_linked_decision_counts_js_1 = require("./entity-linked-decision-counts.js");
|
|
6
|
-
function normalizeBrowseLabel(input) {
|
|
7
|
-
return input.normalize('NFKC').trim().replace(/\s+/gu, ' ').toLowerCase();
|
|
8
|
-
}
|
|
9
|
-
function buildBrowseCollapseKey(row) {
|
|
10
|
-
return JSON.stringify([
|
|
11
|
-
row.kind,
|
|
12
|
-
row.scope_kind,
|
|
13
|
-
row.scope_id ?? '',
|
|
14
|
-
normalizeBrowseLabel(row.preferred_label),
|
|
15
|
-
]);
|
|
16
|
-
}
|
|
17
|
-
function compareCanonicalRows(left, right) {
|
|
18
|
-
if (right.created_at !== left.created_at) {
|
|
19
|
-
return right.created_at - left.created_at;
|
|
20
|
-
}
|
|
21
|
-
const labelCompare = left.preferred_label.localeCompare(right.preferred_label);
|
|
22
|
-
if (labelCompare !== 0) {
|
|
23
|
-
return labelCompare;
|
|
24
|
-
}
|
|
25
|
-
return left.id.localeCompare(right.id);
|
|
26
|
-
}
|
|
27
|
-
function decodeCursor(cursor) {
|
|
28
|
-
if (!cursor) {
|
|
29
|
-
return 0;
|
|
30
|
-
}
|
|
31
|
-
try {
|
|
32
|
-
const decoded = Number(Buffer.from(cursor, 'base64').toString('utf8'));
|
|
33
|
-
return Number.isFinite(decoded) && decoded >= 0 ? decoded : 0;
|
|
34
|
-
}
|
|
35
|
-
catch {
|
|
36
|
-
return 0;
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
function encodeCursor(offset) {
|
|
40
|
-
return Buffer.from(String(offset), 'utf8').toString('base64');
|
|
41
|
-
}
|
|
42
|
-
function isNoisyCanonicalEntity(row) {
|
|
43
|
-
const label = row.preferred_label.trim().toLowerCase();
|
|
44
|
-
if (row.kind === 'person') {
|
|
45
|
-
return label === 'user' || label === 'claude';
|
|
46
|
-
}
|
|
47
|
-
if (row.kind === 'project') {
|
|
48
|
-
return label === 'workspace';
|
|
49
|
-
}
|
|
50
|
-
return false;
|
|
51
|
-
}
|
|
52
|
-
async function listCanonicalEntities(input) {
|
|
53
|
-
await (0, db_manager_js_1.initDB)();
|
|
54
|
-
const adapter = (0, db_manager_js_1.getAdapter)();
|
|
55
|
-
const rows = adapter
|
|
56
|
-
.prepare(`
|
|
57
|
-
SELECT id, kind, preferred_label, scope_kind, scope_id, created_at
|
|
58
|
-
FROM entity_nodes
|
|
59
|
-
WHERE status = 'active'
|
|
60
|
-
AND merged_into IS NULL
|
|
61
|
-
`)
|
|
62
|
-
.all();
|
|
63
|
-
const sorted = rows.sort(compareCanonicalRows);
|
|
64
|
-
const collapsedByKey = new Map();
|
|
65
|
-
for (const row of sorted) {
|
|
66
|
-
const key = buildBrowseCollapseKey(row);
|
|
67
|
-
const existing = collapsedByKey.get(key);
|
|
68
|
-
if (!existing) {
|
|
69
|
-
collapsedByKey.set(key, row);
|
|
70
|
-
continue;
|
|
71
|
-
}
|
|
72
|
-
if (row.created_at > existing.created_at ||
|
|
73
|
-
(row.created_at === existing.created_at && row.id.localeCompare(existing.id) < 0)) {
|
|
74
|
-
collapsedByKey.set(key, row);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
const collapsed = Array.from(collapsedByKey.values()).sort(compareCanonicalRows);
|
|
78
|
-
const totalCount = sorted.length;
|
|
79
|
-
const visibleBase = input.include_noisy === false
|
|
80
|
-
? collapsed.filter((row) => !isNoisyCanonicalEntity(row))
|
|
81
|
-
: collapsed;
|
|
82
|
-
const offset = decodeCursor(input.cursor);
|
|
83
|
-
const limit = Math.max(1, input.limit);
|
|
84
|
-
const page = visibleBase.slice(offset, offset + limit);
|
|
85
|
-
const linkedCounts = (0, entity_linked_decision_counts_js_1.loadLinkedDecisionCounts)(adapter, page.map((row) => row.id));
|
|
86
|
-
const entities = page.map((row) => ({
|
|
87
|
-
...row,
|
|
88
|
-
linked_decision_count: linkedCounts.get(row.id) ?? 0,
|
|
89
|
-
}));
|
|
90
|
-
const visibleCount = visibleBase.length;
|
|
91
|
-
const nextCursor = offset + limit < visibleBase.length ? encodeCursor(offset + limit) : null;
|
|
92
|
-
return {
|
|
93
|
-
entities,
|
|
94
|
-
next_cursor: nextCursor,
|
|
95
|
-
total_count: totalCount,
|
|
96
|
-
visible_count: visibleCount,
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
3
|
//# sourceMappingURL=entity-list.js.map
|
|
@@ -14,13 +14,4 @@ export interface CanonicalEntityOrphanRow {
|
|
|
14
14
|
last_seen_at: number | null;
|
|
15
15
|
};
|
|
16
16
|
}
|
|
17
|
-
interface ReadAdapter {
|
|
18
|
-
prepare(sql: string): {
|
|
19
|
-
all: (...params: unknown[]) => unknown[];
|
|
20
|
-
};
|
|
21
|
-
}
|
|
22
|
-
export declare function listCanonicalEntityOrphans(adapter?: ReadAdapter, options?: {
|
|
23
|
-
min_age_ms?: number | null;
|
|
24
|
-
}): CanonicalEntityOrphanRow[];
|
|
25
|
-
export {};
|
|
26
17
|
//# sourceMappingURL=entity-orphan-list.d.ts.map
|
|
@@ -1,91 +1,3 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.listCanonicalEntityOrphans = listCanonicalEntityOrphans;
|
|
4
|
-
const db_manager_js_1 = require("../db-manager.js");
|
|
5
|
-
const entity_linked_decision_counts_js_1 = require("./entity-linked-decision-counts.js");
|
|
6
|
-
function buildScopeLabel(scopeKind, scopeId) {
|
|
7
|
-
return scopeKind ? `${scopeKind}:${scopeId ?? ''}` : 'global';
|
|
8
|
-
}
|
|
9
|
-
function isNoisyCanonicalEntity(row) {
|
|
10
|
-
const label = row.title.trim().toLowerCase();
|
|
11
|
-
if (row.kind === 'person') {
|
|
12
|
-
return label === 'user' || label === 'claude';
|
|
13
|
-
}
|
|
14
|
-
if (row.kind === 'project') {
|
|
15
|
-
return label === 'workspace';
|
|
16
|
-
}
|
|
17
|
-
return false;
|
|
18
|
-
}
|
|
19
|
-
function compareOrphanRows(left, right) {
|
|
20
|
-
const leftSeen = typeof left.evidence_summary.last_seen_at === 'number' ? left.evidence_summary.last_seen_at : 0;
|
|
21
|
-
const rightSeen = typeof right.evidence_summary.last_seen_at === 'number'
|
|
22
|
-
? right.evidence_summary.last_seen_at
|
|
23
|
-
: 0;
|
|
24
|
-
if (rightSeen !== leftSeen) {
|
|
25
|
-
return rightSeen - leftSeen;
|
|
26
|
-
}
|
|
27
|
-
return left.entity_id.localeCompare(right.entity_id);
|
|
28
|
-
}
|
|
29
|
-
function listCanonicalEntityOrphans(adapter = (0, db_manager_js_1.getAdapter)(), options = {}) {
|
|
30
|
-
const minAgeMs = typeof options.min_age_ms === 'number' && Number.isFinite(options.min_age_ms)
|
|
31
|
-
? Math.max(0, options.min_age_ms)
|
|
32
|
-
: 0;
|
|
33
|
-
const minCreatedAt = Date.now() - minAgeMs;
|
|
34
|
-
const rows = adapter
|
|
35
|
-
.prepare(`
|
|
36
|
-
SELECT
|
|
37
|
-
n.id AS entity_id,
|
|
38
|
-
n.kind AS kind,
|
|
39
|
-
n.preferred_label AS title,
|
|
40
|
-
n.scope_kind AS scope_kind,
|
|
41
|
-
n.scope_id AS scope_id,
|
|
42
|
-
n.created_at AS created_at,
|
|
43
|
-
COUNT(DISTINCT l.entity_observation_id) AS lineage_rows,
|
|
44
|
-
COUNT(
|
|
45
|
-
DISTINCT CASE
|
|
46
|
-
WHEN COALESCE(o.source_raw_record_id, '') <> '' THEN l.entity_observation_id
|
|
47
|
-
ELSE NULL
|
|
48
|
-
END
|
|
49
|
-
) AS raw_evidence_rows,
|
|
50
|
-
MAX(COALESCE(o.timestamp_observed, o.created_at)) AS last_seen_at
|
|
51
|
-
FROM entity_nodes n
|
|
52
|
-
INNER JOIN entity_lineage_links l
|
|
53
|
-
ON l.canonical_entity_id = n.id
|
|
54
|
-
AND l.status = 'active'
|
|
55
|
-
INNER JOIN entity_observations o
|
|
56
|
-
ON o.id = l.entity_observation_id
|
|
57
|
-
WHERE n.status = 'active'
|
|
58
|
-
AND n.merged_into IS NULL
|
|
59
|
-
AND n.created_at <= ?
|
|
60
|
-
GROUP BY n.id, n.kind, n.preferred_label, n.scope_kind, n.scope_id, n.created_at
|
|
61
|
-
`)
|
|
62
|
-
.all(minCreatedAt);
|
|
63
|
-
const orphanRows = rows.map((row) => ({
|
|
64
|
-
entity_id: row.entity_id,
|
|
65
|
-
title: row.title,
|
|
66
|
-
kind: row.kind,
|
|
67
|
-
scope_kind: row.scope_kind,
|
|
68
|
-
scope_id: row.scope_id,
|
|
69
|
-
scope_label: buildScopeLabel(row.scope_kind, row.scope_id),
|
|
70
|
-
created_at: Number(row.created_at),
|
|
71
|
-
linked_decision_count: 0,
|
|
72
|
-
evidence_summary: {
|
|
73
|
-
lineage_rows: Number(row.lineage_rows ?? 0),
|
|
74
|
-
raw_evidence_rows: Number(row.raw_evidence_rows ?? 0),
|
|
75
|
-
last_seen_at: row.last_seen_at === null || row.last_seen_at === undefined
|
|
76
|
-
? null
|
|
77
|
-
: Number(row.last_seen_at),
|
|
78
|
-
},
|
|
79
|
-
}));
|
|
80
|
-
const linkedCounts = (0, entity_linked_decision_counts_js_1.loadLinkedDecisionCounts)(adapter, orphanRows.map((row) => row.entity_id));
|
|
81
|
-
return orphanRows
|
|
82
|
-
.map((row) => ({
|
|
83
|
-
...row,
|
|
84
|
-
linked_decision_count: linkedCounts.get(row.entity_id) ?? 0,
|
|
85
|
-
}))
|
|
86
|
-
.filter((row) => row.evidence_summary.lineage_rows > 0 &&
|
|
87
|
-
row.linked_decision_count === 0 &&
|
|
88
|
-
!isNoisyCanonicalEntity(row))
|
|
89
|
-
.sort(compareOrphanRows);
|
|
90
|
-
}
|
|
91
3
|
//# sourceMappingURL=entity-orphan-list.js.map
|
|
@@ -17,5 +17,4 @@ export interface SearchCanonicalEntitiesResult {
|
|
|
17
17
|
entities: EntitySearchRow[];
|
|
18
18
|
next_cursor: string | null;
|
|
19
19
|
}
|
|
20
|
-
export declare function searchCanonicalEntities(input: SearchCanonicalEntitiesInput): Promise<SearchCanonicalEntitiesResult>;
|
|
21
20
|
//# sourceMappingURL=entity-search.d.ts.map
|