@jungjaehoon/mama-core 1.4.2 → 1.6.0

package/dist/entities/audit-metrics.js

@@ -0,0 +1,128 @@
+"use strict";
+/**
+ * Entity audit metric helpers.
+ *
+ * Pure functions — no DB, no embeddings. The audit queue assembles the
+ * snapshot from live tables and fixtures, then hands it to these helpers
+ * for deterministic metric calculation + baseline classification.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FALSE_MERGE_HARD_CEILING = void 0;
+exports.computeFalseMergeRate = computeFalseMergeRate;
+exports.computeCrossLanguageRecallAtK = computeCrossLanguageRecallAtK;
+exports.computeCrossLanguageRecallAt5 = computeCrossLanguageRecallAt5;
+exports.computeCrossLanguageRecallAt10 = computeCrossLanguageRecallAt10;
+exports.computeOntologyViolationCount = computeOntologyViolationCount;
+exports.computeProjectionFragmentationRate = computeProjectionFragmentationRate;
+exports.computeAuditMetrics = computeAuditMetrics;
+exports.classifyAuditRun = classifyAuditRun;
+exports.FALSE_MERGE_HARD_CEILING = 0.02;
+const RESOLVED_MERGE_STATUSES = new Set([
+    'auto_merged',
+    'approved',
+]);
+function computeFalseMergeRate(candidates) {
+    let denominator = 0;
+    let numerator = 0;
+    for (const candidate of candidates) {
+        if (!RESOLVED_MERGE_STATUSES.has(candidate.status))
+            continue;
+        denominator += 1;
+        if (candidate.left_canonical_id &&
+            candidate.right_canonical_id &&
+            candidate.left_canonical_id !== candidate.right_canonical_id) {
+            numerator += 1;
+        }
+    }
+    const rate = denominator === 0 ? 0 : numerator / denominator;
+    return { rate, numerator, denominator };
+}
+function computeCrossLanguageRecallAtK(goldPairs, matches, k) {
+    const denominator = goldPairs.length;
+    if (denominator === 0) {
+        return { recall: 0, numerator: 0, denominator: 0 };
+    }
+    const topRanked = [...matches]
+        .sort((a, b) => b.score_total - a.score_total)
+        .slice(0, Math.max(0, k))
+        .map((m) => m.matched_pair_key);
+    const matched = new Set(topRanked);
+    let numerator = 0;
+    for (const pair of goldPairs) {
+        const key = `${pair.canonical_id}:${pair.left_obs_id}:${pair.right_obs_id}`;
+        if (matched.has(key)) {
+            numerator += 1;
+        }
+    }
+    return { recall: numerator / denominator, numerator, denominator };
+}
+function computeCrossLanguageRecallAt5(goldPairs, matches) {
+    return computeCrossLanguageRecallAtK(goldPairs, matches, 5);
+}
+function computeCrossLanguageRecallAt10(goldPairs, matches) {
+    return computeCrossLanguageRecallAtK(goldPairs, matches, 10);
+}
+function computeOntologyViolationCount(candidates) {
+    let count = 0;
+    for (const candidate of candidates) {
+        if (candidate.left_kind &&
+            candidate.right_kind &&
+            candidate.left_kind !== candidate.right_kind) {
+            count += 1;
+        }
+    }
+    return count;
+}
+function computeProjectionFragmentationRate(counts) {
+    const keys = Object.keys(counts);
+    if (keys.length === 0)
+        return 0;
+    let fragmented = 0;
+    for (const key of keys) {
+        if ((counts[key] ?? 0) > 1)
+            fragmented += 1;
+    }
+    return fragmented / keys.length;
+}
+function computeAuditMetrics(snapshot) {
+    const falseMerge = computeFalseMergeRate(snapshot.candidates);
+    const recall = computeCrossLanguageRecallAt10(snapshot.gold_pairs, snapshot.candidate_gold_matches);
+    const ontology = computeOntologyViolationCount(snapshot.candidates);
+    const fragmentation = computeProjectionFragmentationRate(snapshot.gold_canonical_projection_counts);
+    return {
+        false_merge_rate: falseMerge.rate,
+        false_merge_numerator: falseMerge.numerator,
+        false_merge_denominator: falseMerge.denominator,
+        cross_language_candidate_recall_at_10: recall.recall,
+        cross_language_recall_numerator: recall.numerator,
+        cross_language_recall_denominator: recall.denominator,
+        ontology_violation_count: ontology,
+        projection_fragmentation_rate: fragmentation,
+    };
+}
+function classifyAuditRun(input) {
+    const { current, baseline } = input;
+    if (current.false_merge_rate > exports.FALSE_MERGE_HARD_CEILING) {
+        return 'regressed';
+    }
+    if (!baseline) {
+        return 'inconclusive';
+    }
+    if (current.false_merge_rate > baseline.false_merge_rate) {
+        return 'regressed';
+    }
+    const recallUp = current.cross_language_candidate_recall_at_10 > baseline.cross_language_candidate_recall_at_10;
+    const fragmentationDown = current.projection_fragmentation_rate < baseline.projection_fragmentation_rate;
+    const ontologyDown = current.ontology_violation_count < baseline.ontology_violation_count;
+    if (recallUp || fragmentationDown || ontologyDown) {
+        const anyRegression = current.cross_language_candidate_recall_at_10 <
+            baseline.cross_language_candidate_recall_at_10 ||
+            current.projection_fragmentation_rate > baseline.projection_fragmentation_rate ||
+            current.ontology_violation_count > baseline.ontology_violation_count;
+        if (anyRegression)
+            return 'stable';
+        return 'improved';
+    }
+    return 'stable';
+}
+//# sourceMappingURL=audit-metrics.js.map

package/dist/entities/candidate-generator.d.ts

@@ -0,0 +1,9 @@
+import type { EntityObservation, EntityResolutionCandidate } from './types.js';
+export declare const ENTITY_EMBEDDING_TOPN = 50;
+export interface CandidateGeneratorOptions {
+    embeddingScorer?: (left: EntityObservation, right: EntityObservation) => Promise<number>;
+    topN?: number;
+}
+export declare function dedupeObservationsBySource(observations: EntityObservation[]): EntityObservation[];
+export declare function generateResolutionCandidates(observations: EntityObservation[], options?: CandidateGeneratorOptions): Promise<EntityResolutionCandidate[]>;
+//# sourceMappingURL=candidate-generator.d.ts.map

package/dist/entities/candidate-generator.js

@@ -0,0 +1,436 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ENTITY_EMBEDDING_TOPN = void 0;
+exports.dedupeObservationsBySource = dedupeObservationsBySource;
+exports.generateResolutionCandidates = generateResolutionCandidates;
+const errors_js_1 = require("./errors.js");
+const normalization_js_1 = require("./normalization.js");
+exports.ENTITY_EMBEDDING_TOPN = 50;
+const CROSS_SCOPE_PROBE_MIN_EMBEDDING_SCORE = 0.84;
+const MAX_DETERMINISTIC_SCOPE_BUCKET = 32;
+const MAX_CROSS_SCOPE_SCOPE_BUCKET = 8;
+function tokenize(input) {
+    return input
+        .toLowerCase()
+        .split(/[\s,.!?;:()[\]{}"'/_-]+/u)
+        .map((token) => token.trim())
+        .filter((token) => token.length > 1);
+}
+function contextScore(left, right) {
+    const leftTokens = new Set(tokenize(left.context_summary ?? ''));
+    const rightTokens = new Set(tokenize(right.context_summary ?? ''));
+    if (leftTokens.size === 0 || rightTokens.size === 0) {
+        return 0;
+    }
+    let overlap = 0;
+    for (const token of leftTokens) {
+        if (rightTokens.has(token)) {
+            overlap += 1;
+        }
+    }
+    return overlap / Math.max(Math.min(leftTokens.size, rightTokens.size), 1);
+}
+function structuralScore(left, right) {
+    if (left.normalized_form === right.normalized_form) {
+        return 1;
+    }
+    const leftIds = (0, normalization_js_1.extractStructuredIdentifiers)(left.surface_form);
+    const rightIds = (0, normalization_js_1.extractStructuredIdentifiers)(right.surface_form);
+    const hasSharedEmail = leftIds.emails.some((value) => rightIds.emails.includes(value));
+    const hasSharedHandle = leftIds.handles.some((value) => rightIds.handles.includes(value));
+    const hasSharedDomain = leftIds.domains.some((value) => rightIds.domains.includes(value));
+    if (hasSharedEmail || hasSharedHandle || hasSharedDomain) {
+        return 0.75;
+    }
+    return 0;
+}
+function stringScore(left, right) {
+    if (left.normalized_form === right.normalized_form) {
+        return 1;
+    }
+    const leftText = left.surface_form.toLowerCase();
+    const rightText = right.surface_form.toLowerCase();
+    const sharedPrefix = leftText.length > 0 && rightText.length > 0 && leftText[0] === rightText[0];
+    return sharedPrefix ? 0.1 : 0;
+}
+function scorePair(left, right, probeKind) {
+    const score_structural = structuralScore(left, right);
+    const score_string = stringScore(left, right);
+    const score_context = contextScore(left, right);
+    const score_graph = 0;
+    const score_total = score_structural + score_string + score_context + score_graph;
+    return {
+        left,
+        right,
+        score_structural,
+        score_string,
+        score_context,
+        score_graph,
+        score_total,
+        probe_kind: probeKind,
+    };
+}
+function shouldConsiderPair(left, right) {
+    if (left.entity_kind_hint &&
+        right.entity_kind_hint &&
+        left.entity_kind_hint !== right.entity_kind_hint) {
+        return false;
+    }
+    if (left.scope_kind !== right.scope_kind) {
+        return false;
+    }
+    if (left.scope_id !== right.scope_id) {
+        return false;
+    }
+    return true;
+}
+function shouldConsiderCrossScopeProbe(left, right) {
+    if (!left.entity_kind_hint ||
+        !right.entity_kind_hint ||
+        left.entity_kind_hint !== right.entity_kind_hint) {
+        return false;
+    }
+    if (left.observation_type !== right.observation_type) {
+        return false;
+    }
+    if (left.scope_kind !== right.scope_kind) {
+        return false;
+    }
+    if (left.scope_id === right.scope_id) {
+        return false;
+    }
+    return true;
+}
+function buildBlockKeys(observation) {
+    const keys = new Set();
+    keys.add(`norm:${observation.normalized_form}`);
+    for (const related of observation.related_surface_forms) {
+        keys.add(`related:${related}`);
+    }
+    const ids = (0, normalization_js_1.extractStructuredIdentifiers)(observation.surface_form);
+    for (const email of ids.emails) {
+        keys.add(`email:${email}`);
+    }
+    for (const handle of ids.handles) {
+        keys.add(`handle:${handle}`);
+    }
+    for (const domain of ids.domains) {
+        keys.add(`domain:${domain}`);
+    }
+    const contextTokens = tokenize(observation.context_summary ?? '').slice(0, 3);
+    for (const token of contextTokens) {
+        keys.add(`ctx:${observation.scope_kind}:${observation.scope_id ?? 'global'}:${token}`);
+    }
+    return Array.from(keys);
+}
+function normalizeSurfaceForm(value) {
+    if (value.trim().length === 0) {
+        return '';
+    }
+    return (0, normalization_js_1.normalizeEntityLabel)(value).normalized;
+}
+function sharedNormalizedRelatedCount(left, right) {
+    const leftRelated = new Set(left.related_surface_forms.map((value) => normalizeSurfaceForm(value)));
+    const rightRelated = new Set(right.related_surface_forms.map((value) => normalizeSurfaceForm(value)));
+    if (leftRelated.size === 0 || rightRelated.size === 0) {
+        return 0;
+    }
+    let overlap = 0;
+    for (const related of leftRelated) {
+        if (rightRelated.has(related)) {
+            overlap += 1;
+        }
+    }
+    return overlap;
+}
+function getTimestampDistance(left, right) {
+    if (left.timestamp_observed === null || right.timestamp_observed === null) {
+        return Number.POSITIVE_INFINITY;
+    }
+    return Math.abs(left.timestamp_observed - right.timestamp_observed);
+}
+function buildPairKey(left, right) {
+    return `${left.id}::${right.id}`;
+}
+function compareObservationForSampling(left, right) {
+    const leftRelated = left.related_surface_forms.length;
+    const rightRelated = right.related_surface_forms.length;
+    if (rightRelated !== leftRelated) {
+        return rightRelated - leftRelated;
+    }
+    const leftContext = left.context_summary?.length ?? 0;
+    const rightContext = right.context_summary?.length ?? 0;
+    if (rightContext !== leftContext) {
+        return rightContext - leftContext;
+    }
+    const leftObserved = left.timestamp_observed ?? Number.NEGATIVE_INFINITY;
+    const rightObserved = right.timestamp_observed ?? Number.NEGATIVE_INFINITY;
+    if (rightObserved !== leftObserved) {
+        return rightObserved - leftObserved;
+    }
+    return left.id.localeCompare(right.id);
+}
+function capObservationGroup(observations, limit) {
+    if (observations.length <= limit) {
+        return observations;
+    }
+    return [...observations].sort(compareObservationForSampling).slice(0, limit);
+}
+function pushTopProbe(probes, candidate, maxPairs) {
+    probes.push(candidate);
+    probes.sort(compareProbePairs);
+    if (probes.length > maxPairs) {
+        probes.length = maxPairs;
+    }
+}
+function buildCrossLanguageFamilyKey(left, right) {
+    if (!left.lang || !right.lang || left.lang === right.lang) {
+        return null;
+    }
+    const forms = [
+        `${left.lang}:${left.normalized_form}`,
+        `${right.lang}:${right.normalized_form}`,
+    ].sort();
+    const observationTypes = [left.observation_type, right.observation_type].sort();
+    return JSON.stringify([
+        left.entity_kind_hint ?? right.entity_kind_hint ?? 'unknown',
+        observationTypes,
+        forms,
+    ]);
+}
+function compareProbePairs(left, right) {
+    const relatedDelta = sharedNormalizedRelatedCount(right.left, right.right) -
+        sharedNormalizedRelatedCount(left.left, left.right);
+    if (relatedDelta !== 0) {
+        return relatedDelta;
+    }
+    if (right.score_total !== left.score_total) {
+        return right.score_total - left.score_total;
+    }
+    const timestampDelta = getTimestampDistance(left.left, left.right) - getTimestampDistance(right.left, right.right);
+    if (timestampDelta !== 0) {
+        return timestampDelta;
+    }
+    return buildPairKey(left.left, left.right).localeCompare(buildPairKey(right.left, right.right));
+}
+function dedupeCrossLanguageFamilies(candidates, observationLookup) {
+    const deduped = [];
+    const bestByFamily = new Map();
+    for (const candidate of candidates) {
+        const left = observationLookup.get(candidate.left_ref);
+        const right = observationLookup.get(candidate.right_ref);
+        if (!left || !right) {
+            deduped.push(candidate);
+            continue;
+        }
+        const familyKey = buildCrossLanguageFamilyKey(left, right);
+        if (!familyKey) {
+            deduped.push(candidate);
+            continue;
+        }
+        const existing = bestByFamily.get(familyKey);
+        if (!existing ||
+            candidate.score_total > existing.score_total ||
+            (candidate.score_total === existing.score_total &&
+                candidate.id.localeCompare(existing.id) < 0)) {
+            bestByFamily.set(familyKey, candidate);
+        }
+    }
+    return deduped.concat(Array.from(bestByFamily.values()));
+}
+function buildCrossScopeProbePairs(observations, seenPairs, maxPairs) {
+    if (maxPairs <= 0) {
+        return [];
+    }
+    const probes = [];
+    const bucketMap = new Map();
+    for (const observation of observations) {
+        if (!observation.entity_kind_hint) {
+            continue;
+        }
+        const bucketKey = JSON.stringify([
+            observation.entity_kind_hint,
+            observation.observation_type,
+            observation.scope_kind,
+        ]);
+        const bucket = bucketMap.get(bucketKey) ?? [];
+        bucket.push(observation);
+        bucketMap.set(bucketKey, bucket);
+    }
+    for (const bucket of bucketMap.values()) {
+        const scopeBuckets = new Map();
+        for (const observation of bucket) {
+            const scopeKey = observation.scope_id ?? 'global';
+            const scopeBucket = scopeBuckets.get(scopeKey) ?? [];
+            scopeBucket.push(observation);
+            scopeBuckets.set(scopeKey, scopeBucket);
+        }
+        const limitedScopeBuckets = Array.from(scopeBuckets.values())
+            .filter((scopeBucket) => scopeBucket.length > 0)
+            .map((scopeBucket) => capObservationGroup(scopeBucket, MAX_CROSS_SCOPE_SCOPE_BUCKET));
+        for (let i = 0; i < limitedScopeBuckets.length; i += 1) {
+            for (let j = i + 1; j < limitedScopeBuckets.length; j += 1) {
+                const leftBucket = limitedScopeBuckets[i];
+                const rightBucket = limitedScopeBuckets[j];
+                for (const first of leftBucket) {
+                    for (const second of rightBucket) {
+                        const [left, right] = first.id.localeCompare(second.id) <= 0 ? [first, second] : [second, first];
+                        const pairKey = buildPairKey(left, right);
+                        if (seenPairs.has(pairKey)) {
+                            continue;
+                        }
+                        if (!shouldConsiderCrossScopeProbe(left, right)) {
+                            continue;
+                        }
+                        seenPairs.add(pairKey);
+                        pushTopProbe(probes, scorePair(left, right, 'cross_scope'), maxPairs);
+                    }
+                }
+            }
+        }
+    }
+    return probes;
+}
+function hasMultipleScopes(observations) {
+    const scopes = new Set(observations.map((observation) => `${observation.scope_kind}:${observation.scope_id ?? 'global'}`));
+    return scopes.size > 1;
+}
+function dedupeObservationsBySource(observations) {
+    const seen = new Set();
+    const deduped = [];
+    for (const observation of observations) {
+        const key = JSON.stringify([
+            observation.source_connector,
+            observation.source_locator ?? `missing-locator:${observation.id}`,
+            observation.source_raw_record_id,
+            observation.observation_type,
+        ]);
+        if (seen.has(key)) {
+            continue;
+        }
+        seen.add(key);
+        deduped.push(observation);
+    }
+    return deduped;
+}
+async function generateResolutionCandidates(observations, options = {}) {
+    const deduped = dedupeObservationsBySource(observations);
+    const observationLookup = new Map(deduped.map((observation) => [observation.id, observation]));
+    const blockMap = new Map();
+    for (const observation of deduped) {
+        const keys = buildBlockKeys(observation);
+        for (const key of keys) {
+            const existing = blockMap.get(key) ?? [];
+            existing.push(observation);
+            blockMap.set(key, existing);
+        }
+    }
+    const preliminary = [];
+    const seenPairs = new Set();
+    for (const bucket of blockMap.values()) {
+        if (bucket.length < 2) {
+            continue;
+        }
+        const scopeBuckets = new Map();
+        for (const observation of bucket) {
+            const scopeKey = `${observation.scope_kind}:${observation.scope_id ?? 'global'}`;
+            const scopeBucket = scopeBuckets.get(scopeKey) ?? [];
+            scopeBucket.push(observation);
+            scopeBuckets.set(scopeKey, scopeBucket);
+        }
+        for (const scopeBucket of scopeBuckets.values()) {
+            const limitedBucket = capObservationGroup(scopeBucket, MAX_DETERMINISTIC_SCOPE_BUCKET);
+            if (limitedBucket.length < 2) {
+                continue;
+            }
+            for (let i = 0; i < limitedBucket.length; i += 1) {
+                for (let j = i + 1; j < limitedBucket.length; j += 1) {
+                    const first = limitedBucket[i];
+                    const second = limitedBucket[j];
+                    const [left, right] = first.id.localeCompare(second.id) <= 0 ? [first, second] : [second, first];
+                    const pairKey = `${left.id}::${right.id}`;
+                    if (seenPairs.has(pairKey)) {
+                        continue;
+                    }
+                    if (!shouldConsiderPair(left, right)) {
+                        continue;
+                    }
+                    seenPairs.add(pairKey);
+                    const scored = scorePair(left, right, 'deterministic');
+                    if (scored.score_total <= 0 && !options.embeddingScorer) {
+                        continue;
+                    }
+                    preliminary.push(scored);
+                }
+            }
+        }
+    }
+    preliminary.sort((left, right) => right.score_total - left.score_total);
+    const topNLimit = options.topN ?? exports.ENTITY_EMBEDDING_TOPN;
+    const topN = preliminary.slice(0, topNLimit);
+    topN.sort((left, right) => right.score_total - left.score_total);
+    const pairBudget = Math.ceil(topNLimit ** 2 / 2);
+    const crossScopeBudget = options.embeddingScorer ? Math.max(0, pairBudget - topN.length) : 0;
+    const crossScopePairs = options.embeddingScorer && crossScopeBudget > 0 && hasMultipleScopes(deduped)
+        ? buildCrossScopeProbePairs(deduped, seenPairs, crossScopeBudget)
+        : [];
+    const candidates = [];
+    for (const item of [...topN, ...crossScopePairs]) {
+        let score_embedding = 0;
+        if (options.embeddingScorer) {
+            try {
+                score_embedding = await options.embeddingScorer(item.left, item.right);
+            }
+            catch (error) {
+                if (error instanceof errors_js_1.EmbeddingUnavailableError) {
+                    throw error;
+                }
+                throw new errors_js_1.EmbeddingUnavailableError({
+                    reason: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        const finalScore = item.score_total + score_embedding;
+        if (finalScore <= 0) {
+            continue;
+        }
+        if (item.probe_kind === 'cross_scope' &&
+            score_embedding < CROSS_SCOPE_PROBE_MIN_EMBEDDING_SCORE) {
+            continue;
+        }
+        const createdAt = Date.now();
+        candidates.push({
+            id: `candidate_${item.left.id}_${item.right.id}`,
+            candidate_kind: 'cluster',
+            left_ref: item.left.id,
+            right_ref: item.right.id,
+            status: 'pending',
+            score_total: finalScore,
+            score_structural: item.score_structural,
+            score_string: item.score_string,
+            score_context: item.score_context,
+            score_graph: item.score_graph,
+            score_embedding,
+            rule_trace: JSON.stringify({
+                structural: item.score_structural,
+                string: item.score_string,
+                context: item.score_context,
+            }),
+            extractor_version: item.left.extractor_version,
+            embedding_model_version: item.left.embedding_model_version ?? item.right.embedding_model_version,
+            created_at: createdAt,
+            updated_at: createdAt,
+        });
+    }
+    const dedupedFamilies = options.embeddingScorer
+        ? dedupeCrossLanguageFamilies(candidates, observationLookup)
+        : candidates;
+    dedupedFamilies.sort((left, right) => {
+        if (right.score_total !== left.score_total) {
+            return right.score_total - left.score_total;
+        }
+        return left.id.localeCompare(right.id);
+    });
+    return dedupedFamilies.slice(0, topNLimit);
+}
+//# sourceMappingURL=candidate-generator.js.map

package/dist/entities/entity-impact.d.ts

@@ -0,0 +1,41 @@
+import { type SourceLocatorKind } from './source-locator.js';
+import type { EntityIngestRun, EntityLineageLink, EntityNode } from './types.js';
+interface InspectorDetail {
+    entity: EntityNode;
+    history_incomplete: boolean;
+}
+interface InspectorLineageRow extends EntityLineageLink {
+    source_connector: string;
+    source_raw_record_id: string;
+    source_locator: string | null;
+    source_locator_kind: SourceLocatorKind;
+    surface_form: string;
+}
+interface InspectorLineageResult {
+    rows: InspectorLineageRow[];
+    history_incomplete: boolean;
+}
+interface ImpactMemoryRow {
+    id: string;
+    topic: string;
+    summary: string;
+    created_at: number;
+}
+interface ImpactAuditRunRow {
+    id: string;
+    classification: string | null;
+    status: string;
+    created_at: number;
+    completed_at: number | null;
+}
+interface EntityImpactResult {
+    related_memories: ImpactMemoryRow[];
+    ingest_runs: EntityIngestRun[];
+    audit_runs: ImpactAuditRunRow[];
+}
+export declare function getEntityInspectorDetail(entityId: string): Promise<InspectorDetail>;
+export declare function listEntityLineageForInspector(entityId: string, limit?: number): Promise<InspectorLineageResult>;
+export declare function getEntityImpact(entityId: string): Promise<EntityImpactResult>;
+export declare function getEntityIngestRun(runId: string): Promise<EntityIngestRun | null>;
+export {};
+//# sourceMappingURL=entity-impact.d.ts.map