@jungjaehoon/mama-core 1.4.2 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -10
- package/db/migrations/026-create-canonical-entity-tables.sql +99 -0
- package/db/migrations/027-create-entity-review-support.sql +44 -0
- package/db/migrations/028-create-entity-audit-tables.sql +45 -0
- package/db/migrations/029-extend-entity-observations-observation-type.sql +60 -0
- package/db/migrations/030-case-first-memory-substrate.sql +901 -0
- package/db/migrations/031-wiki-page-index-provenance-columns.sql +2 -0
- package/db/migrations/032-add-memory-provenance-columns.sql +21 -0
- package/db/migrations/033-create-model-runs-and-tool-traces.sql +53 -0
- package/db/migrations/034-add-connector-event-scope-columns.sql +14 -0
- package/db/migrations/035-create-twin-edges.sql +63 -0
- package/db/migrations/036-create-agent-situation-packets.sql +61 -0
- package/dist/agent-graph/alias-write.d.ts +3 -0
- package/dist/agent-graph/alias-write.js +231 -0
- package/dist/agent-graph/entity-resolve.d.ts +3 -0
- package/dist/agent-graph/entity-resolve.js +275 -0
- package/dist/agent-graph/errors.d.ts +4 -0
- package/dist/agent-graph/errors.js +11 -0
- package/dist/agent-graph/graph-query.d.ts +5 -0
- package/dist/agent-graph/graph-query.js +359 -0
- package/dist/agent-graph/index.d.ts +6 -0
- package/dist/agent-graph/index.js +22 -0
- package/dist/agent-graph/types.d.ts +162 -0
- package/dist/agent-graph/types.js +3 -0
- package/dist/agent-situation/builder.d.ts +8 -0
- package/dist/agent-situation/builder.js +239 -0
- package/dist/agent-situation/cache-key.d.ts +4 -0
- package/dist/agent-situation/cache-key.js +64 -0
- package/dist/agent-situation/index.d.ts +7 -0
- package/dist/agent-situation/index.js +28 -0
- package/dist/agent-situation/packet-store.d.ts +27 -0
- package/dist/agent-situation/packet-store.js +228 -0
- package/dist/agent-situation/ranking-policy.d.ts +18 -0
- package/dist/agent-situation/ranking-policy.js +75 -0
- package/dist/agent-situation/source-readers.d.ts +52 -0
- package/dist/agent-situation/source-readers.js +300 -0
- package/dist/agent-situation/types.d.ts +195 -0
- package/dist/agent-situation/types.js +3 -0
- package/dist/canonicalize.d.ts +38 -0
- package/dist/canonicalize.js +106 -0
- package/dist/cases/case-links.d.ts +116 -0
- package/dist/cases/case-links.js +794 -0
- package/dist/cases/composition-overrides.d.ts +90 -0
- package/dist/cases/composition-overrides.js +724 -0
- package/dist/cases/corrections.d.ts +156 -0
- package/dist/cases/corrections.js +1045 -0
- package/dist/cases/freshness.d.ts +57 -0
- package/dist/cases/freshness.js +283 -0
- package/dist/cases/live-state.d.ts +48 -0
- package/dist/cases/live-state.js +256 -0
- package/dist/cases/membership-explain.d.ts +56 -0
- package/dist/cases/membership-explain.js +140 -0
- package/dist/cases/membership-matcher.d.ts +23 -0
- package/dist/cases/membership-matcher.js +193 -0
- package/dist/cases/merge-split.d.ts +61 -0
- package/dist/cases/merge-split.js +368 -0
- package/dist/cases/role-inference.d.ts +10 -0
- package/dist/cases/role-inference.js +85 -0
- package/dist/cases/search-rollup.d.ts +27 -0
- package/dist/cases/search-rollup.js +309 -0
- package/dist/cases/sqlite-transaction.d.ts +10 -0
- package/dist/cases/sqlite-transaction.js +62 -0
- package/dist/cases/store.d.ts +74 -0
- package/dist/cases/store.js +816 -0
- package/dist/cases/target-ref.d.ts +25 -0
- package/dist/cases/target-ref.js +22 -0
- package/dist/cases/timeline-range.d.ts +6 -0
- package/dist/cases/timeline-range.js +322 -0
- package/dist/cases/tombstone-sweeper.d.ts +13 -0
- package/dist/cases/tombstone-sweeper.js +128 -0
- package/dist/cases/types.d.ts +244 -0
- package/dist/cases/types.js +47 -0
- package/dist/cases/wiki-page-index.d.ts +50 -0
- package/dist/cases/wiki-page-index.js +350 -0
- package/dist/connectors/event-index.d.ts +45 -0
- package/dist/connectors/event-index.js +381 -0
- package/dist/connectors/raw-query.d.ts +18 -0
- package/dist/connectors/raw-query.js +305 -0
- package/dist/connectors/types.d.ts +109 -0
- package/dist/connectors/types.js +3 -0
- package/dist/db-adapter/node-sqlite-adapter.d.ts +6 -0
- package/dist/db-adapter/node-sqlite-adapter.js +131 -0
- package/dist/db-manager.d.ts +12 -1
- package/dist/db-manager.js +60 -5
- package/dist/decision-formatter.d.ts +2 -0
- package/dist/decision-formatter.js +11 -8
- package/dist/edges/ref-validation.d.ts +8 -0
- package/dist/edges/ref-validation.js +262 -0
- package/dist/edges/store.d.ts +10 -0
- package/dist/edges/store.js +343 -0
- package/dist/edges/types.d.ts +86 -0
- package/dist/edges/types.js +18 -0
- package/dist/embedding-server/mobile/daemon.d.ts +1 -0
- package/dist/embedding-server/mobile/daemon.js +11 -1
- package/dist/embeddings.js +13 -0
- package/dist/entities/audit-metrics.d.ts +73 -0
- package/dist/entities/audit-metrics.js +128 -0
- package/dist/entities/candidate-generator.d.ts +9 -0
- package/dist/entities/candidate-generator.js +436 -0
- package/dist/entities/entity-impact.d.ts +41 -0
- package/dist/entities/entity-impact.js +140 -0
- package/dist/entities/entity-linked-decision-counts.d.ts +2 -0
- package/dist/entities/entity-linked-decision-counts.js +29 -0
- package/dist/entities/entity-list.d.ts +23 -0
- package/dist/entities/entity-list.js +99 -0
- package/dist/entities/entity-orphan-list.d.ts +26 -0
- package/dist/entities/entity-orphan-list.js +91 -0
- package/dist/entities/entity-search.d.ts +21 -0
- package/dist/entities/entity-search.js +150 -0
- package/dist/entities/errors.d.ts +65 -0
- package/dist/entities/errors.js +124 -0
- package/dist/entities/exact-merge-backfill.d.ts +11 -0
- package/dist/entities/exact-merge-backfill.js +161 -0
- package/dist/entities/lineage-backfill.d.ts +12 -0
- package/dist/entities/lineage-backfill.js +101 -0
- package/dist/entities/lineage-store.d.ts +51 -0
- package/dist/entities/lineage-store.js +308 -0
- package/dist/entities/normalization.d.ts +14 -0
- package/dist/entities/normalization.js +63 -0
- package/dist/entities/policy-store.d.ts +17 -0
- package/dist/entities/policy-store.js +263 -0
- package/dist/entities/policy-types.d.ts +72 -0
- package/dist/entities/policy-types.js +59 -0
- package/dist/entities/projection.d.ts +8 -0
- package/dist/entities/projection.js +82 -0
- package/dist/entities/provenance-query.d.ts +33 -0
- package/dist/entities/provenance-query.js +167 -0
- package/dist/entities/read-identity.d.ts +31 -0
- package/dist/entities/read-identity.js +264 -0
- package/dist/entities/recall-bridge.d.ts +5 -0
- package/dist/entities/recall-bridge.js +171 -0
- package/dist/entities/resolution-engine.d.ts +15 -0
- package/dist/entities/resolution-engine.js +69 -0
- package/dist/entities/rollback-preview.d.ts +39 -0
- package/dist/entities/rollback-preview.js +235 -0
- package/dist/entities/source-locator.d.ts +3 -0
- package/dist/entities/source-locator.js +32 -0
- package/dist/entities/store.d.ts +75 -0
- package/dist/entities/store.js +465 -0
- package/dist/entities/types.d.ts +162 -0
- package/dist/entities/types.js +30 -0
- package/dist/index.d.ts +62 -3
- package/dist/index.js +116 -26
- package/dist/mama-api.d.ts +53 -35
- package/dist/mama-api.js +437 -78
- package/dist/memory/api.d.ts +26 -38
- package/dist/memory/api.js +772 -68
- package/dist/memory/event-store.d.ts +3 -0
- package/dist/memory/event-store.js +21 -2
- package/dist/memory/provenance-audit.d.ts +22 -0
- package/dist/memory/provenance-audit.js +112 -0
- package/dist/memory/provenance-query.d.ts +11 -0
- package/dist/memory/provenance-query.js +193 -0
- package/dist/memory/provenance.d.ts +29 -0
- package/dist/memory/provenance.js +136 -0
- package/dist/memory/scope-backfill.d.ts +17 -0
- package/dist/memory/scope-backfill.js +77 -0
- package/dist/memory/types.d.ts +97 -16
- package/dist/model-runs/store.d.ts +13 -0
- package/dist/model-runs/store.js +457 -0
- package/dist/model-runs/tool-trace-store.d.ts +4 -0
- package/dist/model-runs/tool-trace-store.js +114 -0
- package/dist/model-runs/types.d.ts +69 -0
- package/dist/model-runs/types.js +5 -0
- package/dist/runtime/trusted-provenance.d.ts +3 -0
- package/dist/runtime/trusted-provenance.js +6 -0
- package/dist/search/feedback-store.d.ts +59 -0
- package/dist/search/feedback-store.js +211 -0
- package/dist/search/question-type.d.ts +5 -0
- package/dist/search/question-type.js +41 -0
- package/dist/search/ranker-features.d.ts +23 -0
- package/dist/search/ranker-features.js +104 -0
- package/dist/search/ranker-rescore.d.ts +31 -0
- package/dist/search/ranker-rescore.js +226 -0
- package/dist/search/ranker-trainer.d.ts +94 -0
- package/dist/search/ranker-trainer.js +860 -0
- package/dist/search/search-quality.d.ts +36 -0
- package/dist/search/search-quality.js +29 -0
- package/dist/test-utils.d.ts +14 -0
- package/dist/test-utils.js +52 -0
- package/package.json +28 -3
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CaseMergeChainCycleError = exports.AuditRunInProgressError = exports.CandidateStaleError = exports.EntityLabelMissingError = exports.MergeTargetStaleError = exports.OntologyViolationError = exports.EmbeddingUnavailableError = exports.InvalidEntityLabelError = exports.EntityError = void 0;
|
|
4
|
+
class EntityError extends Error {
|
|
5
|
+
context;
|
|
6
|
+
hint;
|
|
7
|
+
constructor(opts) {
|
|
8
|
+
super(opts.message);
|
|
9
|
+
this.name = new.target.name;
|
|
10
|
+
this.context = opts.context ?? {};
|
|
11
|
+
this.hint = opts.hint;
|
|
12
|
+
if (Error.captureStackTrace) {
|
|
13
|
+
Error.captureStackTrace(this, new.target);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
toErrorEnvelope() {
|
|
17
|
+
return {
|
|
18
|
+
error: {
|
|
19
|
+
code: this.code,
|
|
20
|
+
message: this.message,
|
|
21
|
+
hint: this.hint,
|
|
22
|
+
doc_url: `docs/operations/entity-substrate-runbook.md${this.doc_section}`,
|
|
23
|
+
},
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
exports.EntityError = EntityError;
|
|
28
|
+
class InvalidEntityLabelError extends EntityError {
|
|
29
|
+
code = 'entity.invalid_label';
|
|
30
|
+
doc_section = '#invalid-label';
|
|
31
|
+
constructor(context = {}) {
|
|
32
|
+
super({
|
|
33
|
+
message: 'Invalid entity label.',
|
|
34
|
+
context,
|
|
35
|
+
hint: 'Provide a non-empty entity label before normalization.',
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
exports.InvalidEntityLabelError = InvalidEntityLabelError;
|
|
40
|
+
class EmbeddingUnavailableError extends EntityError {
|
|
41
|
+
code = 'entity.embedding_unavailable';
|
|
42
|
+
doc_section = '#embedding-unavailable';
|
|
43
|
+
constructor(context = {}) {
|
|
44
|
+
super({
|
|
45
|
+
message: 'Entity embedding model is unavailable.',
|
|
46
|
+
context,
|
|
47
|
+
hint: 'Verify the embedding model is installed and reachable before generating candidates.',
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
exports.EmbeddingUnavailableError = EmbeddingUnavailableError;
|
|
52
|
+
class OntologyViolationError extends EntityError {
|
|
53
|
+
code = 'entity.ontology_violation';
|
|
54
|
+
doc_section = '#ontology-violation';
|
|
55
|
+
constructor(context = {}) {
|
|
56
|
+
super({
|
|
57
|
+
message: 'Entity operation violates ontology constraints.',
|
|
58
|
+
context,
|
|
59
|
+
hint: 'Check entity kinds and relation signatures before applying the change.',
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
exports.OntologyViolationError = OntologyViolationError;
|
|
64
|
+
class MergeTargetStaleError extends EntityError {
|
|
65
|
+
code = 'entity.merge_target_stale';
|
|
66
|
+
doc_section = '#merge-target-stale';
|
|
67
|
+
constructor(context = {}) {
|
|
68
|
+
super({
|
|
69
|
+
message: 'Merge target is stale.',
|
|
70
|
+
context,
|
|
71
|
+
hint: 'Reload the target entity state and retry the merge against the latest version.',
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
exports.MergeTargetStaleError = MergeTargetStaleError;
|
|
76
|
+
class EntityLabelMissingError extends EntityError {
|
|
77
|
+
code = 'entity.label_missing';
|
|
78
|
+
doc_section = '#label-missing';
|
|
79
|
+
constructor(context = {}) {
|
|
80
|
+
super({
|
|
81
|
+
message: 'Canonical entity label is missing.',
|
|
82
|
+
context,
|
|
83
|
+
hint: 'Set a preferred label before projecting or displaying the entity.',
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
exports.EntityLabelMissingError = EntityLabelMissingError;
|
|
88
|
+
class CandidateStaleError extends EntityError {
|
|
89
|
+
code = 'entity.candidate_stale';
|
|
90
|
+
doc_section = '#candidate-stale';
|
|
91
|
+
constructor(context = {}) {
|
|
92
|
+
super({
|
|
93
|
+
message: 'Resolution candidate is stale.',
|
|
94
|
+
context,
|
|
95
|
+
hint: 'Refresh the candidate list and review the latest evidence before acting.',
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
exports.CandidateStaleError = CandidateStaleError;
|
|
100
|
+
class AuditRunInProgressError extends EntityError {
|
|
101
|
+
code = 'entity.audit_run_in_progress';
|
|
102
|
+
doc_section = '#audit-run-in-progress';
|
|
103
|
+
constructor(context = {}) {
|
|
104
|
+
super({
|
|
105
|
+
message: 'Another entity audit run is already in progress.',
|
|
106
|
+
context,
|
|
107
|
+
hint: 'Wait for the current audit to finish or inspect its run status before starting a new one.',
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
exports.AuditRunInProgressError = AuditRunInProgressError;
|
|
112
|
+
class CaseMergeChainCycleError extends EntityError {
|
|
113
|
+
code = 'case.merge_chain_cycle';
|
|
114
|
+
doc_section = '#case-merge-chain-cycle';
|
|
115
|
+
constructor(context) {
|
|
116
|
+
super({
|
|
117
|
+
message: `Case merge chain cycle detected at depth ${context.detected_at_depth} starting from case_id=${context.case_id}. Chain=${context.chain.join(' -> ')}.`,
|
|
118
|
+
context,
|
|
119
|
+
hint: 'Inspect case_truth.canonical_case_id links and remove the cycle before retrying.',
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
exports.CaseMergeChainCycleError = CaseMergeChainCycleError;
|
|
124
|
+
//# sourceMappingURL=errors.js.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface ExactMergeBackfillOptions {
|
|
2
|
+
dryRun?: boolean;
|
|
3
|
+
}
|
|
4
|
+
export interface ExactMergeBackfillResult {
|
|
5
|
+
groups: number;
|
|
6
|
+
merged: number;
|
|
7
|
+
incomplete: number;
|
|
8
|
+
skipped: number;
|
|
9
|
+
}
|
|
10
|
+
export declare function backfillExactDuplicateCanonicals(opts?: ExactMergeBackfillOptions): Promise<ExactMergeBackfillResult>;
|
|
11
|
+
//# sourceMappingURL=exact-merge-backfill.d.ts.map
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.backfillExactDuplicateCanonicals = backfillExactDuplicateCanonicals;
|
|
4
|
+
const db_manager_js_1 = require("../db-manager.js");
|
|
5
|
+
const store_js_1 = require("./store.js");
|
|
6
|
+
const lineage_store_js_1 = require("./lineage-store.js");
|
|
7
|
+
function normalizeExactLabel(input) {
|
|
8
|
+
return input.normalize('NFKC').trim().replace(/\s+/gu, ' ').toLowerCase();
|
|
9
|
+
}
|
|
10
|
+
function buildExactGroupKey(row) {
|
|
11
|
+
if (row.normalized_form) {
|
|
12
|
+
return {
|
|
13
|
+
key: JSON.stringify([row.kind, row.scope_kind, row.scope_id ?? '', row.normalized_form]),
|
|
14
|
+
fromObservation: true,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
const fallback = normalizeExactLabel(row.preferred_label);
|
|
18
|
+
if (!fallback) {
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
return {
|
|
22
|
+
key: JSON.stringify([row.kind, row.scope_kind, row.scope_id ?? '', fallback]),
|
|
23
|
+
fromObservation: false,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
function compareRows(left, right) {
|
|
27
|
+
if (left.created_at !== right.created_at) {
|
|
28
|
+
return left.created_at - right.created_at;
|
|
29
|
+
}
|
|
30
|
+
return left.id.localeCompare(right.id);
|
|
31
|
+
}
|
|
32
|
+
function runAdapterTransaction(adapter, fn) {
|
|
33
|
+
const result = adapter.transaction(fn);
|
|
34
|
+
return typeof result === 'function' ? result() : result;
|
|
35
|
+
}
|
|
36
|
+
function requireTransactionAdapter(adapter) {
|
|
37
|
+
if (!('transaction' in adapter) || typeof adapter.transaction !== 'function') {
|
|
38
|
+
throw new Error('exact-merge-backfill requires adapter.transaction');
|
|
39
|
+
}
|
|
40
|
+
return adapter;
|
|
41
|
+
}
|
|
42
|
+
function mergeAndAdoptLineage(input) {
|
|
43
|
+
const mergeResult = (0, store_js_1.mergeEntityNodes)({
|
|
44
|
+
adapter: input.adapter,
|
|
45
|
+
source_id: input.source.id,
|
|
46
|
+
target_id: input.target.id,
|
|
47
|
+
actor_type: 'system',
|
|
48
|
+
actor_id: 'exact-duplicate-backfill',
|
|
49
|
+
reason: 'Exact duplicate canonical backfill',
|
|
50
|
+
candidate_id: null,
|
|
51
|
+
evidence_json: JSON.stringify({
|
|
52
|
+
strategy: 'exact_duplicate_backfill',
|
|
53
|
+
group_key: input.groupKey,
|
|
54
|
+
normalized_form: input.source.normalized_form,
|
|
55
|
+
target_id: input.target.id,
|
|
56
|
+
}),
|
|
57
|
+
});
|
|
58
|
+
(0, lineage_store_js_1.adoptLineageAfterMergeSync)({
|
|
59
|
+
adapter: input.adapter,
|
|
60
|
+
source_entity_id: input.source.id,
|
|
61
|
+
target_entity_id: input.target.id,
|
|
62
|
+
candidate_id: null,
|
|
63
|
+
review_action_id: mergeResult.merge_action_id,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
async function backfillExactDuplicateCanonicals(opts = {}) {
|
|
67
|
+
await (0, db_manager_js_1.initDB)();
|
|
68
|
+
const adapter = (0, db_manager_js_1.getAdapter)();
|
|
69
|
+
const dryRun = opts.dryRun === true;
|
|
70
|
+
let groups = 0;
|
|
71
|
+
let merged = 0;
|
|
72
|
+
let incomplete = 0;
|
|
73
|
+
let skipped = 0;
|
|
74
|
+
const rows = adapter
|
|
75
|
+
.prepare(`
|
|
76
|
+
SELECT
|
|
77
|
+
n.id,
|
|
78
|
+
n.kind,
|
|
79
|
+
n.preferred_label,
|
|
80
|
+
n.scope_kind,
|
|
81
|
+
n.scope_id,
|
|
82
|
+
n.created_at,
|
|
83
|
+
MIN(o.normalized_form) AS normalized_form
|
|
84
|
+
FROM entity_nodes n
|
|
85
|
+
LEFT JOIN entity_lineage_links l
|
|
86
|
+
ON l.canonical_entity_id = n.id
|
|
87
|
+
AND l.status = 'active'
|
|
88
|
+
LEFT JOIN entity_observations o
|
|
89
|
+
ON o.id = l.entity_observation_id
|
|
90
|
+
WHERE n.status = 'active'
|
|
91
|
+
AND n.merged_into IS NULL
|
|
92
|
+
GROUP BY n.id, n.kind, n.preferred_label, n.scope_kind, n.scope_id, n.created_at
|
|
93
|
+
`)
|
|
94
|
+
.all();
|
|
95
|
+
const grouped = new Map();
|
|
96
|
+
const fallbackRows = new Set();
|
|
97
|
+
for (const row of rows) {
|
|
98
|
+
const keyInfo = buildExactGroupKey(row);
|
|
99
|
+
if (!keyInfo) {
|
|
100
|
+
incomplete += 1;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (!keyInfo.fromObservation) {
|
|
104
|
+
fallbackRows.add(row.id);
|
|
105
|
+
}
|
|
106
|
+
const bucket = grouped.get(keyInfo.key) ?? [];
|
|
107
|
+
bucket.push(row);
|
|
108
|
+
grouped.set(keyInfo.key, bucket);
|
|
109
|
+
}
|
|
110
|
+
for (const bucket of grouped.values()) {
|
|
111
|
+
if (bucket.length >= 2) {
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
for (const row of bucket) {
|
|
115
|
+
if (fallbackRows.has(row.id)) {
|
|
116
|
+
incomplete += 1;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
for (const [groupKey, bucket] of grouped.entries()) {
|
|
121
|
+
if (bucket.length < 2) {
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
groups += 1;
|
|
125
|
+
const sorted = [...bucket].sort(compareRows);
|
|
126
|
+
const target = sorted[0];
|
|
127
|
+
const sources = sorted.slice(1);
|
|
128
|
+
if (dryRun) {
|
|
129
|
+
merged += sources.length;
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
try {
|
|
133
|
+
const transactionalAdapter = requireTransactionAdapter(adapter);
|
|
134
|
+
runAdapterTransaction(transactionalAdapter, () => {
|
|
135
|
+
for (const source of sources) {
|
|
136
|
+
mergeAndAdoptLineage({
|
|
137
|
+
adapter,
|
|
138
|
+
source,
|
|
139
|
+
target,
|
|
140
|
+
groupKey,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
});
|
|
144
|
+
merged += sources.length;
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
if (error instanceof store_js_1.EntityMergeError) {
|
|
148
|
+
skipped += sources.length;
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
throw error;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
groups,
|
|
156
|
+
merged,
|
|
157
|
+
incomplete,
|
|
158
|
+
skipped,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
//# sourceMappingURL=exact-merge-backfill.js.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
interface BackfillEntityLineageOptions {
|
|
2
|
+
dryRun?: boolean;
|
|
3
|
+
now?: number;
|
|
4
|
+
}
|
|
5
|
+
interface BackfillEntityLineageResult {
|
|
6
|
+
seeded: number;
|
|
7
|
+
adopted: number;
|
|
8
|
+
incomplete: number;
|
|
9
|
+
}
|
|
10
|
+
export declare function backfillEntityLineage(opts?: BackfillEntityLineageOptions): Promise<BackfillEntityLineageResult>;
|
|
11
|
+
export {};
|
|
12
|
+
//# sourceMappingURL=lineage-backfill.d.ts.map
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.backfillEntityLineage = backfillEntityLineage;
|
|
4
|
+
const db_manager_js_1 = require("../db-manager.js");
|
|
5
|
+
const lineage_store_js_1 = require("./lineage-store.js");
|
|
6
|
+
async function backfillEntityLineage(opts = {}) {
|
|
7
|
+
await (0, db_manager_js_1.initDB)();
|
|
8
|
+
const adapter = (0, db_manager_js_1.getAdapter)();
|
|
9
|
+
const dryRun = opts.dryRun === true;
|
|
10
|
+
const seedConfidence = 0.75;
|
|
11
|
+
const adoptConfidence = 0.6;
|
|
12
|
+
let seeded = 0;
|
|
13
|
+
let adopted = 0;
|
|
14
|
+
let incomplete = 0;
|
|
15
|
+
const activeRows = adapter
|
|
16
|
+
.prepare(`
|
|
17
|
+
SELECT n.id, o.id AS observation_id
|
|
18
|
+
FROM entity_nodes n
|
|
19
|
+
LEFT JOIN entity_observations o ON o.id = n.id
|
|
20
|
+
WHERE n.status = 'active'
|
|
21
|
+
AND n.merged_into IS NULL
|
|
22
|
+
`)
|
|
23
|
+
.all();
|
|
24
|
+
for (const row of activeRows) {
|
|
25
|
+
if (!row.observation_id) {
|
|
26
|
+
incomplete += 1;
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
const existing = adapter
|
|
30
|
+
.prepare(`
|
|
31
|
+
SELECT id
|
|
32
|
+
FROM entity_lineage_links
|
|
33
|
+
WHERE canonical_entity_id = ?
|
|
34
|
+
AND entity_observation_id = ?
|
|
35
|
+
AND status = 'active'
|
|
36
|
+
LIMIT 1
|
|
37
|
+
`)
|
|
38
|
+
.get(row.id, row.observation_id);
|
|
39
|
+
if (existing) {
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
seeded += 1;
|
|
43
|
+
if (!dryRun) {
|
|
44
|
+
await (0, lineage_store_js_1.appendEntityLineageLink)({
|
|
45
|
+
canonical_entity_id: row.id,
|
|
46
|
+
entity_observation_id: row.observation_id,
|
|
47
|
+
source_entity_id: null,
|
|
48
|
+
contribution_kind: 'seed',
|
|
49
|
+
run_id: null,
|
|
50
|
+
candidate_id: null,
|
|
51
|
+
review_action_id: null,
|
|
52
|
+
capture_mode: 'backfilled',
|
|
53
|
+
confidence: seedConfidence,
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const mergedRows = adapter
|
|
58
|
+
.prepare(`
|
|
59
|
+
SELECT n.id AS source_entity_id, n.merged_into AS target_entity_id, o.id AS observation_id
|
|
60
|
+
FROM entity_nodes n
|
|
61
|
+
LEFT JOIN entity_observations o ON o.id = n.id
|
|
62
|
+
WHERE n.status = 'merged'
|
|
63
|
+
AND n.merged_into IS NOT NULL
|
|
64
|
+
`)
|
|
65
|
+
.all();
|
|
66
|
+
for (const row of mergedRows) {
|
|
67
|
+
if (!row.observation_id) {
|
|
68
|
+
incomplete += 1;
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
const existing = adapter
|
|
72
|
+
.prepare(`
|
|
73
|
+
SELECT id
|
|
74
|
+
FROM entity_lineage_links
|
|
75
|
+
WHERE canonical_entity_id = ?
|
|
76
|
+
AND entity_observation_id = ?
|
|
77
|
+
AND status = 'active'
|
|
78
|
+
LIMIT 1
|
|
79
|
+
`)
|
|
80
|
+
.get(row.target_entity_id, row.observation_id);
|
|
81
|
+
if (existing) {
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
adopted += 1;
|
|
85
|
+
if (!dryRun) {
|
|
86
|
+
await (0, lineage_store_js_1.appendEntityLineageLink)({
|
|
87
|
+
canonical_entity_id: row.target_entity_id,
|
|
88
|
+
entity_observation_id: row.observation_id,
|
|
89
|
+
source_entity_id: row.source_entity_id,
|
|
90
|
+
contribution_kind: 'merge_adopt',
|
|
91
|
+
run_id: null,
|
|
92
|
+
candidate_id: null,
|
|
93
|
+
review_action_id: null,
|
|
94
|
+
capture_mode: 'backfilled',
|
|
95
|
+
confidence: adoptConfidence,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return { seeded, adopted, incomplete };
|
|
100
|
+
}
|
|
101
|
+
//# sourceMappingURL=lineage-backfill.js.map
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { DatabaseAdapter } from '../db-manager.js';
|
|
2
|
+
import type { EntityIngestRun, EntityLineageLink } from './types.js';
|
|
3
|
+
import type { EntityStoreAdapter } from './store.js';
|
|
4
|
+
type CreateEntityIngestRunInput = Omit<EntityIngestRun, 'status' | 'raw_count' | 'observation_count' | 'candidate_count' | 'reviewable_count' | 'audit_run_id' | 'audit_classification' | 'error_reason' | 'created_at' | 'completed_at'> & {
|
|
5
|
+
id?: string;
|
|
6
|
+
};
|
|
7
|
+
type AppendEntityLineageLinkInput = Omit<EntityLineageLink, 'id' | 'status' | 'created_at' | 'superseded_at'>;
|
|
8
|
+
interface CompleteEntityIngestRunInput {
|
|
9
|
+
raw_count: number;
|
|
10
|
+
observation_count: number;
|
|
11
|
+
candidate_count: number;
|
|
12
|
+
reviewable_count: number;
|
|
13
|
+
audit_run_id?: string | null;
|
|
14
|
+
audit_classification?: EntityIngestRun['audit_classification'];
|
|
15
|
+
}
|
|
16
|
+
interface AdoptLineageAfterMergeInput {
|
|
17
|
+
adapter?: LineageMutationAdapter;
|
|
18
|
+
source_entity_id: string;
|
|
19
|
+
target_entity_id: string;
|
|
20
|
+
candidate_id?: string | null;
|
|
21
|
+
review_action_id?: string | null;
|
|
22
|
+
confidence?: number;
|
|
23
|
+
capture_mode?: EntityLineageLink['capture_mode'];
|
|
24
|
+
}
|
|
25
|
+
type LineageMutationAdapter = Pick<DatabaseAdapter, 'prepare'>;
|
|
26
|
+
export interface AppendEntityLineageLinkResult {
|
|
27
|
+
link: EntityLineageLink;
|
|
28
|
+
created: boolean;
|
|
29
|
+
}
|
|
30
|
+
interface AdoptLineageAfterMergeSyncInput extends Omit<AdoptLineageAfterMergeInput, 'adapter'> {
|
|
31
|
+
adapter: LineageMutationAdapter;
|
|
32
|
+
}
|
|
33
|
+
export declare function parseEntityIngestRunRow(row: Record<string, unknown>): EntityIngestRun;
|
|
34
|
+
export declare function createEntityIngestRun(input: CreateEntityIngestRunInput): Promise<EntityIngestRun>;
|
|
35
|
+
export declare function completeEntityIngestRun(id: string, input: CompleteEntityIngestRunInput): Promise<EntityIngestRun>;
|
|
36
|
+
export declare function failEntityIngestRun(id: string, errorReason: string): Promise<EntityIngestRun>;
|
|
37
|
+
export declare function appendEntityLineageLink(input: AppendEntityLineageLinkInput): Promise<AppendEntityLineageLinkResult>;
|
|
38
|
+
export declare function supersedeEntityLineageForEntity(entityId: string): Promise<number>;
|
|
39
|
+
export declare function seedLineageForEntityMaterialization(input: {
|
|
40
|
+
canonical_entity_id: string;
|
|
41
|
+
entity_observation_id: string;
|
|
42
|
+
run_id?: string | null;
|
|
43
|
+
confidence?: number;
|
|
44
|
+
capture_mode?: EntityLineageLink['capture_mode'];
|
|
45
|
+
}): Promise<EntityLineageLink>;
|
|
46
|
+
export declare function adoptLineageAfterMerge(input: AdoptLineageAfterMergeInput): Promise<EntityLineageLink[]>;
|
|
47
|
+
export declare function adoptLineageAfterMergeSync(input: AdoptLineageAfterMergeSyncInput): EntityLineageLink[];
|
|
48
|
+
export declare function listActiveEntityLineage(entityId: string, adapter?: EntityStoreAdapter): Promise<EntityLineageLink[]>;
|
|
49
|
+
export declare function listEntityLineageHistory(entityId: string, adapter?: EntityStoreAdapter): Promise<EntityLineageLink[]>;
|
|
50
|
+
export {};
|
|
51
|
+
//# sourceMappingURL=lineage-store.d.ts.map
|