@x12i/memorix-retrieval 1.8.2 → 1.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -5
- package/dist/client/types.d.ts +24 -0
- package/dist/client/types.d.ts.map +1 -1
- package/dist/data/collection-parser.d.ts +3 -0
- package/dist/data/collection-parser.d.ts.map +1 -1
- package/dist/data/collection-parser.js +3 -0
- package/dist/data/collection-parser.js.map +1 -1
- package/dist/data/record-identity.d.ts +3 -1
- package/dist/data/record-identity.d.ts.map +1 -1
- package/dist/data/record-identity.js +25 -4
- package/dist/data/record-identity.js.map +1 -1
- package/dist/explorer/collection-inventory.d.ts.map +1 -1
- package/dist/explorer/collection-inventory.js +23 -6
- package/dist/explorer/collection-inventory.js.map +1 -1
- package/dist/explorer/collection-records.d.ts.map +1 -1
- package/dist/explorer/collection-records.js +6 -2
- package/dist/explorer/collection-records.js.map +1 -1
- package/dist/explorer/entity-graph.d.ts +36 -1
- package/dist/explorer/entity-graph.d.ts.map +1 -1
- package/dist/explorer/entity-graph.js +161 -0
- package/dist/explorer/entity-graph.js.map +1 -1
- package/dist/explorer/health.d.ts +54 -0
- package/dist/explorer/health.d.ts.map +1 -1
- package/dist/explorer/health.js +135 -122
- package/dist/explorer/health.js.map +1 -1
- package/dist/explorer/raw-collection-records.d.ts +45 -7
- package/dist/explorer/raw-collection-records.d.ts.map +1 -1
- package/dist/explorer/raw-collection-records.js +46 -7
- package/dist/explorer/raw-collection-records.js.map +1 -1
- package/dist/explorer/unified-inventory.d.ts +80 -17
- package/dist/explorer/unified-inventory.d.ts.map +1 -1
- package/dist/explorer/unified-inventory.js +324 -89
- package/dist/explorer/unified-inventory.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/retrieval/fetch-item.d.ts.map +1 -1
- package/dist/retrieval/fetch-item.js +52 -3
- package/dist/retrieval/fetch-item.js.map +1 -1
- package/dist/retrieval/fetch-list.d.ts.map +1 -1
- package/dist/retrieval/fetch-list.js +46 -0
- package/dist/retrieval/fetch-list.js.map +1 -1
- package/dist/retrieval/fetch-slices.d.ts.map +1 -1
- package/dist/retrieval/fetch-slices.js +30 -9
- package/dist/retrieval/fetch-slices.js.map +1 -1
- package/dist/runtime/source-provenance.d.ts +94 -0
- package/dist/runtime/source-provenance.d.ts.map +1 -0
- package/dist/runtime/source-provenance.js +129 -0
- package/dist/runtime/source-provenance.js.map +1 -0
- package/dist/runtime/target-bindings.d.ts +29 -0
- package/dist/runtime/target-bindings.d.ts.map +1 -0
- package/dist/runtime/target-bindings.js +170 -0
- package/dist/runtime/target-bindings.js.map +1 -0
- package/dist/tests/collection-parser.test.js +8 -1
- package/dist/tests/collection-parser.test.js.map +1 -1
- package/dist/tests/entity-graph.test.js +21 -0
- package/dist/tests/entity-graph.test.js.map +1 -1
- package/dist/tests/source-provenance.test.d.ts +2 -0
- package/dist/tests/source-provenance.test.d.ts.map +1 -0
- package/dist/tests/source-provenance.test.js +94 -0
- package/dist/tests/source-provenance.test.js.map +1 -0
- package/dist/tests/unified-inventory.test.d.ts +2 -0
- package/dist/tests/unified-inventory.test.d.ts.map +1 -0
- package/dist/tests/unified-inventory.test.js +122 -0
- package/dist/tests/unified-inventory.test.js.map +1 -0
- package/docs/DATA-TIER-CONTRACT.md +20 -2
- package/docs/EXPLORER-HOST-APIS.md +21 -5
- package/docs/MEMORIX-CATALOX-CONTRACTS.md +14 -2
- package/docs/MEMORIX-DATABASE-CONVENTIONS.md +10 -5
- package/package.json +2 -2
|
@@ -1,14 +1,30 @@
|
|
|
1
1
|
import { parseMemorixCollectionName } from "../data/collection-parser.js";
|
|
2
2
|
import { resolveMemorixCollectionName } from "../data/collection-name.js";
|
|
3
|
+
import { resolveMemorixRecordIdentity } from "../data/record-identity.js";
|
|
4
|
+
import { serializeMemorixDocumentForJson } from "../data/record-id.js";
|
|
5
|
+
import { memorixReadByTarget } from "../data/target-read.js";
|
|
3
6
|
import { DEFAULT_MEMORIX_XRONOX_ROLES, } from "../data/memorix-read.js";
|
|
4
7
|
import { discoverMemorixEntities, loadMemorixEntityDescriptorByName, } from "../descriptors/discover-entities.js";
|
|
5
8
|
import { listEntityContentTypeKeys } from "../descriptors/content-type-labels.js";
|
|
6
|
-
import { MEMORIX_LIST_DESCRIPTORS_CATALOG } from "../descriptors/catalog-ids.js";
|
|
9
|
+
import { MEMORIX_ENTITY_DESCRIPTORS_CATALOG, MEMORIX_ITEM_DESCRIPTORS_CATALOG, MEMORIX_LIST_DESCRIPTORS_CATALOG, } from "../descriptors/catalog-ids.js";
|
|
7
10
|
import { throwRetrievalError } from "../errors/errors.js";
|
|
8
11
|
import { ALL_MEMORIX_TARGETS, resolveMemorixDatabaseName, resolveMemorixTargetFromDescriptor, } from "../mongo/target-config.js";
|
|
9
12
|
import { parseDescriptorData } from "../descriptors/validate-descriptor.js";
|
|
10
13
|
import { isMemorixCollectionIgnored, loadMemorixInventoryPolicy, } from "./inventory-policy.js";
|
|
14
|
+
import { mongoPayloadSourceRef, objectTypeDescriptorSourceRef, runtimeProjectionSourceRef, } from "../runtime/source-provenance.js";
|
|
15
|
+
import { countConfiguredTargets, countPhysicalDatabases, resolveMemorixTargetBindings, } from "../runtime/target-bindings.js";
|
|
11
16
|
const XRONOX_DB_SOURCE = "_db";
|
|
17
|
+
const DEFAULT_INVENTORY_SAMPLE_LIMIT = 3;
|
|
18
|
+
const INVENTORY_SAMPLE_PREVIEW_FIELDS = [
|
|
19
|
+
"name",
|
|
20
|
+
"title",
|
|
21
|
+
"label",
|
|
22
|
+
"entityId",
|
|
23
|
+
"eventId",
|
|
24
|
+
"knowledgeId",
|
|
25
|
+
"recordId",
|
|
26
|
+
"id",
|
|
27
|
+
];
|
|
12
28
|
function resolveXronoxRole(client, target) {
|
|
13
29
|
return client.xronoxRoles?.[target] ?? DEFAULT_MEMORIX_XRONOX_ROLES[target];
|
|
14
30
|
}
|
|
@@ -37,34 +53,47 @@ function rowKey(databaseName, collectionName) {
|
|
|
37
53
|
function makeRowId(target, databaseName, collectionName) {
|
|
38
54
|
return `${target}:${databaseName}:${collectionName}`;
|
|
39
55
|
}
|
|
56
|
+
function normalizeMappingStatus(status) {
|
|
57
|
+
if (status === "object-missing")
|
|
58
|
+
return "object-type-missing";
|
|
59
|
+
return status;
|
|
60
|
+
}
|
|
40
61
|
function actionsForStatus(status, listable) {
|
|
62
|
+
const normalized = normalizeMappingStatus(status);
|
|
41
63
|
const actions = [];
|
|
42
|
-
if (
|
|
43
|
-
actions.push(listable ? "open-records" : "open-raw-records");
|
|
64
|
+
if (normalized === "mapped" || normalized === "collection-mismatch") {
|
|
65
|
+
actions.push(listable ? "open-descriptor-backed-records" : "open-raw-records");
|
|
44
66
|
actions.push("inspect-descriptor");
|
|
67
|
+
actions.push("inspect-collection");
|
|
45
68
|
}
|
|
46
|
-
else if (
|
|
69
|
+
else if (normalized === "orphan-collection" || normalized === "unparseable") {
|
|
47
70
|
actions.push("open-raw-records");
|
|
48
|
-
actions.push("create-descriptor");
|
|
71
|
+
actions.push("create-descriptor-draft");
|
|
72
|
+
actions.push("ask-user");
|
|
49
73
|
}
|
|
50
|
-
else if (
|
|
74
|
+
else if (normalized === "missing-collection") {
|
|
51
75
|
actions.push("inspect-descriptor");
|
|
52
|
-
actions.push("fix-binding");
|
|
76
|
+
actions.push("fix-binding-draft");
|
|
53
77
|
}
|
|
54
|
-
else if (
|
|
55
|
-
|
|
56
|
-
actions.push("
|
|
78
|
+
else if (normalized === "object-type-missing" ||
|
|
79
|
+
normalized === "content-type-missing") {
|
|
80
|
+
actions.push("add-content-type-draft");
|
|
81
|
+
actions.push("create-descriptor-draft");
|
|
82
|
+
actions.push("ask-user");
|
|
57
83
|
}
|
|
58
|
-
if (
|
|
59
|
-
actions.push("ignore");
|
|
84
|
+
if (normalized === "ignored")
|
|
85
|
+
actions.push("ignore-collection");
|
|
60
86
|
return actions;
|
|
61
87
|
}
|
|
62
88
|
function issueSeverity(kind) {
|
|
63
89
|
switch (kind) {
|
|
64
90
|
case "empty-declared-collection":
|
|
91
|
+
case "target-db-not-configured":
|
|
92
|
+
case "target-db-empty":
|
|
65
93
|
return "info";
|
|
66
94
|
case "orphan-collection":
|
|
67
95
|
case "collection-mismatch":
|
|
96
|
+
case "object-type-missing":
|
|
68
97
|
case "object-missing":
|
|
69
98
|
case "content-type-missing":
|
|
70
99
|
case "unparseable-collection":
|
|
@@ -76,22 +105,152 @@ function issueSeverity(kind) {
|
|
|
76
105
|
function suggestedActionsForKind(kind) {
|
|
77
106
|
switch (kind) {
|
|
78
107
|
case "missing-collection":
|
|
79
|
-
return ["inspect", "fix-binding"];
|
|
108
|
+
return ["inspect", "fix-binding-draft"];
|
|
80
109
|
case "empty-declared-collection":
|
|
110
|
+
case "target-db-empty":
|
|
81
111
|
return ["inspect"];
|
|
112
|
+
case "target-db-not-configured":
|
|
113
|
+
return ["inspect", "ask-user"];
|
|
82
114
|
case "orphan-collection":
|
|
83
|
-
return ["inspect", "create-descriptor", "delete-or-archive"];
|
|
115
|
+
return ["inspect", "create-descriptor-draft", "delete-or-archive"];
|
|
116
|
+
case "object-type-missing":
|
|
84
117
|
case "object-missing":
|
|
85
118
|
case "content-type-missing":
|
|
86
|
-
return ["create-descriptor", "add-content-type"];
|
|
119
|
+
return ["create-descriptor-draft", "add-content-type-draft", "ask-user"];
|
|
87
120
|
case "collection-mismatch":
|
|
88
|
-
return ["inspect", "fix-binding"];
|
|
121
|
+
return ["inspect", "fix-binding-draft"];
|
|
89
122
|
case "unparseable-collection":
|
|
90
|
-
return ["inspect", "ignore"];
|
|
123
|
+
return ["inspect", "ignore-collection", "ask-user"];
|
|
91
124
|
default:
|
|
92
125
|
return ["inspect"];
|
|
93
126
|
}
|
|
94
127
|
}
|
|
128
|
+
function requiredUserInputsForStatus(status, row) {
|
|
129
|
+
const normalized = normalizeMappingStatus(status);
|
|
130
|
+
if (normalized === "unparseable") {
|
|
131
|
+
return [
|
|
132
|
+
{
|
|
133
|
+
key: "objectType",
|
|
134
|
+
label: "Object type",
|
|
135
|
+
reason: "Collection name does not follow {objectName}-{contentType}.",
|
|
136
|
+
required: true,
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
key: "contentType",
|
|
140
|
+
label: "Content type",
|
|
141
|
+
reason: "Collection name does not expose content type.",
|
|
142
|
+
required: true,
|
|
143
|
+
},
|
|
144
|
+
];
|
|
145
|
+
}
|
|
146
|
+
if (normalized === "object-type-missing") {
|
|
147
|
+
return [
|
|
148
|
+
{
|
|
149
|
+
key: "confirmObjectType",
|
|
150
|
+
label: "Confirm object type",
|
|
151
|
+
reason: `Parsed object type "${row.objectType ?? row.objectName}" does not exist in Catalox descriptors.`,
|
|
152
|
+
required: true,
|
|
153
|
+
options: ["create descriptor draft", "ignore collection"],
|
|
154
|
+
},
|
|
155
|
+
];
|
|
156
|
+
}
|
|
157
|
+
return undefined;
|
|
158
|
+
}
|
|
159
|
+
function buildDescriptorRefs(slot, listDescriptorIds) {
|
|
160
|
+
if (!slot)
|
|
161
|
+
return {};
|
|
162
|
+
return {
|
|
163
|
+
objectTypeDescriptor: {
|
|
164
|
+
catalogId: MEMORIX_ENTITY_DESCRIPTORS_CATALOG,
|
|
165
|
+
itemId: slot.entityDescriptorId,
|
|
166
|
+
},
|
|
167
|
+
listDescriptors: listDescriptorIds?.map((itemId) => ({
|
|
168
|
+
catalogId: MEMORIX_LIST_DESCRIPTORS_CATALOG,
|
|
169
|
+
itemId,
|
|
170
|
+
kind: "entity",
|
|
171
|
+
})),
|
|
172
|
+
itemDescriptors: slot.defaultItemDescriptorId
|
|
173
|
+
? [
|
|
174
|
+
{
|
|
175
|
+
catalogId: MEMORIX_ITEM_DESCRIPTORS_CATALOG,
|
|
176
|
+
itemId: slot.defaultItemDescriptorId,
|
|
177
|
+
},
|
|
178
|
+
]
|
|
179
|
+
: undefined,
|
|
180
|
+
defaultListDescriptorId: slot.defaultListDescriptorId,
|
|
181
|
+
defaultItemDescriptorId: slot.defaultItemDescriptorId,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
function buildLegacyDescriptor(descriptorRefs, listDescriptorIds) {
|
|
185
|
+
return {
|
|
186
|
+
entityDescriptorId: descriptorRefs.objectTypeDescriptor?.itemId,
|
|
187
|
+
listDescriptorIds,
|
|
188
|
+
defaultListDescriptorId: descriptorRefs.defaultListDescriptorId,
|
|
189
|
+
defaultItemDescriptorId: descriptorRefs.defaultItemDescriptorId,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
function buildRowSourceRefs(client, input) {
|
|
193
|
+
const sources = [];
|
|
194
|
+
if (input.entityDescriptorId) {
|
|
195
|
+
sources.push(objectTypeDescriptorSourceRef(input.entityDescriptorId, {
|
|
196
|
+
appId: client.appId,
|
|
197
|
+
objectType: input.objectName,
|
|
198
|
+
target: input.target,
|
|
199
|
+
}));
|
|
200
|
+
}
|
|
201
|
+
if (input.rowSource === "mongo-discovered" || input.rowSource === "merged") {
|
|
202
|
+
sources.push(mongoPayloadSourceRef(input.target, input.databaseName, input.collectionName, {
|
|
203
|
+
appId: client.appId,
|
|
204
|
+
objectType: input.objectName,
|
|
205
|
+
contentType: input.contentType,
|
|
206
|
+
}));
|
|
207
|
+
}
|
|
208
|
+
const source = runtimeProjectionSourceRef("buildMemorixUnifiedInventory", {
|
|
209
|
+
appId: client.appId,
|
|
210
|
+
});
|
|
211
|
+
return { source, sources };
|
|
212
|
+
}
|
|
213
|
+
function buildInventoryRow(client, sourceLens, input) {
|
|
214
|
+
const objectName = input.objectName ?? "";
|
|
215
|
+
const objectType = objectName || undefined;
|
|
216
|
+
const descriptorRefs = buildDescriptorRefs(input.slot, input.listDescriptorIds);
|
|
217
|
+
const mappingStatus = normalizeMappingStatus(input.mappingStatus);
|
|
218
|
+
const { source, sources } = buildRowSourceRefs(client, {
|
|
219
|
+
rowSource: input.rowSource,
|
|
220
|
+
target: input.target,
|
|
221
|
+
databaseName: input.databaseName,
|
|
222
|
+
collectionName: input.collectionName,
|
|
223
|
+
objectName,
|
|
224
|
+
contentType: input.contentType,
|
|
225
|
+
entityDescriptorId: input.slot?.entityDescriptorId ?? descriptorRefs.objectTypeDescriptor?.itemId,
|
|
226
|
+
});
|
|
227
|
+
return {
|
|
228
|
+
rowId: input.rowId,
|
|
229
|
+
source,
|
|
230
|
+
sources,
|
|
231
|
+
rowSource: input.rowSource,
|
|
232
|
+
sourceLens,
|
|
233
|
+
target: input.target,
|
|
234
|
+
databaseName: input.databaseName,
|
|
235
|
+
objectName,
|
|
236
|
+
objectType,
|
|
237
|
+
entityName: objectType,
|
|
238
|
+
contentType: input.contentType,
|
|
239
|
+
collectionName: input.collectionName,
|
|
240
|
+
expectedCollectionName: input.expectedCollectionName,
|
|
241
|
+
parse: input.parse,
|
|
242
|
+
descriptorRefs,
|
|
243
|
+
descriptor: buildLegacyDescriptor(descriptorRefs, input.listDescriptorIds),
|
|
244
|
+
counts: input.counts,
|
|
245
|
+
mappingStatus,
|
|
246
|
+
actions: actionsForStatus(mappingStatus, input.listable),
|
|
247
|
+
requiredUserInputs: requiredUserInputsForStatus(mappingStatus, {
|
|
248
|
+
collectionName: input.collectionName,
|
|
249
|
+
objectName,
|
|
250
|
+
objectType,
|
|
251
|
+
}),
|
|
252
|
+
};
|
|
253
|
+
}
|
|
95
254
|
async function scanMongoCollections(client, targets, includeExactCounts) {
|
|
96
255
|
const listCollections = requireXronoxListCollections(client);
|
|
97
256
|
const countDocuments = requireXronoxCountDocuments(client);
|
|
@@ -210,8 +369,9 @@ function buildIssuesFromRows(rows) {
|
|
|
210
369
|
kind = "collection-mismatch";
|
|
211
370
|
message = `Collection "${row.collectionName}" does not match expected "${row.expectedCollectionName}"`;
|
|
212
371
|
break;
|
|
372
|
+
case "object-type-missing":
|
|
213
373
|
case "object-missing":
|
|
214
|
-
kind = "object-missing";
|
|
374
|
+
kind = "object-type-missing";
|
|
215
375
|
message = `Parsed object "${row.objectName}" has no entity descriptor`;
|
|
216
376
|
break;
|
|
217
377
|
case "content-type-missing":
|
|
@@ -233,14 +393,18 @@ function buildIssuesFromRows(rows) {
|
|
|
233
393
|
issueId,
|
|
234
394
|
severity: issueSeverity(kind),
|
|
235
395
|
kind,
|
|
396
|
+
message,
|
|
397
|
+
source: row.source,
|
|
398
|
+
sources: row.sources,
|
|
236
399
|
target: row.target,
|
|
237
400
|
databaseName: row.databaseName,
|
|
238
|
-
|
|
239
|
-
contentType: row.contentType
|
|
401
|
+
objectType: row.objectType ?? row.objectName,
|
|
402
|
+
contentType: row.contentType,
|
|
240
403
|
collectionName: row.collectionName,
|
|
241
|
-
|
|
242
|
-
|
|
404
|
+
rowId: row.rowId,
|
|
405
|
+
descriptorId: row.descriptorRefs.objectTypeDescriptor?.itemId,
|
|
243
406
|
suggestedActions: suggestedActionsForKind(kind),
|
|
407
|
+
requiredUserInputs: row.requiredUserInputs,
|
|
244
408
|
});
|
|
245
409
|
if (row.mappingStatus === "mapped" && (row.counts.exact === 0 || row.counts.estimated === 0)) {
|
|
246
410
|
const emptyIssueId = `issue-${++counter}`;
|
|
@@ -249,13 +413,16 @@ function buildIssuesFromRows(rows) {
|
|
|
249
413
|
issueId: emptyIssueId,
|
|
250
414
|
severity: "info",
|
|
251
415
|
kind: "empty-declared-collection",
|
|
416
|
+
message: `Declared collection "${row.collectionName}" is empty`,
|
|
417
|
+
source: row.source,
|
|
418
|
+
sources: row.sources,
|
|
252
419
|
target: row.target,
|
|
253
420
|
databaseName: row.databaseName,
|
|
254
|
-
|
|
421
|
+
objectType: row.objectType ?? row.objectName,
|
|
255
422
|
contentType: row.contentType,
|
|
256
423
|
collectionName: row.collectionName,
|
|
257
|
-
|
|
258
|
-
|
|
424
|
+
rowId: row.rowId,
|
|
425
|
+
descriptorId: row.descriptorRefs.objectTypeDescriptor?.itemId,
|
|
259
426
|
suggestedActions: ["inspect"],
|
|
260
427
|
});
|
|
261
428
|
}
|
|
@@ -266,7 +433,7 @@ function summarizeTargets(rows, targets, dbs) {
|
|
|
266
433
|
const out = {};
|
|
267
434
|
for (const target of ALL_MEMORIX_TARGETS) {
|
|
268
435
|
const targetRows = rows.filter((r) => r.target === target);
|
|
269
|
-
const declared = targetRows.filter((r) => r.
|
|
436
|
+
const declared = targetRows.filter((r) => r.rowSource !== "mongo-discovered");
|
|
270
437
|
out[target] = {
|
|
271
438
|
databaseName: dbs[target],
|
|
272
439
|
declaredCollections: declared.length,
|
|
@@ -275,7 +442,7 @@ function summarizeTargets(rows, targets, dbs) {
|
|
|
275
442
|
orphanCollections: targetRows.filter((r) => r.mappingStatus === "orphan-collection").length,
|
|
276
443
|
emptyDeclaredCollections: targetRows.filter((r) => r.mappingStatus === "mapped" &&
|
|
277
444
|
(r.counts.exact === 0 || r.counts.estimated === 0)).length,
|
|
278
|
-
mongoCollections: targetRows.filter((r) => r.
|
|
445
|
+
mongoCollections: targetRows.filter((r) => r.rowSource === "mongo-discovered" || r.rowSource === "merged").length,
|
|
279
446
|
totalRecords: targetRows.reduce((sum, r) => sum + (r.counts.exact ?? r.counts.estimated ?? 0), 0),
|
|
280
447
|
};
|
|
281
448
|
}
|
|
@@ -294,6 +461,47 @@ function summarizeTargets(rows, targets, dbs) {
|
|
|
294
461
|
}
|
|
295
462
|
return out;
|
|
296
463
|
}
|
|
464
|
+
function buildSamplePreview(record, identityField) {
|
|
465
|
+
const preview = {};
|
|
466
|
+
for (const field of INVENTORY_SAMPLE_PREVIEW_FIELDS) {
|
|
467
|
+
const value = record[field];
|
|
468
|
+
if (value != null && typeof value !== "object") {
|
|
469
|
+
preview[field] = value;
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
if (identityField && record[identityField] != null) {
|
|
473
|
+
preview[identityField] = record[identityField];
|
|
474
|
+
}
|
|
475
|
+
return preview;
|
|
476
|
+
}
|
|
477
|
+
async function attachInventorySamples(client, rows) {
|
|
478
|
+
await Promise.all(rows.map(async (row) => {
|
|
479
|
+
if (!row.collectionName || row.mappingStatus === "missing-collection")
|
|
480
|
+
return;
|
|
481
|
+
const count = row.counts.exact ?? row.counts.estimated ?? 0;
|
|
482
|
+
if (count <= 0)
|
|
483
|
+
return;
|
|
484
|
+
try {
|
|
485
|
+
const docs = await memorixReadByTarget(client, {
|
|
486
|
+
target: row.target,
|
|
487
|
+
collection: row.collectionName,
|
|
488
|
+
limit: DEFAULT_INVENTORY_SAMPLE_LIMIT,
|
|
489
|
+
});
|
|
490
|
+
row.samples = docs.map((doc) => {
|
|
491
|
+
const serialized = serializeMemorixDocumentForJson(doc);
|
|
492
|
+
const identity = resolveMemorixRecordIdentity(row.target, serialized);
|
|
493
|
+
return {
|
|
494
|
+
recordId: identity.recordId,
|
|
495
|
+
identityField: identity.identityField,
|
|
496
|
+
preview: buildSamplePreview(serialized, identity.identityField),
|
|
497
|
+
};
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
catch {
|
|
501
|
+
/* samples are optional enrichment */
|
|
502
|
+
}
|
|
503
|
+
}));
|
|
504
|
+
}
|
|
297
505
|
export async function buildMemorixUnifiedInventory(client, options = {}) {
|
|
298
506
|
const sourceLens = options.sourceLens ?? "catalox-first";
|
|
299
507
|
const targets = options.targets ?? [...ALL_MEMORIX_TARGETS];
|
|
@@ -324,14 +532,17 @@ export async function buildMemorixUnifiedInventory(client, options = {}) {
|
|
|
324
532
|
if (!targets.includes(row.target))
|
|
325
533
|
return;
|
|
326
534
|
if (inventoryPolicy &&
|
|
535
|
+
row.collectionName &&
|
|
327
536
|
isMemorixCollectionIgnored(inventoryPolicy, row.target, row.collectionName, row.databaseName)) {
|
|
328
537
|
row.mappingStatus = "ignored";
|
|
329
|
-
row.actions = ["ignore"];
|
|
538
|
+
row.actions = ["ignore-collection"];
|
|
330
539
|
}
|
|
331
540
|
if (row.mappingStatus === "ignored" && !options.includeIgnored)
|
|
332
541
|
return;
|
|
333
542
|
rows.push(row);
|
|
334
|
-
|
|
543
|
+
if (row.databaseName && row.collectionName) {
|
|
544
|
+
seenKeys.add(rowKey(row.databaseName, row.collectionName));
|
|
545
|
+
}
|
|
335
546
|
}
|
|
336
547
|
if (sourceLens === "catalox-first") {
|
|
337
548
|
for (const slot of declaredSlots) {
|
|
@@ -339,17 +550,10 @@ export async function buildMemorixUnifiedInventory(client, options = {}) {
|
|
|
339
550
|
const scannedCol = scanned.get(key);
|
|
340
551
|
const count = scannedCol?.count ?? 0;
|
|
341
552
|
const exists = scannedCol != null;
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
else if (count > 0) {
|
|
347
|
-
mappingStatus = "mapped";
|
|
348
|
-
}
|
|
349
|
-
else {
|
|
350
|
-
mappingStatus = "mapped";
|
|
351
|
-
}
|
|
352
|
-
addRow({
|
|
553
|
+
const mappingStatus = !exists
|
|
554
|
+
? "missing-collection"
|
|
555
|
+
: "mapped";
|
|
556
|
+
addRow(buildInventoryRow(client, sourceLens, {
|
|
353
557
|
rowId: makeRowId(slot.target, slot.databaseName, slot.collectionName),
|
|
354
558
|
target: slot.target,
|
|
355
559
|
databaseName: slot.databaseName,
|
|
@@ -357,20 +561,16 @@ export async function buildMemorixUnifiedInventory(client, options = {}) {
|
|
|
357
561
|
contentType: slot.contentType,
|
|
358
562
|
collectionName: slot.collectionName,
|
|
359
563
|
expectedCollectionName: slot.expectedCollectionName,
|
|
360
|
-
|
|
564
|
+
rowSource: exists ? "merged" : "catalox-declared",
|
|
361
565
|
mappingStatus,
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
listDescriptorIds: listIdsByEntity.get(slot.objectName),
|
|
365
|
-
defaultListDescriptorId: slot.defaultListDescriptorId,
|
|
366
|
-
defaultItemDescriptorId: slot.defaultItemDescriptorId,
|
|
367
|
-
},
|
|
566
|
+
slot,
|
|
567
|
+
listDescriptorIds: listIdsByEntity.get(slot.objectName),
|
|
368
568
|
counts: {
|
|
369
569
|
...(includeExactCounts ? { exact: count } : { estimated: count }),
|
|
370
570
|
countMode: exists ? countMode : "none",
|
|
371
571
|
},
|
|
372
|
-
|
|
373
|
-
});
|
|
572
|
+
listable: slot.listable,
|
|
573
|
+
}));
|
|
374
574
|
}
|
|
375
575
|
for (const [key, col] of scanned) {
|
|
376
576
|
if (declaredByKey.has(key))
|
|
@@ -393,21 +593,22 @@ export async function buildMemorixUnifiedInventory(client, options = {}) {
|
|
|
393
593
|
mappingStatus = col.count > 0 ? "orphan-collection" : "ignored";
|
|
394
594
|
}
|
|
395
595
|
}
|
|
396
|
-
addRow({
|
|
596
|
+
addRow(buildInventoryRow(client, sourceLens, {
|
|
397
597
|
rowId: makeRowId(col.target, col.databaseName, col.collectionName),
|
|
398
598
|
target: col.target,
|
|
399
599
|
databaseName: col.databaseName,
|
|
400
600
|
objectName,
|
|
401
601
|
contentType,
|
|
402
602
|
collectionName: col.collectionName,
|
|
403
|
-
|
|
603
|
+
rowSource: "mongo-discovered",
|
|
404
604
|
mappingStatus,
|
|
405
605
|
counts: {
|
|
406
606
|
...(includeExactCounts ? { exact: col.count } : { estimated: col.count }),
|
|
407
607
|
countMode,
|
|
408
608
|
},
|
|
409
|
-
|
|
410
|
-
|
|
609
|
+
listable: false,
|
|
610
|
+
parse: parsed.ok ? parsed : parsed,
|
|
611
|
+
}));
|
|
411
612
|
}
|
|
412
613
|
}
|
|
413
614
|
else {
|
|
@@ -415,7 +616,7 @@ export async function buildMemorixUnifiedInventory(client, options = {}) {
|
|
|
415
616
|
const parsed = parseMemorixCollectionName(col.collectionName);
|
|
416
617
|
const slot = declaredByKey.get(key);
|
|
417
618
|
if (slot) {
|
|
418
|
-
addRow({
|
|
619
|
+
addRow(buildInventoryRow(client, sourceLens, {
|
|
419
620
|
rowId: makeRowId(col.target, col.databaseName, col.collectionName),
|
|
420
621
|
target: col.target,
|
|
421
622
|
databaseName: col.databaseName,
|
|
@@ -423,38 +624,36 @@ export async function buildMemorixUnifiedInventory(client, options = {}) {
|
|
|
423
624
|
contentType: slot.contentType,
|
|
424
625
|
collectionName: col.collectionName,
|
|
425
626
|
expectedCollectionName: slot.expectedCollectionName,
|
|
426
|
-
|
|
427
|
-
mappingStatus:
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
listDescriptorIds: listIdsByEntity.get(slot.objectName),
|
|
431
|
-
defaultListDescriptorId: slot.defaultListDescriptorId,
|
|
432
|
-
defaultItemDescriptorId: slot.defaultItemDescriptorId,
|
|
433
|
-
},
|
|
627
|
+
rowSource: "merged",
|
|
628
|
+
mappingStatus: "mapped",
|
|
629
|
+
slot,
|
|
630
|
+
listDescriptorIds: listIdsByEntity.get(slot.objectName),
|
|
434
631
|
counts: {
|
|
435
632
|
...(includeExactCounts ? { exact: col.count } : { estimated: col.count }),
|
|
436
633
|
countMode,
|
|
437
634
|
},
|
|
438
|
-
|
|
439
|
-
|
|
635
|
+
listable: slot.listable,
|
|
636
|
+
parse: parsed.ok ? parsed : undefined,
|
|
637
|
+
}));
|
|
440
638
|
continue;
|
|
441
639
|
}
|
|
442
640
|
if (!parsed.ok) {
|
|
443
|
-
addRow({
|
|
641
|
+
addRow(buildInventoryRow(client, sourceLens, {
|
|
444
642
|
rowId: makeRowId(col.target, col.databaseName, col.collectionName),
|
|
445
643
|
target: col.target,
|
|
446
644
|
databaseName: col.databaseName,
|
|
447
645
|
objectName: col.collectionName,
|
|
448
646
|
contentType: "",
|
|
449
647
|
collectionName: col.collectionName,
|
|
450
|
-
|
|
648
|
+
rowSource: "mongo-discovered",
|
|
451
649
|
mappingStatus: col.count > 0 ? "unparseable" : "ignored",
|
|
452
650
|
counts: {
|
|
453
651
|
...(includeExactCounts ? { exact: col.count } : { estimated: col.count }),
|
|
454
652
|
countMode,
|
|
455
653
|
},
|
|
456
|
-
|
|
457
|
-
|
|
654
|
+
listable: false,
|
|
655
|
+
parse: parsed,
|
|
656
|
+
}));
|
|
458
657
|
continue;
|
|
459
658
|
}
|
|
460
659
|
const hasObject = objectNames.has(parsed.objectName);
|
|
@@ -471,27 +670,28 @@ export async function buildMemorixUnifiedInventory(client, options = {}) {
|
|
|
471
670
|
mappingStatus = col.count > 0 ? "orphan-collection" : "ignored";
|
|
472
671
|
}
|
|
473
672
|
}
|
|
474
|
-
addRow({
|
|
673
|
+
addRow(buildInventoryRow(client, sourceLens, {
|
|
475
674
|
rowId: makeRowId(col.target, col.databaseName, col.collectionName),
|
|
476
675
|
target: col.target,
|
|
477
676
|
databaseName: col.databaseName,
|
|
478
677
|
objectName: parsed.objectName,
|
|
479
678
|
contentType: parsed.contentType,
|
|
480
679
|
collectionName: col.collectionName,
|
|
481
|
-
|
|
680
|
+
rowSource: "mongo-discovered",
|
|
482
681
|
mappingStatus,
|
|
483
682
|
counts: {
|
|
484
683
|
...(includeExactCounts ? { exact: col.count } : { estimated: col.count }),
|
|
485
684
|
countMode,
|
|
486
685
|
},
|
|
487
|
-
|
|
488
|
-
|
|
686
|
+
listable: false,
|
|
687
|
+
parse: parsed,
|
|
688
|
+
}));
|
|
489
689
|
}
|
|
490
690
|
for (const slot of declaredSlots) {
|
|
491
691
|
const key = rowKey(slot.databaseName, slot.collectionName);
|
|
492
692
|
if (seenKeys.has(key))
|
|
493
693
|
continue;
|
|
494
|
-
addRow({
|
|
694
|
+
addRow(buildInventoryRow(client, sourceLens, {
|
|
495
695
|
rowId: makeRowId(slot.target, slot.databaseName, slot.collectionName),
|
|
496
696
|
target: slot.target,
|
|
497
697
|
databaseName: slot.databaseName,
|
|
@@ -499,32 +699,62 @@ export async function buildMemorixUnifiedInventory(client, options = {}) {
|
|
|
499
699
|
contentType: slot.contentType,
|
|
500
700
|
collectionName: slot.collectionName,
|
|
501
701
|
expectedCollectionName: slot.expectedCollectionName,
|
|
502
|
-
|
|
702
|
+
rowSource: "catalox-declared",
|
|
503
703
|
mappingStatus: "missing-collection",
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
listDescriptorIds: listIdsByEntity.get(slot.objectName),
|
|
507
|
-
defaultListDescriptorId: slot.defaultListDescriptorId,
|
|
508
|
-
defaultItemDescriptorId: slot.defaultItemDescriptorId,
|
|
509
|
-
},
|
|
704
|
+
slot,
|
|
705
|
+
listDescriptorIds: listIdsByEntity.get(slot.objectName),
|
|
510
706
|
counts: { countMode: "none" },
|
|
511
|
-
|
|
512
|
-
});
|
|
707
|
+
listable: slot.listable,
|
|
708
|
+
}));
|
|
513
709
|
}
|
|
514
710
|
}
|
|
515
711
|
rows.sort((a, b) => {
|
|
516
|
-
const db = a.databaseName.localeCompare(b.databaseName);
|
|
712
|
+
const db = (a.databaseName ?? "").localeCompare(b.databaseName ?? "");
|
|
517
713
|
if (db !== 0)
|
|
518
714
|
return db;
|
|
519
|
-
return a.collectionName.localeCompare(b.collectionName);
|
|
715
|
+
return (a.collectionName ?? "").localeCompare(b.collectionName ?? "");
|
|
520
716
|
});
|
|
521
|
-
|
|
717
|
+
if (options.includeSamples) {
|
|
718
|
+
await attachInventorySamples(client, rows);
|
|
719
|
+
}
|
|
720
|
+
const issues = options.includeIssues === false ? [] : buildIssuesFromRows(rows);
|
|
522
721
|
const targetSummary = summarizeTargets(rows, targets, dbs);
|
|
523
|
-
const
|
|
722
|
+
const targetBindings = await resolveMemorixTargetBindings(client, {
|
|
723
|
+
targets,
|
|
724
|
+
includeCounts: includeExactCounts,
|
|
725
|
+
});
|
|
726
|
+
const declaredObjectTypes = new Set(declaredSlots.map((s) => s.objectName)).size;
|
|
727
|
+
const inventorySource = runtimeProjectionSourceRef("buildMemorixUnifiedInventory", {
|
|
728
|
+
appId: client.appId,
|
|
729
|
+
sourceLens,
|
|
730
|
+
});
|
|
731
|
+
const warnings = [];
|
|
732
|
+
const errors = [];
|
|
733
|
+
for (const target of ALL_MEMORIX_TARGETS) {
|
|
734
|
+
const binding = targetBindings[target];
|
|
735
|
+
for (const warning of binding.warnings) {
|
|
736
|
+
warnings.push({
|
|
737
|
+
...warning,
|
|
738
|
+
source: mongoPayloadSourceRef(target, binding.databaseName ?? binding.defaultDatabaseName, "_db", { appId: client.appId }),
|
|
739
|
+
});
|
|
740
|
+
}
|
|
741
|
+
for (const error of binding.errors) {
|
|
742
|
+
errors.push({
|
|
743
|
+
...error,
|
|
744
|
+
source: mongoPayloadSourceRef(target, binding.databaseName ?? binding.defaultDatabaseName, "_db", { appId: client.appId }),
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
}
|
|
524
748
|
return {
|
|
749
|
+
source: inventorySource,
|
|
525
750
|
sourceLens,
|
|
751
|
+
generatedAt: new Date().toISOString(),
|
|
752
|
+
appId: client.appId,
|
|
753
|
+
targetBindings,
|
|
526
754
|
summary: {
|
|
527
|
-
|
|
755
|
+
declaredObjectTypes,
|
|
756
|
+
physicalDatabases: countPhysicalDatabases(targetBindings),
|
|
757
|
+
configuredTargets: countConfiguredTargets(targetBindings),
|
|
528
758
|
mongoCollections: scanned.size,
|
|
529
759
|
declaredCollections: declaredSlots.length,
|
|
530
760
|
mappedCollections: rows.filter((r) => r.mappingStatus === "mapped").length,
|
|
@@ -532,13 +762,18 @@ export async function buildMemorixUnifiedInventory(client, options = {}) {
|
|
|
532
762
|
orphanCollections: rows.filter((r) => r.mappingStatus === "orphan-collection").length,
|
|
533
763
|
emptyDeclaredCollections: rows.filter((r) => r.mappingStatus === "mapped" &&
|
|
534
764
|
(r.counts.exact === 0 || r.counts.estimated === 0)).length,
|
|
765
|
+
collectionMismatches: rows.filter((r) => r.mappingStatus === "collection-mismatch").length,
|
|
766
|
+
unparseableCollections: rows.filter((r) => r.mappingStatus === "unparseable").length,
|
|
767
|
+
ignoredCollections: rows.filter((r) => r.mappingStatus === "ignored").length,
|
|
535
768
|
totalRecords: rows.reduce((sum, r) => sum + (r.counts.exact ?? r.counts.estimated ?? 0), 0),
|
|
536
769
|
issueCount: issues.length,
|
|
770
|
+
cataloxObjects: declaredObjectTypes,
|
|
537
771
|
},
|
|
538
|
-
targets: targetSummary,
|
|
539
772
|
rows,
|
|
540
773
|
issues,
|
|
541
|
-
|
|
774
|
+
warnings,
|
|
775
|
+
errors,
|
|
776
|
+
targets: targetSummary,
|
|
542
777
|
};
|
|
543
778
|
}
|
|
544
779
|
//# sourceMappingURL=unified-inventory.js.map
|