@x12i/memorix-retrieval 1.2.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +64 -24
  2. package/catalox-seeds/inputs/entity-descriptors/assets.json +2 -0
  3. package/catalox-seeds/inputs/entity-descriptors/variabilities-groups.json +2 -0
  4. package/catalox-seeds/inputs/entity-descriptors/vulnerabilities.json +2 -0
  5. package/catalox-seeds/inputs/list-descriptors/memorix-workspace-records-list.json +22 -0
  6. package/catalox-seeds/inputs/manifest.json +3 -1
  7. package/catalox-seeds/inputs/memorix-list-descriptors.catalog.json +22 -0
  8. package/catalox-seeds/inputs/memorix-list-descriptors.items.json +22 -0
  9. package/catalox-seeds/memorix-retrieval-descriptors.manifest.json +40 -0
  10. package/dist/client/create-client.js +2 -2
  11. package/dist/client/create-client.js.map +1 -1
  12. package/dist/client/create-stack-from-env.js +4 -6
  13. package/dist/client/create-stack-from-env.js.map +1 -1
  14. package/dist/client/types.d.ts +48 -0
  15. package/dist/client/types.d.ts.map +1 -1
  16. package/dist/descriptors/descriptor-types.d.ts +35 -10
  17. package/dist/descriptors/descriptor-types.d.ts.map +1 -1
  18. package/dist/descriptors/discover-entities.d.ts +3 -1
  19. package/dist/descriptors/discover-entities.d.ts.map +1 -1
  20. package/dist/descriptors/discover-entities.js +14 -43
  21. package/dist/descriptors/discover-entities.js.map +1 -1
  22. package/dist/descriptors/load-list-descriptor.d.ts +5 -1
  23. package/dist/descriptors/load-list-descriptor.d.ts.map +1 -1
  24. package/dist/descriptors/load-list-descriptor.js +23 -3
  25. package/dist/descriptors/load-list-descriptor.js.map +1 -1
  26. package/dist/descriptors/resolve-app-id.d.ts +4 -0
  27. package/dist/descriptors/resolve-app-id.d.ts.map +1 -0
  28. package/dist/descriptors/resolve-app-id.js +8 -0
  29. package/dist/descriptors/resolve-app-id.js.map +1 -0
  30. package/dist/descriptors/resolve-default-descriptors.d.ts +11 -0
  31. package/dist/descriptors/resolve-default-descriptors.d.ts.map +1 -0
  32. package/dist/descriptors/resolve-default-descriptors.js +22 -0
  33. package/dist/descriptors/resolve-default-descriptors.js.map +1 -0
  34. package/dist/descriptors/validate-descriptor.d.ts +6 -2
  35. package/dist/descriptors/validate-descriptor.d.ts.map +1 -1
  36. package/dist/descriptors/validate-descriptor.js +42 -5
  37. package/dist/descriptors/validate-descriptor.js.map +1 -1
  38. package/dist/explorer/entity-graph.d.ts +3 -1
  39. package/dist/explorer/entity-graph.d.ts.map +1 -1
  40. package/dist/explorer/entity-graph.js +4 -0
  41. package/dist/explorer/entity-graph.js.map +1 -1
  42. package/dist/index.d.ts +16 -6
  43. package/dist/index.d.ts.map +1 -1
  44. package/dist/index.js +13 -4
  45. package/dist/index.js.map +1 -1
  46. package/dist/retrieval/compose-row.d.ts +2 -2
  47. package/dist/retrieval/compose-row.d.ts.map +1 -1
  48. package/dist/retrieval/compose-row.js.map +1 -1
  49. package/dist/retrieval/fetch-for-entity.d.ts +18 -0
  50. package/dist/retrieval/fetch-for-entity.d.ts.map +1 -0
  51. package/dist/retrieval/fetch-for-entity.js +21 -0
  52. package/dist/retrieval/fetch-for-entity.js.map +1 -0
  53. package/dist/retrieval/fetch-list.d.ts.map +1 -1
  54. package/dist/retrieval/fetch-list.js +18 -3
  55. package/dist/retrieval/fetch-list.js.map +1 -1
  56. package/dist/retrieval/fetch-workspace-list.d.ts +4 -0
  57. package/dist/retrieval/fetch-workspace-list.d.ts.map +1 -0
  58. package/dist/retrieval/fetch-workspace-list.js +164 -0
  59. package/dist/retrieval/fetch-workspace-list.js.map +1 -0
  60. package/dist/retrieval/resolve-filters.d.ts +4 -2
  61. package/dist/retrieval/resolve-filters.d.ts.map +1 -1
  62. package/dist/retrieval/resolve-filters.js +24 -3
  63. package/dist/retrieval/resolve-filters.js.map +1 -1
  64. package/dist/retrieval/resolve-list-search.d.ts +6 -0
  65. package/dist/retrieval/resolve-list-search.d.ts.map +1 -0
  66. package/dist/retrieval/resolve-list-search.js +25 -0
  67. package/dist/retrieval/resolve-list-search.js.map +1 -0
  68. package/dist/retrieval/resolve-pagination-driver.d.ts +2 -2
  69. package/dist/retrieval/resolve-pagination-driver.d.ts.map +1 -1
  70. package/dist/retrieval/resolve-pagination-driver.js.map +1 -1
  71. package/dist/retrieval/resolve-sort.d.ts +2 -2
  72. package/dist/retrieval/resolve-sort.d.ts.map +1 -1
  73. package/dist/retrieval/resolve-sort.js.map +1 -1
  74. package/dist/seeds/build-manifest.d.ts +8 -0
  75. package/dist/seeds/build-manifest.d.ts.map +1 -0
  76. package/dist/seeds/build-manifest.js +83 -0
  77. package/dist/seeds/build-manifest.js.map +1 -0
  78. package/dist/seeds/check-needs-seed.d.ts +10 -0
  79. package/dist/seeds/check-needs-seed.d.ts.map +1 -0
  80. package/dist/seeds/check-needs-seed.js +66 -0
  81. package/dist/seeds/check-needs-seed.js.map +1 -0
  82. package/dist/seeds/default-seed-spec.d.ts +12 -0
  83. package/dist/seeds/default-seed-spec.d.ts.map +1 -0
  84. package/dist/seeds/default-seed-spec.js +90 -0
  85. package/dist/seeds/default-seed-spec.js.map +1 -0
  86. package/dist/seeds/merge-for-apply.d.ts +28 -0
  87. package/dist/seeds/merge-for-apply.d.ts.map +1 -0
  88. package/dist/seeds/merge-for-apply.js +31 -0
  89. package/dist/seeds/merge-for-apply.js.map +1 -0
  90. package/dist/seeds/paths.d.ts +10 -0
  91. package/dist/seeds/paths.d.ts.map +1 -0
  92. package/dist/seeds/paths.js +40 -0
  93. package/dist/seeds/paths.js.map +1 -0
  94. package/dist/seeds/seed-types.d.ts +57 -0
  95. package/dist/seeds/seed-types.d.ts.map +1 -0
  96. package/dist/seeds/seed-types.js +2 -0
  97. package/dist/seeds/seed-types.js.map +1 -0
  98. package/dist/tests/descriptor-validation.test.js +15 -0
  99. package/dist/tests/descriptor-validation.test.js.map +1 -1
  100. package/dist/tests/fetch-workspace-list.test.d.ts +2 -0
  101. package/dist/tests/fetch-workspace-list.test.d.ts.map +1 -0
  102. package/dist/tests/fetch-workspace-list.test.js +187 -0
  103. package/dist/tests/fetch-workspace-list.test.js.map +1 -0
  104. package/dist/tests/fixtures.d.ts +5 -2
  105. package/dist/tests/fixtures.d.ts.map +1 -1
  106. package/dist/tests/fixtures.js +79 -0
  107. package/dist/tests/fixtures.js.map +1 -1
  108. package/dist/tests/relations.test.js +3 -0
  109. package/dist/tests/relations.test.js.map +1 -1
  110. package/dist/tests/resolve-app-id.test.d.ts +2 -0
  111. package/dist/tests/resolve-app-id.test.d.ts.map +1 -0
  112. package/dist/tests/resolve-app-id.test.js +19 -0
  113. package/dist/tests/resolve-app-id.test.js.map +1 -0
  114. package/dist/tests/resolve-filters.test.d.ts +2 -0
  115. package/dist/tests/resolve-filters.test.d.ts.map +1 -0
  116. package/dist/tests/resolve-filters.test.js +47 -0
  117. package/dist/tests/resolve-filters.test.js.map +1 -0
  118. package/dist/tests/seed-check.test.d.ts +2 -0
  119. package/dist/tests/seed-check.test.d.ts.map +1 -0
  120. package/dist/tests/seed-check.test.js +47 -0
  121. package/dist/tests/seed-check.test.js.map +1 -0
  122. package/dist/tests/seed-manifest.test.d.ts +2 -0
  123. package/dist/tests/seed-manifest.test.d.ts.map +1 -0
  124. package/dist/tests/seed-manifest.test.js +80 -0
  125. package/dist/tests/seed-manifest.test.js.map +1 -0
  126. package/docs/DATA-TIER-CONTRACT.md +141 -0
  127. package/docs/EXPLORER-HOST-APIS.md +19 -7
  128. package/docs/MEMORIX-CATALOX-CONTRACTS.md +587 -0
  129. package/package.json +5 -3
@@ -0,0 +1,587 @@
1
+ # Memorix ↔ Catalox contracts
2
+
3
+ **Canonical sync document** for all Memorix data packages and Memorix-consuming UIs (Explorer, completion pipelines, ingestion, graph tools).
4
+
5
+
6
+ | Document | Scope |
7
+ |----------|--------|
8
+ | **This file** | Catalox catalogs, descriptor JSON formats, expected Mongo shape, cross-component behavior |
9
+ | [MEMORIX-DATABASE-CONVENTIONS.md](./MEMORIX-DATABASE-CONVENTIONS.md) | Database names, collection naming, env vars, record envelope |
10
+ | [DATA-TIER-CONTRACT.md](./DATA-TIER-CONTRACT.md) | What host apps may call (retrieval APIs only) |
11
+
12
+ **Reference implementation:** `@x12i/memorix-retrieval` — descriptor types in `src/descriptors/descriptor-types.ts`, seeds in `catalox-seeds/inputs/`, validation in `src/descriptors/validate-descriptor.ts`.
13
+
14
+ If this document and a package disagree, **update both together**.
15
+
16
+ ---
17
+
18
+ ## 1. Split of responsibilities
19
+
20
+ ```text
21
+ ┌─────────────────────────────────────────────────────────────────┐
22
+ │ Catalox (metadata) │
23
+ │ • What entities exist │
24
+ │ • How to list / detail / relate them │
25
+ │ • Property paths, filters, sorts, relations │
26
+ └───────────────────────────────┬─────────────────────────────────┘
27
+ │ listCatalogItems / getCatalogItem
28
+
29
+ ┌─────────────────────────────────────────────────────────────────┐
30
+ │ @x12i/memorix-retrieval (data tier) │
31
+ │ • Load descriptors from Catalox │
32
+ │ • Resolve Mongo DB + collection from descriptor │
33
+ │ • Read, compose, paginate, multi-match, relations, content │
34
+ └───────────────────────────────┬─────────────────────────────────┘
35
+ │ fetchMemorixList / fetchMemorixItem / graph
36
+
37
+ ┌─────────────────────────────────────────────────────────────────┐
38
+ │ Host apps (Explorer, APIs, tools) │
39
+ │ • Never bypass descriptors for entity I/O │
40
+ │ • Never invent entity lists from env or collection names │
41
+ └─────────────────────────────────────────────────────────────────┘
42
+
43
+ ┌─────────────────────────────────────────────────────────────────┐
44
+ │ MongoDB (payload) │
45
+ │ • memorix-entities / memorix-events │
46
+ │ • Domain documents under `data` │
47
+ └─────────────────────────────────────────────────────────────────┘
48
+ ```
49
+
50
+ | Layer | Owns | Must not own |
51
+ |-------|------|--------------|
52
+ | **Catalox** | Descriptor catalogs, item ids, scope tags, graph semantics | Raw Mongo queries, business payload writes |
53
+ | **memorix-retrieval** | Descriptor-driven reads, composition, validation | Ad hoc catalog types outside the three Memorix catalogs |
54
+ | **Completion / ingestion** | Writing `data` on Memorix records per [MEMORIX-DATABASE-CONVENTIONS](./MEMORIX-DATABASE-CONVENTIONS.md) | Defining list columns or UI field layouts (that's descriptors) |
55
+ | **Explorer / UIs** | Presentation, routing, design model | Direct Mongo, xmemory-store, env-based entity discovery |
56
+
57
+ ---
58
+
59
+ ## 2. Catalox app and catalogs
60
+
61
+ ### 2.1 App id
62
+
63
+ | Constant | Value |
64
+ |----------|-------|
65
+ | Default `appId` | `memorix` |
66
+ | Env override | `CATALOX_APP_ID` (fallback: `MEMORIX_APP_ID`) |
67
+
68
+ All Memorix descriptor catalogs live under this app. Peers must use the same `appId` unless a deployment explicitly namespaces apps. `createMemorixRetrieval`, `createMemorixRetrievalFromEnv`, and `createMemorixRetrievalStackFromEnv` resolve the app id via `resolveMemorixAppId()`.
69
+
70
+ ### 2.2 Catalog ids (only these three)
71
+
72
+ | Catalog id | Item id field | Title field | Purpose |
73
+ |------------|---------------|-------------|---------|
74
+ | `memorix-entity-descriptors` | `id` | `entityName` | Entity schema: properties, content types, relations, default list/item |
75
+ | `memorix-list-descriptors` | `id` | `title` | Tabular list views |
76
+ | `memorix-item-descriptors` | `id` | `title` | Record detail layout |
77
+
78
+ Catalogs use **`sourceMode: "native"`** and **`catalogType: "memorix"`** in seed manifests. Do not introduce parallel catalog names (e.g. `memorix_entity_content_types`) for new work.
79
+
80
+ ### 2.3 Item shape in Catalox
81
+
82
+ Each catalog item is stored as:
83
+
84
+ ```json
85
+ {
86
+ "id": "<descriptor-id>",
87
+ "...": "<descriptor fields — see §3>",
88
+ "scope": {
89
+ "domains": ["network", "vulnerabilities"],
90
+ "agents": ["neo"]
91
+ }
92
+ }
93
+ ```
94
+
95
+ - **`id`** must match the descriptor's own `id` field.
96
+ - **`scope`** is applied at seed time from `catalox-seeds/inputs/scope.json`. All items in a seed bundle share the same scope unless a future manifest explicitly overrides per item.
97
+ - Descriptor payload lives in the item **`data`** body when read via Catalox APIs.
98
+ - **Seed apply note:** the Catalox seed CLI upserts `row.data` only, so `scripts/apply-seed-manifest.mjs` merges `scope` into the upsert `data` object. Hosts may see `scope` inside `data` until Catalox stores top-level scope natively; retrieval ignores unknown fields when validating descriptors.
99
+
100
+ ### 2.4 Expected Catalox client behavior
101
+
102
+ Peers that integrate with Catalox must support:
103
+
104
+ | Operation | Used for |
105
+ |-----------|----------|
106
+ | `listCatalogItems(context, catalogId, { limit, offset })` | Entity discovery, paginated catalog reads |
107
+ | `getCatalogItem(context, catalogId, itemId)` | Load list/item/entity descriptor by id |
108
+
109
+ **Discovery rule:** entity existence comes **only** from listing `memorix-entity-descriptors`. There is no env fallback (`MEMORIX_ENTITY_NAMES` and similar bypasses are forbidden in retrieval and hosts).
110
+
111
+ If the catalog is empty or Catalox is unreachable, discovery returns `source: "none"` with a hint to run the seed apply script.
112
+
113
+ ---
114
+
115
+ ## 3. Descriptor formats
116
+
117
+ Types below mirror `MemorixEntityDescriptor`, `MemorixListDescriptor`, and `MemorixItemDescriptor` in `@x12i/memorix-retrieval`.
118
+
119
+ ### 3.1 Entity descriptor (`memorix-entity-descriptors`)
120
+
121
+ **Required top-level fields:**
122
+
123
+ | Field | Type | Rules |
124
+ |-------|------|-------|
125
+ | `id` | string | Stable catalog item id; usually equals `entityName` |
126
+ | `entityName` | string | Kebab-case domain id (`vulnerabilities`, `variabilities-groups`, `assets`) |
127
+ | `defaultListDescriptorId` | string | **Option A** — id in `memorix-list-descriptors` |
128
+ | `defaultItemDescriptorId` | string | **Option A** — id in `memorix-item-descriptors` |
129
+ | `collectionPrefix` | string | Used with content-type `postfix` when `collection` is omitted |
130
+ | `identity` | object | See below |
131
+ | `defaults` | object | Canonical content type, `dataRoot`, effective-date paths |
132
+ | `contentTypes` | object | Named content slices (see §3.4) |
133
+ | `properties` | object | Field dictionary for list/item composition |
134
+
135
+ **Optional:**
136
+
137
+ | Field | Type | Default |
138
+ |-------|------|---------|
139
+ | `target` | `"entity"` \| `"event"` | `"entity"` — selects `memorix-entities` vs `memorix-events` |
140
+ | `contentDefaults` | object | Default content-object storage/format |
141
+ | `relations` | object | Cross-entity joins declared on this entity |
142
+
143
+ **Identity block:**
144
+
145
+ ```json
146
+ {
147
+ "allowedIdFields": ["entityId", "eventId"],
148
+ "requiredExactlyOne": true,
149
+ "defaultIdField": "entityId"
150
+ }
151
+ ```
152
+
153
+ **Defaults block:**
154
+
155
+ ```json
156
+ {
157
+ "canonicalContentType": "snapshots",
158
+ "dataRoot": "data",
159
+ "effectiveDatePath": "capturedAt",
160
+ "fallbackEffectiveDatePaths": ["snapshot.capturedAt", "data.enrichment.enrichedAt"]
161
+ }
162
+ ```
163
+
164
+ **Property descriptor** (each key under `properties`):
165
+
166
+ ```json
167
+ {
168
+ "label": "Asset IP",
169
+ "source": { "contentType": "snapshots", "path": "data.assetIp" },
170
+ "humanReadable": true,
171
+ "sortable": true,
172
+ "filterable": true,
173
+ "list": true,
174
+ "item": true,
175
+ "valueType": "string"
176
+ }
177
+ ```
178
+
179
+ | `valueType` | Notes |
180
+ |-------------|-------|
181
+ | `string`, `number`, `boolean`, `date`, `datetime`, `enum`, `object`, `array` | Scalar / structural |
182
+ | `content` | Value is a `memorix-content-object` (see §3.7) |
183
+
184
+ List views may only expose properties with **`humanReadable: true`**.
185
+
186
+ **Relation descriptor** (under `relations`):
187
+
188
+ ```json
189
+ {
190
+ "targetEntity": "assets",
191
+ "type": "manyToOne",
192
+ "source": { "contentType": "snapshots", "path": "data.assetIp" },
193
+ "target": { "contentType": "snapshots", "path": "data.ip_address" },
194
+ "defaultMode": "extendFields",
195
+ "defaultArrayProperty": "vulnerabilities",
196
+ "targetFields": ["ipAddress", "hostName"]
197
+ }
198
+ ```
199
+
200
+ | `type` | Meaning |
201
+ |--------|---------|
202
+ | `oneToOne`, `oneToMany`, `manyToOne`, `manyToMany` | Graph semantics for Explorer and `includeRelations` |
203
+
204
+ Join paths are **Mongo document paths** on the respective content types, not Catalox ids.
205
+
206
+ ### 3.2 List descriptor (`memorix-list-descriptors`)
207
+
208
+ List descriptors are a **discriminated union** by `kind`:
209
+
210
+ | Kind | When | Key fields |
211
+ |------|------|------------|
212
+ | *(omit or `"entity"`)* | Per-entity tabular list | `entity`, `leadingContentType`, `fields[]` |
213
+ | `"workspace"` | Cross-entity workspace merge | `fields[]` (column hints), optional `entities`, `listDescriptorByEntity` |
214
+
215
+ #### 3.2.1 Entity list (`kind` omitted or `"entity"`)
216
+
217
+ **Required:**
218
+
219
+ | Field | Rules |
220
+ |-------|-------|
221
+ | `id` | Unique list id (e.g. `vulnerabilities-main-list`) |
222
+ | `entity` | `entityName` referencing an entity descriptor |
223
+ | `leadingContentType` | Content type key used to drive pagination/filter/sort |
224
+ | `pagination` | `{ "enabled": true, "defaultLimit": 50, "maxLimit": 200 }` |
225
+ | `fields` | Non-empty array of **property keys** on the entity descriptor |
226
+
227
+ **Common optional fields:**
228
+
229
+ | Field | Purpose |
230
+ |-------|---------|
231
+ | `title` | Human label |
232
+ | `filters` | Default filters (always applied) plus allowlist for request filters; request overrides same property |
233
+ | `allowedSorts`, `defaultSort` | Sort whitelist and default |
234
+ | `extensions` | Pull extra content types by identity (see §3.5) |
235
+ | `includeRelations` | Embed relation data in list rows |
236
+ | `allowSortDrivenLeadingOverride` | Allow sort to switch pagination driver content type |
237
+
238
+ **Example filter:**
239
+
240
+ ```json
241
+ { "property": "severityLevel", "operator": "gte", "value": 4 }
242
+ ```
243
+
244
+ **Filter operators:** `eq`, `ne`, `in`, `nin`, `gt`, `gte`, `lt`, `lte`, `exists`, `regex`.
245
+
246
+ Each entity descriptor should define at least one list whose `id` equals `defaultListDescriptorId`. Additional lists (e.g. `critical-vulnerabilities-list`) are optional alternate views.
247
+
248
+ #### 3.2.2 Workspace list (`kind: "workspace"`)
249
+
250
+ Cross-entity workspace lists merge per-entity list composition — used by Explorer **Records** when no entity filter is selected.
251
+
252
+ **Required:**
253
+
254
+ | Field | Rules |
255
+ |-------|-------|
256
+ | `id` | Unique list id (shipped: `memorix-workspace-records-list`) |
257
+ | `kind` | `"workspace"` |
258
+ | `pagination` | Same shape as entity lists |
259
+ | `fields` | Non-empty column hints for hosts (may include synthetic ids like `recordTitle`; rows include per-entity composed fields) |
260
+
261
+ **Optional:**
262
+
263
+ | Field | Purpose |
264
+ |-------|---------|
265
+ | `title` | Human label |
266
+ | `entities` | Subset of discovered entity names; omit = all from `discoverMemorixEntities` |
267
+ | `listDescriptorByEntity` | Override list id per entity; omit = `defaultListDescriptorId` |
268
+ | `defaultSort` | Global sort after merge (`property`, `direction`) |
269
+ | `scopeFilter` | `"inherit-seed-scope"` — Catalox scope hint (not Mongo namespace) |
270
+
271
+ ### 3.3 Item descriptor (`memorix-item-descriptors`)
272
+
273
+ **Required:**
274
+
275
+ | Field | Rules |
276
+ |-------|-------|
277
+ | `id` | Unique item id (e.g. `vulnerability-detail-item`) |
278
+ | `entity` | `entityName` |
279
+ | `identity.idField` | `"entityId"` or `"eventId"` — must match request |
280
+ | `contentTypes` | Array of `{ contentType, required, multiMatch? }` |
281
+ | `sections` | Array of `{ id, title, fields[] }` — `fields` are property keys |
282
+
283
+ **Optional:**
284
+
285
+ | Field | Purpose |
286
+ |-------|---------|
287
+ | `includeRelations` | `{ relation, mode?, fields? }` — relation key from entity descriptor |
288
+ | `content` | `{ allowed, defaultIncludeFullContent?, allowedFields?, maxBytes? }` |
289
+
290
+ Each entity descriptor should define at least one item whose `id` equals `defaultItemDescriptorId`.
291
+
292
+ ### 3.4 Content type descriptor
293
+
294
+ Under `entity.contentTypes.<name>`:
295
+
296
+ ```json
297
+ {
298
+ "postfix": "snapshots",
299
+ "collection": "vulnerabilities-snapshots",
300
+ "dataRoot": "data",
301
+ "isCanonical": true,
302
+ "effectiveDatePath": "capturedAt",
303
+ "fallbackEffectiveDatePaths": ["snapshot.capturedAt"]
304
+ }
305
+ ```
306
+
307
+ **Collection resolution order** (retrieval):
308
+
309
+ 1. Explicit `collection` on the content type
310
+ 2. Env `MEMORIX_ENTITIES_COLLECTION_<ENTITY>` or `MEMORIX_EVENTS_COLLECTION_<ENTITY>`
311
+ 3. Heuristic: `{collectionPrefix}-{postfix}` (e.g. `assets-snapshots`)
312
+
313
+ **Database resolution:** entity descriptor `target` → `memorix-entities` or `memorix-events` (see [MEMORIX-DATABASE-CONVENTIONS](./MEMORIX-DATABASE-CONVENTIONS.md)).
314
+
315
+ Exactly one content type per entity should set **`isCanonical: true`**; it aligns with `defaults.canonicalContentType`.
316
+
317
+ ### 3.5 Content extensions (list)
318
+
319
+ ```json
320
+ {
321
+ "contentType": "enrichment",
322
+ "mode": "extendFields",
323
+ "join": { "by": "entityId" },
324
+ "multiMatch": { "strategy": "last", "effectiveDatePath": "capturedAt" },
325
+ "fields": { "enrichedAt": "data.enrichment.enrichedAt" }
326
+ }
327
+ ```
328
+
329
+ | `mode` | Behavior |
330
+ |--------|----------|
331
+ | `extendFields` | Merge selected paths onto the row |
332
+ | `array` | Attach array under `arrayProperty` |
333
+
334
+ ### 3.6 Multi-match
335
+
336
+ When multiple Mongo documents share the same identity, retrieval picks records using:
337
+
338
+ ```json
339
+ {
340
+ "strategy": "last" | "first" | "all" | "error" | "ignore",
341
+ "effectiveDatePath": "capturedAt",
342
+ "fallbackEffectiveDatePaths": ["snapshot.capturedAt"]
343
+ }
344
+ ```
345
+
346
+ Default for item fetches: **`last`** by effective date. Strategies are declared on item `contentTypes`, content-type defaults, or extension blocks.
347
+
348
+ ### 3.7 Content objects
349
+
350
+ Properties with `valueType: "content"` store a **Memorix content object** in Mongo:
351
+
352
+ ```json
353
+ {
354
+ "contentKey": "stories/asset-10-150-68-31.md",
355
+ "metadata": { "title": "Impact story" },
356
+ "preview": "First 200 chars…"
357
+ }
358
+ ```
359
+
360
+ Storage is configured on the entity (`contentDefaults`) or property (`content.storage`), never with inline credentials:
361
+
362
+ ```json
363
+ {
364
+ "provider": "gcs" | "s3",
365
+ "bucket": "memorix-content",
366
+ "prefix": "stories/",
367
+ "maxBytes": 65536,
368
+ "encoding": "utf8",
369
+ "credentialsRef": "env:MEMORIX_GCS_CREDENTIALS"
370
+ }
371
+ ```
372
+
373
+ Hosts provide **`contentReaders`** at retrieval bootstrap; list views return preview/metadata only (`listBehavior.fetchContent: false`).
374
+
375
+ ---
376
+
377
+ ## 4. What we expect to find in Mongo
378
+
379
+ ### 4.1 Record envelope
380
+
381
+ Documents in Memorix target databases follow [MEMORIX-DATABASE-CONVENTIONS](./MEMORIX-DATABASE-CONVENTIONS.md):
382
+
383
+ ```json
384
+ {
385
+ "_id": "<ObjectId>",
386
+ "recordId": "<optional>",
387
+ "entityId": "<for entity-targeted records>",
388
+ "eventId": "<for event-targeted records>",
389
+ "capturedAt": "<ISO-8601>",
390
+ "data": { "... domain fields referenced by descriptor paths ..." }
391
+ }
392
+ ```
393
+
394
+ **Peer rules:**
395
+
396
+ - Domain payload lives under **`data`** (or content-type-specific `dataRoot`, almost always `data`).
397
+ - Descriptor `source.path` values are **dot paths from the document root** (e.g. `data.assetIp`, not `assetIp` alone).
398
+ - Identity for list pagination and item fetch: **`entityId`** or **`eventId`** as declared on the item descriptor.
399
+ - Legacy `snapshotId` may be read as `recordId` fallback; do not write it in new code.
400
+
401
+ ### 4.2 Shipped entity ↔ database mapping
402
+
403
+ | Entity (`entityName`) | `target` | Typical collection(s) |
404
+ |-----------------------|----------|------------------------|
405
+ | `assets` | `entity` | `assets-snapshots` in `memorix-entities` |
406
+ | `variabilities-groups` | `entity` | `variabilities-groups-snapshots` |
407
+ | `vulnerabilities` | `event` | `vulnerabilities-snapshots` in `memorix-events` |
408
+
409
+ Ingestion and completion must populate paths referenced in entity descriptors. If a property path is missing, retrieval returns null/omits the field and may emit **`EXTENSION_MISSING`** issues for required content types.
410
+
411
+ ### 4.3 Relations in data
412
+
413
+ Relations are **not** stored as Catalox edges. They are resolved at read time by matching:
414
+
415
+ - `relation.source.path` on the source record
416
+ - `relation.target.path` on target entity records
417
+
418
+ Both sides must use the same **`contentType`** keys declared on the relation. Target field projection uses `targetFields` / `includeRelations.fields`.
419
+
420
+ ---
421
+
422
+ ## 5. Runtime behavior (retrieval)
423
+
424
+ All Memorix-consuming components should rely on these behaviors rather than reimplementing them. Public exports are listed in [DATA-TIER-CONTRACT.md](./DATA-TIER-CONTRACT.md).
425
+
426
+ **Host bootstrap:** prefer `createMemorixRetrievalStackFromEnv()` for Catalox + Mongo wiring; see [EXPLORER-HOST-APIS.md](./EXPLORER-HOST-APIS.md) for graph/raw-read helpers.
427
+
428
+ ### 5.1 Discovery
429
+
430
+ ```
431
+ discoverMemorixEntities(client)
432
+ → list all items in memorix-entity-descriptors
433
+ → validate each as MemorixEntityDescriptor
434
+ → return summaries: id, entityName, label, defaultListDescriptorId, defaultItemDescriptorId, target
435
+ ```
436
+
437
+ ### 5.2 Entity-default list and item
438
+
439
+ ```
440
+ fetchMemorixListForEntity(client, { entityName, ... })
441
+ → load entity descriptor
442
+ → listId = entity.defaultListDescriptorId
443
+ → fetchMemorixList(client, { listId, ... })
444
+
445
+ fetchMemorixItemForEntity(client, { entityName, entityId | eventId, ... })
446
+ → load entity descriptor
447
+ → itemDescriptorId = entity.defaultItemDescriptorId
448
+ → fetchMemorixItem(client, { itemDescriptorId, ... })
449
+ ```
450
+
451
+ ### 5.3 List composition
452
+
453
+ 1. Load list + entity descriptors from Catalox.
454
+ 2. Validate every `fields[]` entry exists and is `humanReadable`.
455
+ 3. Resolve pagination driver = `leadingContentType` (or sort override if allowed).
456
+ 4. Query driver collection with list default filters merged with request filters, plus optional `searchText` across human-readable fields.
457
+ 5. Optionally batch-fetch extensions and relations.
458
+ 6. Return rows as flat property maps + pagination + issues.
459
+
460
+ ### 5.4 Workspace list composition
461
+
462
+ ```
463
+ fetchMemorixWorkspaceList(client, { workspaceListId?, entityName?, page?, searchText?, sort?, includeTotal? })
464
+ → load workspace list descriptor (default: memorix-workspace-records-list)
465
+ → if entityName: delegate to fetchMemorixListForEntity, stamp rows with entityName
466
+ → discoverMemorixEntities → filter by workspace.entities?
467
+ → for each entity: resolve list id → fetchMemorixList (same field composition as §5.3)
468
+ → stamp entityName (+ identity) on each row
469
+ → merge, global sort, paginate in memory (v1)
470
+ → emit issues[] for per-entity failures without failing the whole workspace
471
+ ```
472
+
473
+ When `discovery.source` is `"none"`, returns empty rows with a hint — no env entity fallback.
474
+
475
+ ### 5.5 Item composition
476
+
477
+ 1. Load item + entity descriptors.
478
+ 2. Fetch all documents per `contentTypes[]` matching identity.
479
+ 3. Apply `multiMatch` per content type.
480
+ 4. Build `sections[]` from property definitions.
481
+ 5. Resolve `includeRelations` and optional full content fetches.
482
+ 6. Return `{ identity, sections, relations?, issues? }`.
483
+
484
+ ### 5.6 Entity graph (Explorer)
485
+
486
+ `buildMemorixEntityGraph` uses discovery + entity descriptors only:
487
+
488
+ - Per entity: label, counts, content-type labels, **`relations` → connections**, default list/item ids.
489
+ - `discovery.source` is `"catalox"` or `"none"` — never `"env"`.
490
+
491
+ ---
492
+
493
+ ## 6. Seeding and keeping peers in sync
494
+
495
+ ### 6.1 Source of truth
496
+
497
+ | Artifact | Location |
498
+ |----------|----------|
499
+ | Editable seed inputs | `@x12i/memorix-retrieval` → `catalox-seeds/inputs/` |
500
+ | Built manifest | `catalox-seeds/memorix-retrieval-descriptors.manifest.json` |
501
+ | TypeScript types | `src/descriptors/descriptor-types.ts` |
502
+
503
+ Workflow for descriptor changes:
504
+
505
+ ```bash
506
+ # In memorix-retrieval
507
+ npm run catalox:seed:build
508
+ npm run catalox:seed:memorix-retrieval:validate
509
+ npm run catalox:seed:memorix-retrieval:apply # force apply
510
+ npm run catalox:seed:ensure # apply only if items missing
511
+ ```
512
+
513
+ Seed helpers (`buildMemorixRetrievalSeedManifest`, `checkMemorixRetrievalSeedNeeds`, …) are exported from the package; `scripts/` wrap them for CLI use. See [DATA-TIER-CONTRACT.md](./DATA-TIER-CONTRACT.md#catalox-seed-helpers-library).
514
+
515
+ Then bump `@x12i/memorix-retrieval` and reinstall in consuming apps.
516
+
517
+ ### 6.2 Adding a new entity (checklist)
518
+
519
+ 1. Add `catalox-seeds/inputs/entity-descriptors/<name>.json` with **both default list and item ids**.
520
+ 2. Add at least one list under `list-descriptors/` and one item under `item-descriptors/`.
521
+ 3. Register ids in `catalox-seeds/inputs/manifest.json`.
522
+ 4. Rebuild and apply seed manifest.
523
+ 5. Ensure Mongo collections exist and documents match declared paths.
524
+ 6. Extend `src/tests/fixtures.ts` and validation tests in retrieval.
525
+ 7. Publish retrieval; update Explorer (or other hosts) dependency.
526
+
527
+ ### 6.3 Versioning
528
+
529
+ - **Descriptor schema changes** → minor/major bump of `@x12i/memorix-retrieval` + coordinated seed apply.
530
+ - **Seed-only changes** (fields, filters, new list views) → seed apply + retrieval patch as needed; hosts pick up via graph/list/item APIs automatically.
531
+ - Do not fork catalog ids per app; use **`scope`** for domain/agent filtering instead.
532
+
533
+ ---
534
+
535
+ ## 7. Component matrix
536
+
537
+ | Component | Reads Catalox | Writes Catalox | Reads Mongo | Writes Mongo |
538
+ |-----------|---------------|----------------|-------------|--------------|
539
+ | `@x12i/memorix-retrieval` | Yes (3 catalogs) | No (seeds via CLI) | Yes (descriptor-driven) | No |
540
+ | `@x12i/memorix-completion` | No* | No | Yes (source + targets) | Yes (`data` on targets) |
541
+ | `@x12i/memorix-explorer` | Via retrieval | No | No (via retrieval only) | No |
542
+ | Ingestion pipelines | No* | No | Yes | Yes |
543
+
544
+ \*Completion and ingestion should **align** with descriptor paths and collection names documented here; they do not need Catalox at runtime unless they also ship seed tooling.
545
+
546
+ ---
547
+
548
+ ## 8. Forbidden patterns
549
+
550
+ These break cross-component sync and must not appear in new code:
551
+
552
+ | Pattern | Why |
553
+ |---------|-----|
554
+ | Host Mongo list/get/count by collection name | Bypasses list/item descriptors |
555
+ | `MEMORIX_ENTITY_NAMES` or env entity lists | Bypasses Catalox discovery |
556
+ | `@x12i/xmemory-store` / scoped tier in Explorer | Legacy; replaced by retrieval + descriptors |
557
+ | Parallel catalog types for the same semantics | Fragments truth |
558
+ | Credentials inside descriptor JSON | Use `credentialsRef` + host-injected readers |
559
+ | List fields with `humanReadable: false` | Validation error at fetch time |
560
+ | Entity descriptor missing `defaultListDescriptorId` / `defaultItemDescriptorId` | Breaks Option A wiring |
561
+
562
+ ---
563
+
564
+ ## 9. Quick reference — shipped ids
565
+
566
+ From `catalox-seeds/inputs/manifest.json`:
567
+
568
+ **Entities:** `assets`, `vulnerabilities`, `variabilities-groups`
569
+
570
+ **Lists:** `assets-main-list`, `vulnerabilities-main-list`, `critical-vulnerabilities-list`, `variabilities-groups-main-list`, `memorix-workspace-records-list` (workspace)
571
+
572
+ **Items:** `asset-detail-item`, `vulnerability-detail-item`, `variabilities-group-detail-item`
573
+
574
+ **Scope (all seeded items):** `domains: [network, vulnerabilities]`, `agents: [neo]`
575
+
576
+ ---
577
+
578
+ ## 10. Related packages
579
+
580
+ | Package | Role |
581
+ |---------|------|
582
+ | `@x12i/catalox` | Catalog storage and seed CLI |
583
+ | `@x12i/memorix-retrieval` | Descriptor-driven data tier |
584
+ | `@x12i/memorix-completion` | Source → Memorix enrichment (payload paths must match descriptors) |
585
+ | `@x12i/memorix-explorer` | UI host — consumes retrieval APIs only |
586
+
587
+ When adding a capability all hosts need (e.g. cross-entity workspace list), **extend retrieval and descriptors first**, then update hosts — never shim in a single app.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@x12i/memorix-retrieval",
3
- "version": "1.2.0",
3
+ "version": "1.5.0",
4
4
  "description": "Descriptor-driven Memorix retrieval layer for lists, items, and content objects",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -19,9 +19,10 @@
19
19
  "scripts": {
20
20
  "build": "tsc",
21
21
  "test": "vitest run",
22
- "catalox:seed:build": "node scripts/build-seed-manifest.mjs",
22
+ "catalox:seed:build": "npm run build && node scripts/build-seed-manifest.mjs",
23
23
  "catalox:seed:memorix-retrieval:validate": "npm run catalox:seed:build && catalox seed validate --file catalox-seeds/memorix-retrieval-descriptors.manifest.json",
24
- "catalox:seed:memorix-retrieval:apply": "npm run catalox:seed:build && node scripts/apply-seed-manifest.mjs",
24
+ "catalox:seed:memorix-retrieval:apply": "npm run build && node scripts/apply-seed-manifest.mjs",
25
+ "catalox:seed:ensure": "npm run build && node scripts/ensure-catalox-seed.mjs",
25
26
  "mongo:check-collections": "node scripts/check-mongo-collections.mjs",
26
27
  "smoke:retrieval": "npm run build && node scripts/smoke-retrieval.mjs"
27
28
  },
@@ -31,6 +32,7 @@
31
32
  "dependencies": {
32
33
  "@x12i/catalox": "^4.1.2",
33
34
  "@x12i/helpers": "^1.7.0",
35
+ "@x12i/memorix-retrieval": "^1.4.0",
34
36
  "mongodb": "^6.21.0"
35
37
  },
36
38
  "peerDependencies": {