@x12i/memorix-retrieval 1.2.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -24
- package/catalox-seeds/inputs/entity-descriptors/assets.json +2 -0
- package/catalox-seeds/inputs/entity-descriptors/variabilities-groups.json +2 -0
- package/catalox-seeds/inputs/entity-descriptors/vulnerabilities.json +2 -0
- package/catalox-seeds/inputs/list-descriptors/memorix-workspace-records-list.json +22 -0
- package/catalox-seeds/inputs/manifest.json +3 -1
- package/catalox-seeds/inputs/memorix-list-descriptors.catalog.json +22 -0
- package/catalox-seeds/inputs/memorix-list-descriptors.items.json +22 -0
- package/catalox-seeds/memorix-retrieval-descriptors.manifest.json +40 -0
- package/dist/client/create-client.js +2 -2
- package/dist/client/create-client.js.map +1 -1
- package/dist/client/create-stack-from-env.js +4 -6
- package/dist/client/create-stack-from-env.js.map +1 -1
- package/dist/client/types.d.ts +48 -0
- package/dist/client/types.d.ts.map +1 -1
- package/dist/descriptors/descriptor-types.d.ts +35 -10
- package/dist/descriptors/descriptor-types.d.ts.map +1 -1
- package/dist/descriptors/discover-entities.d.ts +3 -1
- package/dist/descriptors/discover-entities.d.ts.map +1 -1
- package/dist/descriptors/discover-entities.js +14 -43
- package/dist/descriptors/discover-entities.js.map +1 -1
- package/dist/descriptors/load-list-descriptor.d.ts +5 -1
- package/dist/descriptors/load-list-descriptor.d.ts.map +1 -1
- package/dist/descriptors/load-list-descriptor.js +23 -3
- package/dist/descriptors/load-list-descriptor.js.map +1 -1
- package/dist/descriptors/resolve-app-id.d.ts +4 -0
- package/dist/descriptors/resolve-app-id.d.ts.map +1 -0
- package/dist/descriptors/resolve-app-id.js +8 -0
- package/dist/descriptors/resolve-app-id.js.map +1 -0
- package/dist/descriptors/resolve-default-descriptors.d.ts +11 -0
- package/dist/descriptors/resolve-default-descriptors.d.ts.map +1 -0
- package/dist/descriptors/resolve-default-descriptors.js +22 -0
- package/dist/descriptors/resolve-default-descriptors.js.map +1 -0
- package/dist/descriptors/validate-descriptor.d.ts +6 -2
- package/dist/descriptors/validate-descriptor.d.ts.map +1 -1
- package/dist/descriptors/validate-descriptor.js +42 -5
- package/dist/descriptors/validate-descriptor.js.map +1 -1
- package/dist/explorer/entity-graph.d.ts +3 -1
- package/dist/explorer/entity-graph.d.ts.map +1 -1
- package/dist/explorer/entity-graph.js +4 -0
- package/dist/explorer/entity-graph.js.map +1 -1
- package/dist/index.d.ts +16 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -4
- package/dist/index.js.map +1 -1
- package/dist/retrieval/compose-row.d.ts +2 -2
- package/dist/retrieval/compose-row.d.ts.map +1 -1
- package/dist/retrieval/compose-row.js.map +1 -1
- package/dist/retrieval/fetch-for-entity.d.ts +18 -0
- package/dist/retrieval/fetch-for-entity.d.ts.map +1 -0
- package/dist/retrieval/fetch-for-entity.js +21 -0
- package/dist/retrieval/fetch-for-entity.js.map +1 -0
- package/dist/retrieval/fetch-list.d.ts.map +1 -1
- package/dist/retrieval/fetch-list.js +18 -3
- package/dist/retrieval/fetch-list.js.map +1 -1
- package/dist/retrieval/fetch-workspace-list.d.ts +4 -0
- package/dist/retrieval/fetch-workspace-list.d.ts.map +1 -0
- package/dist/retrieval/fetch-workspace-list.js +164 -0
- package/dist/retrieval/fetch-workspace-list.js.map +1 -0
- package/dist/retrieval/resolve-filters.d.ts +4 -2
- package/dist/retrieval/resolve-filters.d.ts.map +1 -1
- package/dist/retrieval/resolve-filters.js +24 -3
- package/dist/retrieval/resolve-filters.js.map +1 -1
- package/dist/retrieval/resolve-list-search.d.ts +6 -0
- package/dist/retrieval/resolve-list-search.d.ts.map +1 -0
- package/dist/retrieval/resolve-list-search.js +25 -0
- package/dist/retrieval/resolve-list-search.js.map +1 -0
- package/dist/retrieval/resolve-pagination-driver.d.ts +2 -2
- package/dist/retrieval/resolve-pagination-driver.d.ts.map +1 -1
- package/dist/retrieval/resolve-pagination-driver.js.map +1 -1
- package/dist/retrieval/resolve-sort.d.ts +2 -2
- package/dist/retrieval/resolve-sort.d.ts.map +1 -1
- package/dist/retrieval/resolve-sort.js.map +1 -1
- package/dist/seeds/build-manifest.d.ts +8 -0
- package/dist/seeds/build-manifest.d.ts.map +1 -0
- package/dist/seeds/build-manifest.js +83 -0
- package/dist/seeds/build-manifest.js.map +1 -0
- package/dist/seeds/check-needs-seed.d.ts +10 -0
- package/dist/seeds/check-needs-seed.d.ts.map +1 -0
- package/dist/seeds/check-needs-seed.js +66 -0
- package/dist/seeds/check-needs-seed.js.map +1 -0
- package/dist/seeds/default-seed-spec.d.ts +12 -0
- package/dist/seeds/default-seed-spec.d.ts.map +1 -0
- package/dist/seeds/default-seed-spec.js +90 -0
- package/dist/seeds/default-seed-spec.js.map +1 -0
- package/dist/seeds/merge-for-apply.d.ts +28 -0
- package/dist/seeds/merge-for-apply.d.ts.map +1 -0
- package/dist/seeds/merge-for-apply.js +31 -0
- package/dist/seeds/merge-for-apply.js.map +1 -0
- package/dist/seeds/paths.d.ts +10 -0
- package/dist/seeds/paths.d.ts.map +1 -0
- package/dist/seeds/paths.js +40 -0
- package/dist/seeds/paths.js.map +1 -0
- package/dist/seeds/seed-types.d.ts +57 -0
- package/dist/seeds/seed-types.d.ts.map +1 -0
- package/dist/seeds/seed-types.js +2 -0
- package/dist/seeds/seed-types.js.map +1 -0
- package/dist/tests/descriptor-validation.test.js +15 -0
- package/dist/tests/descriptor-validation.test.js.map +1 -1
- package/dist/tests/fetch-workspace-list.test.d.ts +2 -0
- package/dist/tests/fetch-workspace-list.test.d.ts.map +1 -0
- package/dist/tests/fetch-workspace-list.test.js +187 -0
- package/dist/tests/fetch-workspace-list.test.js.map +1 -0
- package/dist/tests/fixtures.d.ts +5 -2
- package/dist/tests/fixtures.d.ts.map +1 -1
- package/dist/tests/fixtures.js +79 -0
- package/dist/tests/fixtures.js.map +1 -1
- package/dist/tests/relations.test.js +3 -0
- package/dist/tests/relations.test.js.map +1 -1
- package/dist/tests/resolve-app-id.test.d.ts +2 -0
- package/dist/tests/resolve-app-id.test.d.ts.map +1 -0
- package/dist/tests/resolve-app-id.test.js +19 -0
- package/dist/tests/resolve-app-id.test.js.map +1 -0
- package/dist/tests/resolve-filters.test.d.ts +2 -0
- package/dist/tests/resolve-filters.test.d.ts.map +1 -0
- package/dist/tests/resolve-filters.test.js +47 -0
- package/dist/tests/resolve-filters.test.js.map +1 -0
- package/dist/tests/seed-check.test.d.ts +2 -0
- package/dist/tests/seed-check.test.d.ts.map +1 -0
- package/dist/tests/seed-check.test.js +47 -0
- package/dist/tests/seed-check.test.js.map +1 -0
- package/dist/tests/seed-manifest.test.d.ts +2 -0
- package/dist/tests/seed-manifest.test.d.ts.map +1 -0
- package/dist/tests/seed-manifest.test.js +80 -0
- package/dist/tests/seed-manifest.test.js.map +1 -0
- package/docs/DATA-TIER-CONTRACT.md +141 -0
- package/docs/EXPLORER-HOST-APIS.md +19 -7
- package/docs/MEMORIX-CATALOX-CONTRACTS.md +587 -0
- package/package.json +5 -3
|
@@ -0,0 +1,587 @@
|
|
|
1
|
+
# Memorix ↔ Catalox contracts
|
|
2
|
+
|
|
3
|
+
**Canonical sync document** for all Memorix data packages and Memorix-consuming UIs (Explorer, completion pipelines, ingestion, graph tools).
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
| Document | Scope |
|
|
7
|
+
|----------|--------|
|
|
8
|
+
| **This file** | Catalox catalogs, descriptor JSON formats, expected Mongo shape, cross-component behavior |
|
|
9
|
+
| [MEMORIX-DATABASE-CONVENTIONS.md](./MEMORIX-DATABASE-CONVENTIONS.md) | Database names, collection naming, env vars, record envelope |
|
|
10
|
+
| [DATA-TIER-CONTRACT.md](./DATA-TIER-CONTRACT.md) | What host apps may call (retrieval APIs only) |
|
|
11
|
+
|
|
12
|
+
**Reference implementation:** `@x12i/memorix-retrieval` — descriptor types in `src/descriptors/descriptor-types.ts`, seeds in `catalox-seeds/inputs/`, validation in `src/descriptors/validate-descriptor.ts`.
|
|
13
|
+
|
|
14
|
+
If this document and a package disagree, **update both together**.
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## 1. Split of responsibilities
|
|
19
|
+
|
|
20
|
+
```text
|
|
21
|
+
┌─────────────────────────────────────────────────────────────────┐
|
|
22
|
+
│ Catalox (metadata) │
|
|
23
|
+
│ • What entities exist │
|
|
24
|
+
│ • How to list / detail / relate them │
|
|
25
|
+
│ • Property paths, filters, sorts, relations │
|
|
26
|
+
└───────────────────────────────┬─────────────────────────────────┘
|
|
27
|
+
│ listCatalogItems / getCatalogItem
|
|
28
|
+
▼
|
|
29
|
+
┌─────────────────────────────────────────────────────────────────┐
|
|
30
|
+
│ @x12i/memorix-retrieval (data tier) │
|
|
31
|
+
│ • Load descriptors from Catalox │
|
|
32
|
+
│ • Resolve Mongo DB + collection from descriptor │
|
|
33
|
+
│ • Read, compose, paginate, multi-match, relations, content │
|
|
34
|
+
└───────────────────────────────┬─────────────────────────────────┘
|
|
35
|
+
│ fetchMemorixList / fetchMemorixItem / graph
|
|
36
|
+
▼
|
|
37
|
+
┌─────────────────────────────────────────────────────────────────┐
|
|
38
|
+
│ Host apps (Explorer, APIs, tools) │
|
|
39
|
+
│ • Never bypass descriptors for entity I/O │
|
|
40
|
+
│ • Never invent entity lists from env or collection names │
|
|
41
|
+
└─────────────────────────────────────────────────────────────────┘
|
|
42
|
+
|
|
43
|
+
┌─────────────────────────────────────────────────────────────────┐
|
|
44
|
+
│ MongoDB (payload) │
|
|
45
|
+
│ • memorix-entities / memorix-events │
|
|
46
|
+
│ • Domain documents under `data` │
|
|
47
|
+
└─────────────────────────────────────────────────────────────────┘
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
| Layer | Owns | Must not own |
|
|
51
|
+
|-------|------|--------------|
|
|
52
|
+
| **Catalox** | Descriptor catalogs, item ids, scope tags, graph semantics | Raw Mongo queries, business payload writes |
|
|
53
|
+
| **memorix-retrieval** | Descriptor-driven reads, composition, validation | Ad hoc catalog types outside the three Memorix catalogs |
|
|
54
|
+
| **Completion / ingestion** | Writing `data` on Memorix records per [MEMORIX-DATABASE-CONVENTIONS](./MEMORIX-DATABASE-CONVENTIONS.md) | Defining list columns or UI field layouts (that's descriptors) |
|
|
55
|
+
| **Explorer / UIs** | Presentation, routing, design model | Direct Mongo, xmemory-store, env-based entity discovery |
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## 2. Catalox app and catalogs
|
|
60
|
+
|
|
61
|
+
### 2.1 App id
|
|
62
|
+
|
|
63
|
+
| Constant | Value |
|
|
64
|
+
|----------|-------|
|
|
65
|
+
| Default `appId` | `memorix` |
|
|
66
|
+
| Env override | `CATALOX_APP_ID` (fallback: `MEMORIX_APP_ID`) |
|
|
67
|
+
|
|
68
|
+
All Memorix descriptor catalogs live under this app. Peers must use the same `appId` unless a deployment explicitly namespaces apps. `createMemorixRetrieval`, `createMemorixRetrievalFromEnv`, and `createMemorixRetrievalStackFromEnv` resolve the app id via `resolveMemorixAppId()`.
|
|
69
|
+
|
|
70
|
+
### 2.2 Catalog ids (only these three)
|
|
71
|
+
|
|
72
|
+
| Catalog id | Item id field | Title field | Purpose |
|
|
73
|
+
|------------|---------------|-------------|---------|
|
|
74
|
+
| `memorix-entity-descriptors` | `id` | `entityName` | Entity schema: properties, content types, relations, default list/item |
|
|
75
|
+
| `memorix-list-descriptors` | `id` | `title` | Tabular list views |
|
|
76
|
+
| `memorix-item-descriptors` | `id` | `title` | Record detail layout |
|
|
77
|
+
|
|
78
|
+
Catalogs use **`sourceMode: "native"`** and **`catalogType: "memorix"`** in seed manifests. Do not introduce parallel catalog names (e.g. `memorix_entity_content_types`) for new work.
|
|
79
|
+
|
|
80
|
+
### 2.3 Item shape in Catalox
|
|
81
|
+
|
|
82
|
+
Each catalog item is stored as:
|
|
83
|
+
|
|
84
|
+
```json
|
|
85
|
+
{
|
|
86
|
+
"id": "<descriptor-id>",
|
|
87
|
+
"...": "<descriptor fields — see §3>",
|
|
88
|
+
"scope": {
|
|
89
|
+
"domains": ["network", "vulnerabilities"],
|
|
90
|
+
"agents": ["neo"]
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
- **`id`** must match the descriptor's own `id` field.
|
|
96
|
+
- **`scope`** is applied at seed time from `catalox-seeds/inputs/scope.json`. All items in a seed bundle share the same scope unless a future manifest explicitly overrides per item.
|
|
97
|
+
- Descriptor payload lives in the item **`data`** body when read via Catalox APIs.
|
|
98
|
+
- **Seed apply note:** the Catalox seed CLI upserts `row.data` only, so `scripts/apply-seed-manifest.mjs` merges `scope` into the upsert `data` object. Hosts may see `scope` inside `data` until Catalox stores top-level scope natively; retrieval ignores unknown fields when validating descriptors.
|
|
99
|
+
|
|
100
|
+
### 2.4 Expected Catalox client behavior
|
|
101
|
+
|
|
102
|
+
Peers that integrate with Catalox must support:
|
|
103
|
+
|
|
104
|
+
| Operation | Used for |
|
|
105
|
+
|-----------|----------|
|
|
106
|
+
| `listCatalogItems(context, catalogId, { limit, offset })` | Entity discovery, paginated catalog reads |
|
|
107
|
+
| `getCatalogItem(context, catalogId, itemId)` | Load list/item/entity descriptor by id |
|
|
108
|
+
|
|
109
|
+
**Discovery rule:** entity existence comes **only** from listing `memorix-entity-descriptors`. There is no env fallback (`MEMORIX_ENTITY_NAMES` and similar bypasses are forbidden in retrieval and hosts).
|
|
110
|
+
|
|
111
|
+
If the catalog is empty or Catalox is unreachable, discovery returns `source: "none"` with a hint to run the seed apply script.
|
|
112
|
+
|
|
113
|
+
---
|
|
114
|
+
|
|
115
|
+
## 3. Descriptor formats
|
|
116
|
+
|
|
117
|
+
Types below mirror `MemorixEntityDescriptor`, `MemorixListDescriptor`, and `MemorixItemDescriptor` in `@x12i/memorix-retrieval`.
|
|
118
|
+
|
|
119
|
+
### 3.1 Entity descriptor (`memorix-entity-descriptors`)
|
|
120
|
+
|
|
121
|
+
**Required top-level fields:**
|
|
122
|
+
|
|
123
|
+
| Field | Type | Rules |
|
|
124
|
+
|-------|------|-------|
|
|
125
|
+
| `id` | string | Stable catalog item id; usually equals `entityName` |
|
|
126
|
+
| `entityName` | string | Kebab-case domain id (`vulnerabilities`, `variabilities-groups`, `assets`) |
|
|
127
|
+
| `defaultListDescriptorId` | string | **Option A** — id in `memorix-list-descriptors` |
|
|
128
|
+
| `defaultItemDescriptorId` | string | **Option A** — id in `memorix-item-descriptors` |
|
|
129
|
+
| `collectionPrefix` | string | Used with content-type `postfix` when `collection` is omitted |
|
|
130
|
+
| `identity` | object | See below |
|
|
131
|
+
| `defaults` | object | Canonical content type, `dataRoot`, effective-date paths |
|
|
132
|
+
| `contentTypes` | object | Named content slices (see §3.4) |
|
|
133
|
+
| `properties` | object | Field dictionary for list/item composition |
|
|
134
|
+
|
|
135
|
+
**Optional:**
|
|
136
|
+
|
|
137
|
+
| Field | Type | Default |
|
|
138
|
+
|-------|------|---------|
|
|
139
|
+
| `target` | `"entity"` \| `"event"` | `"entity"` — selects `memorix-entities` vs `memorix-events` |
|
|
140
|
+
| `contentDefaults` | object | Default content-object storage/format |
|
|
141
|
+
| `relations` | object | Cross-entity joins declared on this entity |
|
|
142
|
+
|
|
143
|
+
**Identity block:**
|
|
144
|
+
|
|
145
|
+
```json
|
|
146
|
+
{
|
|
147
|
+
"allowedIdFields": ["entityId", "eventId"],
|
|
148
|
+
"requiredExactlyOne": true,
|
|
149
|
+
"defaultIdField": "entityId"
|
|
150
|
+
}
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
**Defaults block:**
|
|
154
|
+
|
|
155
|
+
```json
|
|
156
|
+
{
|
|
157
|
+
"canonicalContentType": "snapshots",
|
|
158
|
+
"dataRoot": "data",
|
|
159
|
+
"effectiveDatePath": "capturedAt",
|
|
160
|
+
"fallbackEffectiveDatePaths": ["snapshot.capturedAt", "data.enrichment.enrichedAt"]
|
|
161
|
+
}
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
**Property descriptor** (each key under `properties`):
|
|
165
|
+
|
|
166
|
+
```json
|
|
167
|
+
{
|
|
168
|
+
"label": "Asset IP",
|
|
169
|
+
"source": { "contentType": "snapshots", "path": "data.assetIp" },
|
|
170
|
+
"humanReadable": true,
|
|
171
|
+
"sortable": true,
|
|
172
|
+
"filterable": true,
|
|
173
|
+
"list": true,
|
|
174
|
+
"item": true,
|
|
175
|
+
"valueType": "string"
|
|
176
|
+
}
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
| `valueType` | Notes |
|
|
180
|
+
|-------------|-------|
|
|
181
|
+
| `string`, `number`, `boolean`, `date`, `datetime`, `enum`, `object`, `array` | Scalar / structural |
|
|
182
|
+
| `content` | Value is a `memorix-content-object` (see §3.7) |
|
|
183
|
+
|
|
184
|
+
List views may only expose properties with **`humanReadable: true`**.
|
|
185
|
+
|
|
186
|
+
**Relation descriptor** (under `relations`):
|
|
187
|
+
|
|
188
|
+
```json
|
|
189
|
+
{
|
|
190
|
+
"targetEntity": "assets",
|
|
191
|
+
"type": "manyToOne",
|
|
192
|
+
"source": { "contentType": "snapshots", "path": "data.assetIp" },
|
|
193
|
+
"target": { "contentType": "snapshots", "path": "data.ip_address" },
|
|
194
|
+
"defaultMode": "extendFields",
|
|
195
|
+
"defaultArrayProperty": "vulnerabilities",
|
|
196
|
+
"targetFields": ["ipAddress", "hostName"]
|
|
197
|
+
}
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
| `type` | Meaning |
|
|
201
|
+
|--------|---------|
|
|
202
|
+
| `oneToOne`, `oneToMany`, `manyToOne`, `manyToMany` | Graph semantics for Explorer and `includeRelations` |
|
|
203
|
+
|
|
204
|
+
Join paths are **Mongo document paths** on the respective content types, not Catalox ids.
|
|
205
|
+
|
|
206
|
+
### 3.2 List descriptor (`memorix-list-descriptors`)
|
|
207
|
+
|
|
208
|
+
List descriptors are a **discriminated union** by `kind`:
|
|
209
|
+
|
|
210
|
+
| Kind | When | Key fields |
|
|
211
|
+
|------|------|------------|
|
|
212
|
+
| *(omit or `"entity"`)* | Per-entity tabular list | `entity`, `leadingContentType`, `fields[]` |
|
|
213
|
+
| `"workspace"` | Cross-entity workspace merge | `fields[]` (column hints), optional `entities`, `listDescriptorByEntity` |
|
|
214
|
+
|
|
215
|
+
#### 3.2.1 Entity list (`kind` omitted or `"entity"`)
|
|
216
|
+
|
|
217
|
+
**Required:**
|
|
218
|
+
|
|
219
|
+
| Field | Rules |
|
|
220
|
+
|-------|-------|
|
|
221
|
+
| `id` | Unique list id (e.g. `vulnerabilities-main-list`) |
|
|
222
|
+
| `entity` | `entityName` referencing an entity descriptor |
|
|
223
|
+
| `leadingContentType` | Content type key used to drive pagination/filter/sort |
|
|
224
|
+
| `pagination` | `{ "enabled": true, "defaultLimit": 50, "maxLimit": 200 }` |
|
|
225
|
+
| `fields` | Non-empty array of **property keys** on the entity descriptor |
|
|
226
|
+
|
|
227
|
+
**Common optional fields:**
|
|
228
|
+
|
|
229
|
+
| Field | Purpose |
|
|
230
|
+
|-------|---------|
|
|
231
|
+
| `title` | Human label |
|
|
232
|
+
| `filters` | Default filters (always applied) plus allowlist for request filters; request overrides same property |
|
|
233
|
+
| `allowedSorts`, `defaultSort` | Sort whitelist and default |
|
|
234
|
+
| `extensions` | Pull extra content types by identity (see §3.5) |
|
|
235
|
+
| `includeRelations` | Embed relation data in list rows |
|
|
236
|
+
| `allowSortDrivenLeadingOverride` | Allow sort to switch pagination driver content type |
|
|
237
|
+
|
|
238
|
+
**Example filter:**
|
|
239
|
+
|
|
240
|
+
```json
|
|
241
|
+
{ "property": "severityLevel", "operator": "gte", "value": 4 }
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
**Filter operators:** `eq`, `ne`, `in`, `nin`, `gt`, `gte`, `lt`, `lte`, `exists`, `regex`.
|
|
245
|
+
|
|
246
|
+
Each entity descriptor should define at least one list whose `id` equals `defaultListDescriptorId`. Additional lists (e.g. `critical-vulnerabilities-list`) are optional alternate views.
|
|
247
|
+
|
|
248
|
+
#### 3.2.2 Workspace list (`kind: "workspace"`)
|
|
249
|
+
|
|
250
|
+
Cross-entity workspace lists merge per-entity list composition — used by Explorer **Records** when no entity filter is selected.
|
|
251
|
+
|
|
252
|
+
**Required:**
|
|
253
|
+
|
|
254
|
+
| Field | Rules |
|
|
255
|
+
|-------|-------|
|
|
256
|
+
| `id` | Unique list id (shipped: `memorix-workspace-records-list`) |
|
|
257
|
+
| `kind` | `"workspace"` |
|
|
258
|
+
| `pagination` | Same shape as entity lists |
|
|
259
|
+
| `fields` | Non-empty column hints for hosts (may include synthetic ids like `recordTitle`; rows include per-entity composed fields) |
|
|
260
|
+
|
|
261
|
+
**Optional:**
|
|
262
|
+
|
|
263
|
+
| Field | Purpose |
|
|
264
|
+
|-------|---------|
|
|
265
|
+
| `title` | Human label |
|
|
266
|
+
| `entities` | Subset of discovered entity names; omit = all from `discoverMemorixEntities` |
|
|
267
|
+
| `listDescriptorByEntity` | Override list id per entity; omit = `defaultListDescriptorId` |
|
|
268
|
+
| `defaultSort` | Global sort after merge (`property`, `direction`) |
|
|
269
|
+
| `scopeFilter` | `"inherit-seed-scope"` — Catalox scope hint (not Mongo namespace) |
|
|
270
|
+
|
|
271
|
+
### 3.3 Item descriptor (`memorix-item-descriptors`)
|
|
272
|
+
|
|
273
|
+
**Required:**
|
|
274
|
+
|
|
275
|
+
| Field | Rules |
|
|
276
|
+
|-------|-------|
|
|
277
|
+
| `id` | Unique item id (e.g. `vulnerability-detail-item`) |
|
|
278
|
+
| `entity` | `entityName` |
|
|
279
|
+
| `identity.idField` | `"entityId"` or `"eventId"` — must match request |
|
|
280
|
+
| `contentTypes` | Array of `{ contentType, required, multiMatch? }` |
|
|
281
|
+
| `sections` | Array of `{ id, title, fields[] }` — `fields` are property keys |
|
|
282
|
+
|
|
283
|
+
**Optional:**
|
|
284
|
+
|
|
285
|
+
| Field | Purpose |
|
|
286
|
+
|-------|---------|
|
|
287
|
+
| `includeRelations` | `{ relation, mode?, fields? }` — relation key from entity descriptor |
|
|
288
|
+
| `content` | `{ allowed, defaultIncludeFullContent?, allowedFields?, maxBytes? }` |
|
|
289
|
+
|
|
290
|
+
Each entity descriptor should define at least one item whose `id` equals `defaultItemDescriptorId`.
|
|
291
|
+
|
|
292
|
+
### 3.4 Content type descriptor
|
|
293
|
+
|
|
294
|
+
Under `entity.contentTypes.<name>`:
|
|
295
|
+
|
|
296
|
+
```json
|
|
297
|
+
{
|
|
298
|
+
"postfix": "snapshots",
|
|
299
|
+
"collection": "vulnerabilities-snapshots",
|
|
300
|
+
"dataRoot": "data",
|
|
301
|
+
"isCanonical": true,
|
|
302
|
+
"effectiveDatePath": "capturedAt",
|
|
303
|
+
"fallbackEffectiveDatePaths": ["snapshot.capturedAt"]
|
|
304
|
+
}
|
|
305
|
+
```
|
|
306
|
+
|
|
307
|
+
**Collection resolution order** (retrieval):
|
|
308
|
+
|
|
309
|
+
1. Explicit `collection` on the content type
|
|
310
|
+
2. Env `MEMORIX_ENTITIES_COLLECTION_<ENTITY>` or `MEMORIX_EVENTS_COLLECTION_<ENTITY>`
|
|
311
|
+
3. Heuristic: `{collectionPrefix}-{postfix}` (e.g. `assets-snapshots`)
|
|
312
|
+
|
|
313
|
+
**Database resolution:** entity descriptor `target` → `memorix-entities` or `memorix-events` (see [MEMORIX-DATABASE-CONVENTIONS](./MEMORIX-DATABASE-CONVENTIONS.md)).
|
|
314
|
+
|
|
315
|
+
Exactly one content type per entity should set **`isCanonical: true`**; it aligns with `defaults.canonicalContentType`.
|
|
316
|
+
|
|
317
|
+
### 3.5 Content extensions (list)
|
|
318
|
+
|
|
319
|
+
```json
|
|
320
|
+
{
|
|
321
|
+
"contentType": "enrichment",
|
|
322
|
+
"mode": "extendFields",
|
|
323
|
+
"join": { "by": "entityId" },
|
|
324
|
+
"multiMatch": { "strategy": "last", "effectiveDatePath": "capturedAt" },
|
|
325
|
+
"fields": { "enrichedAt": "data.enrichment.enrichedAt" }
|
|
326
|
+
}
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
| `mode` | Behavior |
|
|
330
|
+
|--------|----------|
|
|
331
|
+
| `extendFields` | Merge selected paths onto the row |
|
|
332
|
+
| `array` | Attach array under `arrayProperty` |
|
|
333
|
+
|
|
334
|
+
### 3.6 Multi-match
|
|
335
|
+
|
|
336
|
+
When multiple Mongo documents share the same identity, retrieval picks records using:
|
|
337
|
+
|
|
338
|
+
```json
|
|
339
|
+
{
|
|
340
|
+
"strategy": "last" | "first" | "all" | "error" | "ignore",
|
|
341
|
+
"effectiveDatePath": "capturedAt",
|
|
342
|
+
"fallbackEffectiveDatePaths": ["snapshot.capturedAt"]
|
|
343
|
+
}
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
Default for item fetches: **`last`** by effective date. Strategies are declared on item `contentTypes`, content-type defaults, or extension blocks.
|
|
347
|
+
|
|
348
|
+
### 3.7 Content objects
|
|
349
|
+
|
|
350
|
+
Properties with `valueType: "content"` store a **Memorix content object** in Mongo:
|
|
351
|
+
|
|
352
|
+
```json
|
|
353
|
+
{
|
|
354
|
+
"contentKey": "stories/asset-10-150-68-31.md",
|
|
355
|
+
"metadata": { "title": "Impact story" },
|
|
356
|
+
"preview": "First 200 chars…"
|
|
357
|
+
}
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
Storage is configured on the entity (`contentDefaults`) or property (`content.storage`), never with inline credentials:
|
|
361
|
+
|
|
362
|
+
```json
|
|
363
|
+
{
|
|
364
|
+
"provider": "gcs" | "s3",
|
|
365
|
+
"bucket": "memorix-content",
|
|
366
|
+
"prefix": "stories/",
|
|
367
|
+
"maxBytes": 65536,
|
|
368
|
+
"encoding": "utf8",
|
|
369
|
+
"credentialsRef": "env:MEMORIX_GCS_CREDENTIALS"
|
|
370
|
+
}
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
Hosts provide **`contentReaders`** at retrieval bootstrap; list views return preview/metadata only (`listBehavior.fetchContent: false`).
|
|
374
|
+
|
|
375
|
+
---
|
|
376
|
+
|
|
377
|
+
## 4. What we expect to find in Mongo
|
|
378
|
+
|
|
379
|
+
### 4.1 Record envelope
|
|
380
|
+
|
|
381
|
+
Documents in Memorix target databases follow [MEMORIX-DATABASE-CONVENTIONS](./MEMORIX-DATABASE-CONVENTIONS.md):
|
|
382
|
+
|
|
383
|
+
```json
|
|
384
|
+
{
|
|
385
|
+
"_id": "<ObjectId>",
|
|
386
|
+
"recordId": "<optional>",
|
|
387
|
+
"entityId": "<for entity-targeted records>",
|
|
388
|
+
"eventId": "<for event-targeted records>",
|
|
389
|
+
"capturedAt": "<ISO-8601>",
|
|
390
|
+
"data": { "... domain fields referenced by descriptor paths ..." }
|
|
391
|
+
}
|
|
392
|
+
```
|
|
393
|
+
|
|
394
|
+
**Peer rules:**
|
|
395
|
+
|
|
396
|
+
- Domain payload lives under **`data`** (or content-type-specific `dataRoot`, almost always `data`).
|
|
397
|
+
- Descriptor `source.path` values are **dot paths from the document root** (e.g. `data.assetIp`, not `assetIp` alone).
|
|
398
|
+
- Identity for list pagination and item fetch: **`entityId`** or **`eventId`** as declared on the item descriptor.
|
|
399
|
+
- Legacy `snapshotId` may be read as `recordId` fallback; do not write it in new code.
|
|
400
|
+
|
|
401
|
+
### 4.2 Shipped entity ↔ database mapping
|
|
402
|
+
|
|
403
|
+
| Entity (`entityName`) | `target` | Typical collection(s) |
|
|
404
|
+
|-----------------------|----------|------------------------|
|
|
405
|
+
| `assets` | `entity` | `assets-snapshots` in `memorix-entities` |
|
|
406
|
+
| `variabilities-groups` | `entity` | `variabilities-groups-snapshots` |
|
|
407
|
+
| `vulnerabilities` | `event` | `vulnerabilities-snapshots` in `memorix-events` |
|
|
408
|
+
|
|
409
|
+
Ingestion and completion must populate paths referenced in entity descriptors. If a property path is missing, retrieval returns null/omits the field and may emit **`EXTENSION_MISSING`** issues for required content types.
|
|
410
|
+
|
|
411
|
+
### 4.3 Relations in data
|
|
412
|
+
|
|
413
|
+
Relations are **not** stored as Catalox edges. They are resolved at read time by matching:
|
|
414
|
+
|
|
415
|
+
- `relation.source.path` on the source record
|
|
416
|
+
- `relation.target.path` on target entity records
|
|
417
|
+
|
|
418
|
+
Both sides must use the same **`contentType`** keys declared on the relation. Target field projection uses `targetFields` / `includeRelations.fields`.
|
|
419
|
+
|
|
420
|
+
---
|
|
421
|
+
|
|
422
|
+
## 5. Runtime behavior (retrieval)
|
|
423
|
+
|
|
424
|
+
All Memorix-consuming components should rely on these behaviors rather than reimplementing them. Public exports are listed in [DATA-TIER-CONTRACT.md](./DATA-TIER-CONTRACT.md).
|
|
425
|
+
|
|
426
|
+
**Host bootstrap:** prefer `createMemorixRetrievalStackFromEnv()` for Catalox + Mongo wiring; see [EXPLORER-HOST-APIS.md](./EXPLORER-HOST-APIS.md) for graph/raw-read helpers.
|
|
427
|
+
|
|
428
|
+
### 5.1 Discovery
|
|
429
|
+
|
|
430
|
+
```
|
|
431
|
+
discoverMemorixEntities(client)
|
|
432
|
+
→ list all items in memorix-entity-descriptors
|
|
433
|
+
→ validate each as MemorixEntityDescriptor
|
|
434
|
+
→ return summaries: id, entityName, label, defaultListDescriptorId, defaultItemDescriptorId, target
|
|
435
|
+
```
|
|
436
|
+
|
|
437
|
+
### 5.2 Entity-default list and item
|
|
438
|
+
|
|
439
|
+
```
|
|
440
|
+
fetchMemorixListForEntity(client, { entityName, ... })
|
|
441
|
+
→ load entity descriptor
|
|
442
|
+
→ listId = entity.defaultListDescriptorId
|
|
443
|
+
→ fetchMemorixList(client, { listId, ... })
|
|
444
|
+
|
|
445
|
+
fetchMemorixItemForEntity(client, { entityName, entityId | eventId, ... })
|
|
446
|
+
→ load entity descriptor
|
|
447
|
+
→ itemDescriptorId = entity.defaultItemDescriptorId
|
|
448
|
+
→ fetchMemorixItem(client, { itemDescriptorId, ... })
|
|
449
|
+
```
|
|
450
|
+
|
|
451
|
+
### 5.3 List composition
|
|
452
|
+
|
|
453
|
+
1. Load list + entity descriptors from Catalox.
|
|
454
|
+
2. Validate every `fields[]` entry exists and is `humanReadable`.
|
|
455
|
+
3. Resolve pagination driver = `leadingContentType` (or sort override if allowed).
|
|
456
|
+
4. Query driver collection with list default filters merged with request filters, plus optional `searchText` across human-readable fields.
|
|
457
|
+
5. Optionally batch-fetch extensions and relations.
|
|
458
|
+
6. Return rows as flat property maps + pagination + issues.
|
|
459
|
+
|
|
460
|
+
### 5.4 Workspace list composition
|
|
461
|
+
|
|
462
|
+
```
|
|
463
|
+
fetchMemorixWorkspaceList(client, { workspaceListId?, entityName?, page?, searchText?, sort?, includeTotal? })
|
|
464
|
+
→ load workspace list descriptor (default: memorix-workspace-records-list)
|
|
465
|
+
→ if entityName: delegate to fetchMemorixListForEntity, stamp rows with entityName
|
|
466
|
+
→ discoverMemorixEntities → filter by workspace.entities?
|
|
467
|
+
→ for each entity: resolve list id → fetchMemorixList (same field composition as §5.3)
|
|
468
|
+
→ stamp entityName (+ identity) on each row
|
|
469
|
+
→ merge, global sort, paginate in memory (v1)
|
|
470
|
+
→ emit issues[] for per-entity failures without failing the whole workspace
|
|
471
|
+
```
|
|
472
|
+
|
|
473
|
+
When `discovery.source` is `"none"`, returns empty rows with a hint — no env entity fallback.
|
|
474
|
+
|
|
475
|
+
### 5.5 Item composition
|
|
476
|
+
|
|
477
|
+
1. Load item + entity descriptors.
|
|
478
|
+
2. Fetch all documents per `contentTypes[]` matching identity.
|
|
479
|
+
3. Apply `multiMatch` per content type.
|
|
480
|
+
4. Build `sections[]` from property definitions.
|
|
481
|
+
5. Resolve `includeRelations` and optional full content fetches.
|
|
482
|
+
6. Return `{ identity, sections, relations?, issues? }`.
|
|
483
|
+
|
|
484
|
+
### 5.6 Entity graph (Explorer)
|
|
485
|
+
|
|
486
|
+
`buildMemorixEntityGraph` uses discovery + entity descriptors only:
|
|
487
|
+
|
|
488
|
+
- Per entity: label, counts, content-type labels, **`relations` → connections**, default list/item ids.
|
|
489
|
+
- `discovery.source` is `"catalox"` or `"none"` — never `"env"`.
|
|
490
|
+
|
|
491
|
+
---
|
|
492
|
+
|
|
493
|
+
## 6. Seeding and keeping peers in sync
|
|
494
|
+
|
|
495
|
+
### 6.1 Source of truth
|
|
496
|
+
|
|
497
|
+
| Artifact | Location |
|
|
498
|
+
|----------|----------|
|
|
499
|
+
| Editable seed inputs | `@x12i/memorix-retrieval` → `catalox-seeds/inputs/` |
|
|
500
|
+
| Built manifest | `catalox-seeds/memorix-retrieval-descriptors.manifest.json` |
|
|
501
|
+
| TypeScript types | `src/descriptors/descriptor-types.ts` |
|
|
502
|
+
|
|
503
|
+
Workflow for descriptor changes:
|
|
504
|
+
|
|
505
|
+
```bash
|
|
506
|
+
# In memorix-retrieval
|
|
507
|
+
npm run catalox:seed:build
|
|
508
|
+
npm run catalox:seed:memorix-retrieval:validate
|
|
509
|
+
npm run catalox:seed:memorix-retrieval:apply # force apply
|
|
510
|
+
npm run catalox:seed:ensure # apply only if items missing
|
|
511
|
+
```
|
|
512
|
+
|
|
513
|
+
Seed helpers (`buildMemorixRetrievalSeedManifest`, `checkMemorixRetrievalSeedNeeds`, …) are exported from the package; `scripts/` wrap them for CLI use. See [DATA-TIER-CONTRACT.md](./DATA-TIER-CONTRACT.md#catalox-seed-helpers-library).
|
|
514
|
+
|
|
515
|
+
Then bump `@x12i/memorix-retrieval` and reinstall in consuming apps.
|
|
516
|
+
|
|
517
|
+
### 6.2 Adding a new entity (checklist)
|
|
518
|
+
|
|
519
|
+
1. Add `catalox-seeds/inputs/entity-descriptors/<name>.json` with **both default list and item ids**.
|
|
520
|
+
2. Add at least one list under `list-descriptors/` and one item under `item-descriptors/`.
|
|
521
|
+
3. Register ids in `catalox-seeds/inputs/manifest.json`.
|
|
522
|
+
4. Rebuild and apply seed manifest.
|
|
523
|
+
5. Ensure Mongo collections exist and documents match declared paths.
|
|
524
|
+
6. Extend `src/tests/fixtures.ts` and validation tests in retrieval.
|
|
525
|
+
7. Publish retrieval; update Explorer (or other hosts) dependency.
|
|
526
|
+
|
|
527
|
+
### 6.3 Versioning
|
|
528
|
+
|
|
529
|
+
- **Descriptor schema changes** → minor/major bump of `@x12i/memorix-retrieval` + coordinated seed apply.
|
|
530
|
+
- **Seed-only changes** (fields, filters, new list views) → seed apply + retrieval patch as needed; hosts pick up via graph/list/item APIs automatically.
|
|
531
|
+
- Do not fork catalog ids per app; use **`scope`** for domain/agent filtering instead.
|
|
532
|
+
|
|
533
|
+
---
|
|
534
|
+
|
|
535
|
+
## 7. Component matrix
|
|
536
|
+
|
|
537
|
+
| Component | Reads Catalox | Writes Catalox | Reads Mongo | Writes Mongo |
|
|
538
|
+
|-----------|---------------|----------------|-------------|--------------|
|
|
539
|
+
| `@x12i/memorix-retrieval` | Yes (3 catalogs) | No (seeds via CLI) | Yes (descriptor-driven) | No |
|
|
540
|
+
| `@x12i/memorix-completion` | No* | No | Yes (source + targets) | Yes (`data` on targets) |
|
|
541
|
+
| `@x12i/memorix-explorer` | Via retrieval | No | No (via retrieval only) | No |
|
|
542
|
+
| Ingestion pipelines | No* | No | Yes | Yes |
|
|
543
|
+
|
|
544
|
+
\*Completion and ingestion should **align** with descriptor paths and collection names documented here; they do not need Catalox at runtime unless they also ship seed tooling.
|
|
545
|
+
|
|
546
|
+
---
|
|
547
|
+
|
|
548
|
+
## 8. Forbidden patterns
|
|
549
|
+
|
|
550
|
+
These break cross-component sync and must not appear in new code:
|
|
551
|
+
|
|
552
|
+
| Pattern | Why |
|
|
553
|
+
|---------|-----|
|
|
554
|
+
| Host Mongo list/get/count by collection name | Bypasses list/item descriptors |
|
|
555
|
+
| `MEMORIX_ENTITY_NAMES` or env entity lists | Bypasses Catalox discovery |
|
|
556
|
+
| `@x12i/xmemory-store` / scoped tier in Explorer | Legacy; replaced by retrieval + descriptors |
|
|
557
|
+
| Parallel catalog types for the same semantics | Fragments truth |
|
|
558
|
+
| Credentials inside descriptor JSON | Use `credentialsRef` + host-injected readers |
|
|
559
|
+
| List fields with `humanReadable: false` | Validation error at fetch time |
|
|
560
|
+
| Entity descriptor missing `defaultListDescriptorId` / `defaultItemDescriptorId` | Breaks Option A wiring |
|
|
561
|
+
|
|
562
|
+
---
|
|
563
|
+
|
|
564
|
+
## 9. Quick reference — shipped ids
|
|
565
|
+
|
|
566
|
+
From `catalox-seeds/inputs/manifest.json`:
|
|
567
|
+
|
|
568
|
+
**Entities:** `assets`, `vulnerabilities`, `variabilities-groups`
|
|
569
|
+
|
|
570
|
+
**Lists:** `assets-main-list`, `vulnerabilities-main-list`, `critical-vulnerabilities-list`, `variabilities-groups-main-list`, `memorix-workspace-records-list` (workspace)
|
|
571
|
+
|
|
572
|
+
**Items:** `asset-detail-item`, `vulnerability-detail-item`, `variabilities-group-detail-item`
|
|
573
|
+
|
|
574
|
+
**Scope (all seeded items):** `domains: [network, vulnerabilities]`, `agents: [neo]`
|
|
575
|
+
|
|
576
|
+
---
|
|
577
|
+
|
|
578
|
+
## 10. Related packages
|
|
579
|
+
|
|
580
|
+
| Package | Role |
|
|
581
|
+
|---------|------|
|
|
582
|
+
| `@x12i/catalox` | Catalog storage and seed CLI |
|
|
583
|
+
| `@x12i/memorix-retrieval` | Descriptor-driven data tier |
|
|
584
|
+
| `@x12i/memorix-completion` | Source → Memorix enrichment (payload paths must match descriptors) |
|
|
585
|
+
| `@x12i/memorix-explorer` | UI host — consumes retrieval APIs only |
|
|
586
|
+
|
|
587
|
+
When adding a capability all hosts need (e.g. cross-entity workspace list), **extend retrieval and descriptors first**, then update hosts — never shim in a single app.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@x12i/memorix-retrieval",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "Descriptor-driven Memorix retrieval layer for lists, items, and content objects",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -19,9 +19,10 @@
|
|
|
19
19
|
"scripts": {
|
|
20
20
|
"build": "tsc",
|
|
21
21
|
"test": "vitest run",
|
|
22
|
-
"catalox:seed:build": "node scripts/build-seed-manifest.mjs",
|
|
22
|
+
"catalox:seed:build": "npm run build && node scripts/build-seed-manifest.mjs",
|
|
23
23
|
"catalox:seed:memorix-retrieval:validate": "npm run catalox:seed:build && catalox seed validate --file catalox-seeds/memorix-retrieval-descriptors.manifest.json",
|
|
24
|
-
"catalox:seed:memorix-retrieval:apply": "npm run
|
|
24
|
+
"catalox:seed:memorix-retrieval:apply": "npm run build && node scripts/apply-seed-manifest.mjs",
|
|
25
|
+
"catalox:seed:ensure": "npm run build && node scripts/ensure-catalox-seed.mjs",
|
|
25
26
|
"mongo:check-collections": "node scripts/check-mongo-collections.mjs",
|
|
26
27
|
"smoke:retrieval": "npm run build && node scripts/smoke-retrieval.mjs"
|
|
27
28
|
},
|
|
@@ -31,6 +32,7 @@
|
|
|
31
32
|
"dependencies": {
|
|
32
33
|
"@x12i/catalox": "^4.1.2",
|
|
33
34
|
"@x12i/helpers": "^1.7.0",
|
|
35
|
+
"@x12i/memorix-retrieval": "^1.4.0",
|
|
34
36
|
"mongodb": "^6.21.0"
|
|
35
37
|
},
|
|
36
38
|
"peerDependencies": {
|