@objectstack/metadata 3.0.1 → 3.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +10 -10
- package/CHANGELOG.md +19 -0
- package/README.md +238 -6
- package/ROADMAP.md +223 -0
- package/dist/index.d.mts +22 -1
- package/dist/index.d.ts +22 -1
- package/dist/index.js +45 -4
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +45 -4
- package/dist/index.mjs.map +1 -1
- package/package.json +4 -4
- package/src/metadata-manager.ts +62 -5
- package/src/metadata-service.test.ts +85 -0
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
|
|
2
|
-
> @objectstack/metadata@3.0.
|
|
2
|
+
> @objectstack/metadata@3.0.3 build /home/runner/work/spec/spec/packages/metadata
|
|
3
3
|
> tsup --config ../../tsup.config.ts
|
|
4
4
|
|
|
5
5
|
[34mCLI[39m Building entry: src/index.ts
|
|
@@ -10,13 +10,13 @@
|
|
|
10
10
|
[34mCLI[39m Cleaning output folder
|
|
11
11
|
[34mESM[39m Build start
|
|
12
12
|
[34mCJS[39m Build start
|
|
13
|
-
[
|
|
14
|
-
[
|
|
15
|
-
[
|
|
16
|
-
[
|
|
17
|
-
[
|
|
18
|
-
[
|
|
13
|
+
[32mCJS[39m [1mdist/index.js [22m[32m46.91 KB[39m
|
|
14
|
+
[32mCJS[39m [1mdist/index.js.map [22m[32m93.62 KB[39m
|
|
15
|
+
[32mCJS[39m ⚡️ Build success in 82ms
|
|
16
|
+
[32mESM[39m [1mdist/index.mjs [22m[32m45.10 KB[39m
|
|
17
|
+
[32mESM[39m [1mdist/index.mjs.map [22m[32m91.64 KB[39m
|
|
18
|
+
[32mESM[39m ⚡️ Build success in 93ms
|
|
19
19
|
[34mDTS[39m Build start
|
|
20
|
-
[32mDTS[39m ⚡️ Build success in
|
|
21
|
-
[32mDTS[39m [1mdist/index.d.mts [22m[
|
|
22
|
-
[32mDTS[39m [1mdist/index.d.ts [22m[
|
|
20
|
+
[32mDTS[39m ⚡️ Build success in 10467ms
|
|
21
|
+
[32mDTS[39m [1mdist/index.d.mts [22m[32m15.42 KB[39m
|
|
22
|
+
[32mDTS[39m [1mdist/index.d.ts [22m[32m15.42 KB[39m
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# @objectstack/metadata
|
|
2
2
|
|
|
3
|
+
## 3.0.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- c7267f6: Patch release for maintenance updates and improvements.
|
|
8
|
+
- Updated dependencies [c7267f6]
|
|
9
|
+
- @objectstack/spec@3.0.3
|
|
10
|
+
- @objectstack/core@3.0.3
|
|
11
|
+
- @objectstack/types@3.0.3
|
|
12
|
+
|
|
13
|
+
## 3.0.2
|
|
14
|
+
|
|
15
|
+
### Patch Changes
|
|
16
|
+
|
|
17
|
+
- Updated dependencies [28985f5]
|
|
18
|
+
- @objectstack/spec@3.0.2
|
|
19
|
+
- @objectstack/core@3.0.2
|
|
20
|
+
- @objectstack/types@3.0.2
|
|
21
|
+
|
|
3
22
|
## 3.0.1
|
|
4
23
|
|
|
5
24
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -1,13 +1,245 @@
|
|
|
1
1
|
# @objectstack/metadata
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
> **Metadata Loading, Persistence & Customization Layer for ObjectStack.**
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
`@objectstack/metadata` is the central service responsible for loading, validating, persisting and watching all metadata definitions (Objects, Views, Flows, Apps, Agents, etc.) in the ObjectStack platform.
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
It implements the **`IMetadataService`** contract from `@objectstack/spec` and acts as the single source of truth that all other packages depend on.
|
|
8
|
+
|
|
9
|
+
## Architecture Overview
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
┌─────────────────────────────────────────────────────────────┐
|
|
13
|
+
│ IMetadataService │
|
|
14
|
+
│ (Contract: @objectstack/spec) │
|
|
15
|
+
├─────────────────────────────────────────────────────────────┤
|
|
16
|
+
│ MetadataManager │
|
|
17
|
+
│ (Orchestrator: this package) │
|
|
18
|
+
│ │
|
|
19
|
+
│ ┌─────────────┐ ┌──────────────┐ ┌───────────────────┐ │
|
|
20
|
+
│ │ In-Memory │ │ Overlay │ │ Type Registry │ │
|
|
21
|
+
│ │ Registry │ │ System │ │ & Dependencies │ │
|
|
22
|
+
│ └─────────────┘ └──────────────┘ └───────────────────┘ │
|
|
23
|
+
├─────────────────────────────────────────────────────────────┤
|
|
24
|
+
│ Loader Layer │
|
|
25
|
+
│ ┌─────────────┐ ┌──────────────┐ ┌───────────────────┐ │
|
|
26
|
+
│ │ Filesystem │ │ Remote │ │ Memory │ │
|
|
27
|
+
│ │ Loader │ │ Loader │ │ Loader │ │
|
|
28
|
+
│ │ (files) │ │ (HTTP) │ │ (test) │ │
|
|
29
|
+
│ └─────────────┘ └──────────────┘ └───────────────────┘ │
|
|
30
|
+
│ ┌──────────────────────────────────────────────────────┐ │
|
|
31
|
+
│ │ DatabaseLoader (planned — datasource-backed storage) │ │
|
|
32
|
+
│ └──────────────────────────────────────────────────────┘ │
|
|
33
|
+
├─────────────────────────────────────────────────────────────┤
|
|
34
|
+
│ Serializer Layer │
|
|
35
|
+
│ ┌──────────┐ ┌──────────┐ ┌──────────────────────────┐ │
|
|
36
|
+
│ │ JSON │ │ YAML │ │ TypeScript/JavaScript │ │
|
|
37
|
+
│ └──────────┘ └──────────┘ └──────────────────────────┘ │
|
|
38
|
+
└─────────────────────────────────────────────────────────────┘
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Core Concepts
|
|
42
|
+
|
|
43
|
+
### 1. Metadata Sources (Three-Scope Model)
|
|
44
|
+
|
|
45
|
+
ObjectStack adopts a three-scope layered model for metadata:
|
|
46
|
+
|
|
47
|
+
| Scope | Storage | Mutability | Description |
|
|
48
|
+
|:-----------|:-------------|:-------------|:-------------------------------------------|
|
|
49
|
+
| `system` | Filesystem | Read-only | Defined in code, shipped with packages |
|
|
50
|
+
| `platform` | Database | Admin-editable | Created/modified by admins via UI |
|
|
51
|
+
| `user` | Database | User-editable | Personal customizations per user |
|
|
52
|
+
|
|
53
|
+
Resolution order: **system** ← merge(**platform**) ← merge(**user**).
|
|
54
|
+
|
|
55
|
+
### 2. Loaders
|
|
56
|
+
|
|
57
|
+
Loaders are pluggable data sources that know how to read/write metadata from different backends. Each loader declares a `MetadataLoaderContract` with name, protocol, and capabilities:
|
|
58
|
+
|
|
59
|
+
| Loader | Protocol | Read | Write | Watch | Status |
|
|
60
|
+
|:--------------------|:---------------|:-----|:------|:------|:-------------|
|
|
61
|
+
| `FilesystemLoader` | `file:` | ✅ | ✅ | ✅ | Implemented |
|
|
62
|
+
| `MemoryLoader` | `memory:` | ✅ | ✅ | ❌ | Implemented |
|
|
63
|
+
| `RemoteLoader` | `http:` | ✅ | ✅ | ❌ | Implemented |
|
|
64
|
+
| `DatabaseLoader` | `datasource:` | ✅ | ✅ | ✅ | Planned |
|
|
65
|
+
|
|
66
|
+
### 3. Serializers
|
|
67
|
+
|
|
68
|
+
Serializers convert metadata objects to/from different file formats:
|
|
69
|
+
|
|
70
|
+
- **JSONSerializer** — `.json` files with optional key sorting
|
|
71
|
+
- **YAMLSerializer** — `.yaml`/`.yml` files (JSON_SCHEMA for security)
|
|
72
|
+
- **TypeScriptSerializer** — `.ts`/`.js` module exports (for `defineObject()`, `defineView()`, etc.)
|
|
73
|
+
|
|
74
|
+
### 4. Overlay / Customization System
|
|
75
|
+
|
|
76
|
+
The overlay system enables non-destructive customizations on top of package-delivered (system) metadata, following a delta-based approach (JSON Merge Patch):
|
|
77
|
+
|
|
78
|
+
- **getOverlay** / **saveOverlay** / **removeOverlay** — manage customization deltas
|
|
79
|
+
- **getEffective** — returns the merged result of base + platform overlay + user overlay
|
|
80
|
+
- Overlays never modify the base definition — they are additive patches
|
|
81
|
+
|
|
82
|
+
### 5. MetadataManager (IMetadataService Implementation)
|
|
83
|
+
|
|
84
|
+
The `MetadataManager` is the main orchestrator. It provides:
|
|
85
|
+
|
|
86
|
+
- **Core CRUD**: `register`, `get`, `list`, `unregister`, `exists`, `listNames`
|
|
87
|
+
- **Convenience**: `getObject`, `listObjects`
|
|
88
|
+
- **Package Management**: `unregisterPackage` — unload all metadata from a package
|
|
89
|
+
- **Query / Search**: `query` with filtering, pagination, sorting by type/scope/state/tags
|
|
90
|
+
- **Bulk Operations**: `bulkRegister`, `bulkUnregister` with error handling
|
|
91
|
+
- **Import / Export**: `exportMetadata`, `importMetadata` with conflict resolution (skip/overwrite/merge)
|
|
92
|
+
- **Validation**: `validate` — structural validation of metadata items
|
|
93
|
+
- **Type Registry**: `getRegisteredTypes`, `getTypeInfo` — discover available metadata types
|
|
94
|
+
- **Dependency Tracking**: `getDependencies`, `getDependents` — cross-reference analysis
|
|
95
|
+
- **Watch / Subscribe**: `watchService` — observe metadata changes in real-time
|
|
96
|
+
- **Loader Delegation**: `load`, `loadMany`, `save` — delegate I/O to registered loaders
|
|
97
|
+
|
|
98
|
+
### 6. NodeMetadataManager
|
|
99
|
+
|
|
100
|
+
Extends `MetadataManager` with Node.js-specific capabilities:
|
|
101
|
+
|
|
102
|
+
- Auto-configures `FilesystemLoader` for local development
|
|
103
|
+
- File watching via **chokidar** for hot-reload during development
|
|
104
|
+
- Detects file add/change/delete events and notifies subscribers
|
|
105
|
+
|
|
106
|
+
### 7. MetadataPlugin
|
|
107
|
+
|
|
108
|
+
Integrates with the ObjectStack kernel plugin system:
|
|
109
|
+
|
|
110
|
+
- Registers as the primary `IMetadataService` provider
|
|
111
|
+
- Auto-loads all metadata types from the filesystem on startup (sorted by `loadOrder`)
|
|
112
|
+
- Supports YAML, JSON, TypeScript, and JavaScript metadata formats
|
|
113
|
+
|
|
114
|
+
## Metadata Types
|
|
115
|
+
|
|
116
|
+
The platform supports **26 built-in metadata types** across 6 protocol domains:
|
|
117
|
+
|
|
118
|
+
| Domain | Types |
|
|
119
|
+
|:-------------|:----------------------------------------------------------------------------|
|
|
120
|
+
| **Data** | `object`, `field`, `datasource`, `validation` |
|
|
121
|
+
| **UI** | `view`, `app`, `dashboard`, `report`, `action`, `theme` |
|
|
122
|
+
| **Automation** | `flow`, `workflow`, `trigger`, `schedule` |
|
|
123
|
+
| **System** | `manifest`, `translation`, `api`, `permission_set`, `role`, `profile` |
|
|
124
|
+
| **Security** | `permission_set`, `role` |
|
|
125
|
+
| **AI** | `agent`, `rag_pipeline`, `model`, `prompt`, `tool` |
|
|
126
|
+
|
|
127
|
+
Each type has a defined `loadOrder` (dependencies load before dependents), file patterns (e.g. `**/*.object.{ts,json,yaml}`), and overlay support flag.
|
|
128
|
+
|
|
129
|
+
## Spec Protocol References
|
|
130
|
+
|
|
131
|
+
This package depends on schemas and contracts defined in `@objectstack/spec`:
|
|
132
|
+
|
|
133
|
+
| Spec Module | What It Defines |
|
|
134
|
+
|:---------------------------------|:----------------------------------------------------|
|
|
135
|
+
| `spec/contracts/metadata-service` | `IMetadataService` — the async service interface |
|
|
136
|
+
| `spec/kernel/metadata-loader` | Loader contract, load/save/watch schemas, `MetadataManagerConfig` |
|
|
137
|
+
| `spec/kernel/metadata-plugin` | Type registry, plugin manifest, capabilities |
|
|
138
|
+
| `spec/kernel/metadata-customization` | Overlay, merge strategy, customization policy |
|
|
139
|
+
| `spec/system/metadata-persistence` | `MetadataRecord` — DB persistence envelope |
|
|
140
|
+
| `spec/data/datasource` | `DatasourceSchema`, `DriverDefinition`, capabilities |
|
|
141
|
+
| `spec/contracts/data-driver` | `IDataDriver` — database driver interface |
|
|
142
|
+
|
|
143
|
+
## Installation
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
pnpm add @objectstack/metadata
|
|
147
|
+
```
|
|
10
148
|
|
|
11
149
|
## Usage
|
|
12
150
|
|
|
13
|
-
|
|
151
|
+
### Basic (Browser-Compatible)
|
|
152
|
+
|
|
153
|
+
```typescript
|
|
154
|
+
import { MetadataManager, MemoryLoader } from '@objectstack/metadata';
|
|
155
|
+
|
|
156
|
+
const manager = new MetadataManager({
|
|
157
|
+
formats: ['json'],
|
|
158
|
+
loaders: [new MemoryLoader()],
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
// Register metadata
|
|
162
|
+
await manager.register('object', 'account', { name: 'account', label: 'Account', fields: {} });
|
|
163
|
+
|
|
164
|
+
// Retrieve
|
|
165
|
+
const obj = await manager.get('object', 'account');
|
|
166
|
+
|
|
167
|
+
// Query
|
|
168
|
+
const result = await manager.query({ types: ['object'], search: 'account' });
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### Node.js (with Filesystem)
|
|
172
|
+
|
|
173
|
+
```typescript
|
|
174
|
+
import { NodeMetadataManager, MetadataPlugin } from '@objectstack/metadata/node';
|
|
175
|
+
|
|
176
|
+
const manager = new NodeMetadataManager({
|
|
177
|
+
rootDir: './src',
|
|
178
|
+
formats: ['typescript', 'json', 'yaml'],
|
|
179
|
+
watch: true,
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
// Load all objects from filesystem
|
|
183
|
+
const objects = await manager.loadMany('object');
|
|
184
|
+
|
|
185
|
+
// Watch for changes
|
|
186
|
+
manager.watchService('object', (event) => {
|
|
187
|
+
console.log(`Object ${event.name} was ${event.type}`);
|
|
188
|
+
});
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
### With Kernel Plugin
|
|
192
|
+
|
|
193
|
+
```typescript
|
|
194
|
+
import { MetadataPlugin } from '@objectstack/metadata/node';
|
|
195
|
+
|
|
196
|
+
const plugin = MetadataPlugin({
|
|
197
|
+
rootDir: './src',
|
|
198
|
+
watch: process.env.NODE_ENV === 'development',
|
|
199
|
+
});
|
|
200
|
+
// Register with ObjectStack kernel
|
|
201
|
+
kernel.use(plugin);
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
## Package Structure
|
|
205
|
+
|
|
206
|
+
```
|
|
207
|
+
packages/metadata/
|
|
208
|
+
├── src/
|
|
209
|
+
│ ├── index.ts # Main exports (browser-compatible)
|
|
210
|
+
│ ├── node.ts # Node.js exports (filesystem, watching)
|
|
211
|
+
│ ├── metadata-manager.ts # MetadataManager (IMetadataService impl)
|
|
212
|
+
│ ├── node-metadata-manager.ts # NodeMetadataManager (+ file watching)
|
|
213
|
+
│ ├── plugin.ts # MetadataPlugin (kernel integration)
|
|
214
|
+
│ ├── loaders/
|
|
215
|
+
│ │ ├── loader-interface.ts # MetadataLoader contract
|
|
216
|
+
│ │ ├── filesystem-loader.ts # File I/O with glob, cache, ETag
|
|
217
|
+
│ │ ├── memory-loader.ts # In-memory store (tests/overrides)
|
|
218
|
+
│ │ └── remote-loader.ts # HTTP API loader with auth
|
|
219
|
+
│ ├── serializers/
|
|
220
|
+
│ │ ├── serializer-interface.ts # MetadataSerializer contract
|
|
221
|
+
│ │ ├── json-serializer.ts # JSON format
|
|
222
|
+
│ │ ├── yaml-serializer.ts # YAML format
|
|
223
|
+
│ │ └── typescript-serializer.ts # TS/JS module format
|
|
224
|
+
│ └── migration/
|
|
225
|
+
│ ├── index.ts # Barrel export
|
|
226
|
+
│ └── executor.ts # ChangeSet executor (DDL operations)
|
|
227
|
+
├── package.json
|
|
228
|
+
├── tsconfig.json
|
|
229
|
+
├── vitest.config.ts
|
|
230
|
+
├── README.md # This file
|
|
231
|
+
└── ROADMAP.md # Development roadmap
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
## Related Packages
|
|
235
|
+
|
|
236
|
+
| Package | Relationship |
|
|
237
|
+
|:------------------------|:-------------------------------------------------|
|
|
238
|
+
| `@objectstack/spec` | Protocol definitions (schemas, contracts, types) |
|
|
239
|
+
| `@objectstack/core` | Logger, service registry, kernel utilities |
|
|
240
|
+
| `@objectstack/runtime` | Uses this package to bootstrap metadata |
|
|
241
|
+
| `apps/studio` | Visual metadata editor (consumes IMetadataService)|
|
|
242
|
+
|
|
243
|
+
## License
|
|
244
|
+
|
|
245
|
+
Apache-2.0 — see [LICENSE](../../LICENSE) for details.
|
package/ROADMAP.md
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
# @objectstack/metadata — Roadmap
|
|
2
|
+
|
|
3
|
+
> Development roadmap for the ObjectStack Metadata Service.
|
|
4
|
+
|
|
5
|
+
## Current Status (v3.0)
|
|
6
|
+
|
|
7
|
+
### ✅ Implemented
|
|
8
|
+
|
|
9
|
+
| Feature | Description |
|
|
10
|
+
|:---------------------------------|:-----------------------------------------------|
|
|
11
|
+
| `MetadataManager` | Core orchestrator implementing `IMetadataService` |
|
|
12
|
+
| `IMetadataService` contract | Full async service interface (30+ methods) |
|
|
13
|
+
| `FilesystemLoader` | File I/O with glob, caching, ETag, atomic writes |
|
|
14
|
+
| `MemoryLoader` | In-memory storage for tests and overrides |
|
|
15
|
+
| `RemoteLoader` | HTTP API loader with Bearer auth |
|
|
16
|
+
| JSON / YAML / TypeScript serializers | Multi-format metadata serialization |
|
|
17
|
+
| Overlay system (in-memory) | Three-scope delta patches (system/platform/user) |
|
|
18
|
+
| Query / Search | Filtering, pagination, sorting by type/scope/state |
|
|
19
|
+
| Bulk operations | `bulkRegister` / `bulkUnregister` with error handling |
|
|
20
|
+
| Import / Export | Portable bundles with conflict resolution |
|
|
21
|
+
| Type registry | 26 built-in metadata types across 6 domains |
|
|
22
|
+
| Dependency tracking | Cross-reference analysis between metadata items |
|
|
23
|
+
| Watch / Subscribe | Real-time change notification via callbacks |
|
|
24
|
+
| File watching (Node.js) | Chokidar-based hot-reload for development |
|
|
25
|
+
| Kernel plugin | `MetadataPlugin` for ObjectStack kernel integration |
|
|
26
|
+
| Migration executor | ChangeSet-based DDL operations |
|
|
27
|
+
| Structural validation | Basic name/type/label validation |
|
|
28
|
+
|
|
29
|
+
### 🟡 Partially Implemented
|
|
30
|
+
|
|
31
|
+
| Feature | Status |
|
|
32
|
+
|:-----------------------|:-------------------------------------------------|
|
|
33
|
+
| Overlay persistence | In-memory only — not persisted to database yet |
|
|
34
|
+
| Migration executor | `modify_field` and `rename_object` not complete |
|
|
35
|
+
| Schema-level validation | Basic structural checks only — no Zod schema dispatch |
|
|
36
|
+
|
|
37
|
+
---
|
|
38
|
+
|
|
39
|
+
## Phase 1: DatabaseLoader — Datasource-Backed Persistence 🔴
|
|
40
|
+
|
|
41
|
+
**Goal**: Enable metadata read/write via any configured `IDataDriver`, so that platform-scope and user-scope metadata can be stored in a database.
|
|
42
|
+
|
|
43
|
+
**Background**: The spec already defines `MetadataManagerConfig.datasource` (referencing a `DatasourceSchema.name`) and `MetadataRecordSchema` (the DB persistence envelope in `metadata-persistence.zod.ts`). The missing piece is the `DatabaseLoader` that bridges `IMetadataService` ↔ `IDataDriver`.
|
|
44
|
+
|
|
45
|
+
### Tasks
|
|
46
|
+
|
|
47
|
+
- [ ] **Implement `DatabaseLoader`** (`src/loaders/database-loader.ts`)
|
|
48
|
+
- Implement `MetadataLoader` interface with protocol `datasource:`
|
|
49
|
+
- Accept an `IDataDriver` instance (injected at initialization)
|
|
50
|
+
- Read/write to the `sys_metadata` table (configurable via `MetadataManagerConfig.tableName`)
|
|
51
|
+
- Map metadata operations to `IDataDriver` CRUD methods (`find`, `findOne`, `create`, `update`, `delete`)
|
|
52
|
+
- Serialize metadata payload to the `MetadataRecordSchema` envelope
|
|
53
|
+
- Support multi-tenant isolation via `tenantId` filter
|
|
54
|
+
- Support optimistic concurrency via `version` field
|
|
55
|
+
- Support `scope` filtering (system/platform/user)
|
|
56
|
+
- Implement `list()` with type filtering and pagination
|
|
57
|
+
- Implement `exists()` and `stat()` via driver queries
|
|
58
|
+
- Implement `save()` with upsert semantics (create or update)
|
|
59
|
+
- Declare capabilities: `{ read: true, write: true, watch: false, list: true }`
|
|
60
|
+
|
|
61
|
+
- [ ] **Integrate DatabaseLoader into MetadataManager**
|
|
62
|
+
- Auto-configure `DatabaseLoader` when `config.datasource` is set
|
|
63
|
+
- Resolve datasource → `IDataDriver` via kernel service registry (`driver.{name}`)
|
|
64
|
+
- Implement fallback strategy: if DB unavailable, fall back to filesystem or memory per config
|
|
65
|
+
- Loader priority: DatabaseLoader for platform/user scope, FilesystemLoader for system scope
|
|
66
|
+
|
|
67
|
+
- [ ] **Schema bootstrapping**
|
|
68
|
+
- Auto-create `sys_metadata` table on first use via `ISchemaDriver.createCollection()`
|
|
69
|
+
- Define column schema: `id`, `name`, `type`, `namespace`, `scope`, `metadata` (JSON), `state`, `version`, `tenant_id`, audit fields
|
|
70
|
+
- Support schema migration for future column additions
|
|
71
|
+
|
|
72
|
+
- [ ] **Tests**
|
|
73
|
+
- Unit tests with `MemoryLoader` as mock driver
|
|
74
|
+
- Integration test pattern for DatabaseLoader ↔ IDataDriver
|
|
75
|
+
- Fallback behavior tests (datasource unavailable → filesystem)
|
|
76
|
+
|
|
77
|
+
### Spec Dependencies (Already Defined)
|
|
78
|
+
|
|
79
|
+
| Spec | What It Provides |
|
|
80
|
+
|:----------------------------------|:-----------------------------------------|
|
|
81
|
+
| `MetadataManagerConfigSchema` | `datasource`, `tableName`, `fallback` fields |
|
|
82
|
+
| `MetadataRecordSchema` | DB record envelope with scope, state, version |
|
|
83
|
+
| `MetadataLoaderContractSchema` | Protocol `datasource:` declaration |
|
|
84
|
+
| `IDataDriver` | `find`, `findOne`, `create`, `update`, `delete` |
|
|
85
|
+
| `ISchemaDriver` | `createCollection`, `addColumn` for DDL |
|
|
86
|
+
| `DatasourceSchema` | Connection config with pool, SSL, retry |
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
## Phase 2: Overlay Persistence & UI Metadata Support 🔴
|
|
91
|
+
|
|
92
|
+
**Goal**: Persist overlay customizations to the database so that admin and user customizations survive restarts, and expose APIs that the Studio UI can consume.
|
|
93
|
+
|
|
94
|
+
### Tasks
|
|
95
|
+
|
|
96
|
+
- [ ] **Persist overlays to database**
|
|
97
|
+
- Store overlays as `MetadataRecord` entries with `scope: 'platform'` or `scope: 'user'`
|
|
98
|
+
- Use `extends` field to reference the base system metadata
|
|
99
|
+
- Use `strategy` field ('merge' or 'replace') to control overlay application
|
|
100
|
+
- Add `managedBy` tracking ('package', 'platform', 'user')
|
|
101
|
+
|
|
102
|
+
- [ ] **Implement `getEffective()` with database-backed resolution**
|
|
103
|
+
- Load base (system, from filesystem) → merge platform overlay (from DB) → merge user overlay (from DB)
|
|
104
|
+
- Cache effective results with invalidation on overlay changes
|
|
105
|
+
- Support conflict detection when base metadata is upgraded
|
|
106
|
+
|
|
107
|
+
- [ ] **REST API for metadata CRUD**
|
|
108
|
+
- `GET /api/metadata/:type` — list metadata items by type
|
|
109
|
+
- `GET /api/metadata/:type/:name` — get metadata item
|
|
110
|
+
- `GET /api/metadata/:type/:name/effective` — get merged effective metadata
|
|
111
|
+
- `PUT /api/metadata/:type/:name` — create/update metadata (platform scope)
|
|
112
|
+
- `DELETE /api/metadata/:type/:name` — remove metadata
|
|
113
|
+
- `GET /api/metadata/:type/:name/overlays` — list overlays
|
|
114
|
+
- `PUT /api/metadata/:type/:name/overlays/:scope` — save overlay
|
|
115
|
+
- `POST /api/metadata/query` — query with filters, pagination
|
|
116
|
+
- `POST /api/metadata/import` / `GET /api/metadata/export` — bulk operations
|
|
117
|
+
|
|
118
|
+
- [ ] **Permission integration**
|
|
119
|
+
- Scope-based access control: system (read-only), platform (admin), user (self)
|
|
120
|
+
- Integrate with `IAuthService` for permission checks
|
|
121
|
+
- Validate `owner` field for user-scope metadata
|
|
122
|
+
|
|
123
|
+
- [ ] **Watch / Events for database changes**
|
|
124
|
+
- Implement polling-based change detection for DatabaseLoader
|
|
125
|
+
- Emit `MetadataWatchEvent` when database records change
|
|
126
|
+
- Support webhook notifications for external consumers
|
|
127
|
+
|
|
128
|
+
---
|
|
129
|
+
|
|
130
|
+
## Phase 3: Schema Validation & Zod Dispatch 🔴
|
|
131
|
+
|
|
132
|
+
**Goal**: Full schema validation by dispatching to the correct Zod schema based on metadata type.
|
|
133
|
+
|
|
134
|
+
### Tasks
|
|
135
|
+
|
|
136
|
+
- [ ] **Zod schema registry**
|
|
137
|
+
- Map metadata type → Zod schema (e.g., `object` → `ObjectSchema`, `view` → `ViewSchema`)
|
|
138
|
+
- Register schemas from `@objectstack/spec` automatically
|
|
139
|
+
- Support plugin-contributed custom type schemas
|
|
140
|
+
|
|
141
|
+
- [ ] **Enhanced `validate()` method**
|
|
142
|
+
- Dispatch to the correct Zod schema per metadata type
|
|
143
|
+
- Return structured errors with path, message, expected/received
|
|
144
|
+
- Support `strict` mode (reject unknown fields) and `lenient` mode (warn only)
|
|
145
|
+
- Validate cross-references (e.g., view references a valid object)
|
|
146
|
+
|
|
147
|
+
- [ ] **Validation on write**
|
|
148
|
+
- Optionally validate metadata on `register()` and `save()`
|
|
149
|
+
- Configurable via `MetadataManagerConfig.validation.strict`
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
## Phase 4: Advanced Features 🔴
|
|
154
|
+
|
|
155
|
+
### 4a. Metadata Versioning & History
|
|
156
|
+
|
|
157
|
+
- [ ] Track metadata change history in the database
|
|
158
|
+
- [ ] Support `version` field with auto-increment on save
|
|
159
|
+
- [ ] Implement `getHistory(type, name)` to retrieve version timeline
|
|
160
|
+
- [ ] Implement `rollback(type, name, version)` to restore a previous version
|
|
161
|
+
- [ ] Add `checksum` field for change detection
|
|
162
|
+
|
|
163
|
+
### 4b. Package Upgrade & Three-Way Merge
|
|
164
|
+
|
|
165
|
+
- [ ] Implement three-way merge when upgrading package-delivered metadata
|
|
166
|
+
- Base: previous package version
|
|
167
|
+
- Ours: current platform customizations (overlays)
|
|
168
|
+
- Theirs: new package version
|
|
169
|
+
- [ ] Merge conflict detection and resolution UI support
|
|
170
|
+
- [ ] Leverage `MergeStrategyConfigSchema` from spec (keep-custom, accept-incoming, three-way-merge)
|
|
171
|
+
|
|
172
|
+
### 4c. Metadata Sync & Distribution
|
|
173
|
+
|
|
174
|
+
- [ ] `pull` — download metadata from a remote ObjectStack instance
|
|
175
|
+
- [ ] `push` — upload local metadata to a remote instance
|
|
176
|
+
- [ ] Selective sync by type, namespace, or package
|
|
177
|
+
- [ ] Conflict detection across instances
|
|
178
|
+
|
|
179
|
+
### 4d. S3/Cloud Loader
|
|
180
|
+
|
|
181
|
+
- [ ] Implement `S3Loader` for cloud-native metadata storage
|
|
182
|
+
- [ ] Support `s3:` protocol in `MetadataLoaderContract`
|
|
183
|
+
- [ ] Integrate with object storage for large metadata bundles
|
|
184
|
+
|
|
185
|
+
---
|
|
186
|
+
|
|
187
|
+
## Phase 5: Performance & Production Readiness 🔴
|
|
188
|
+
|
|
189
|
+
- [ ] **Caching layer**
|
|
190
|
+
- Implement TTL-based cache with configurable `maxSize`
|
|
191
|
+
- Cache invalidation on write/overlay changes
|
|
192
|
+
- Support distributed cache (Redis) for multi-instance deployments
|
|
193
|
+
- [ ] **Connection pooling**
|
|
194
|
+
- Reuse `IDataDriver` connections efficiently
|
|
195
|
+
- Handle connection failures gracefully with retry policy
|
|
196
|
+
- [ ] **Batch loading optimization**
|
|
197
|
+
- Load multiple types in a single query where possible
|
|
198
|
+
- Implement DataLoader-style batching for N+1 prevention
|
|
199
|
+
- [ ] **Metrics & observability**
|
|
200
|
+
- Track load/save latency, cache hit rates, loader usage
|
|
201
|
+
- Expose metrics via kernel observability contract
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
## Milestone Summary
|
|
206
|
+
|
|
207
|
+
| Phase | Target | Description | Status |
|
|
208
|
+
|:------|:--------|:----------------------------------------------|:-------|
|
|
209
|
+
| — | v3.0 | Core MetadataManager, Filesystem/Memory/Remote | ✅ Done |
|
|
210
|
+
| 1 | v3.1 | DatabaseLoader — datasource-backed persistence | 🔴 Planned |
|
|
211
|
+
| 2 | v3.2 | Overlay persistence, REST API, UI support | 🔴 Planned |
|
|
212
|
+
| 3 | v3.3 | Schema validation & Zod dispatch | 🔴 Planned |
|
|
213
|
+
| 4 | v4.0 | Versioning, merge, sync, S3 loader | 🔴 Planned |
|
|
214
|
+
| 5 | v4.1 | Caching, pooling, observability | 🔴 Planned |
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
218
|
+
## Related Documents
|
|
219
|
+
|
|
220
|
+
- [Root ROADMAP](../../ROADMAP.md) — Full platform evolution (v3.0 → v5.0)
|
|
221
|
+
- [Studio ROADMAP](../../apps/studio/ROADMAP.md) — Visual IDE development phases
|
|
222
|
+
- [Metadata Service Protocol](../../content/docs/protocol/objectos/metadata-service.mdx) — Detailed protocol documentation
|
|
223
|
+
- [DX ROADMAP](../../docs/DX_ROADMAP.md) — Developer experience improvements
|
package/dist/index.d.mts
CHANGED
|
@@ -195,6 +195,22 @@ declare class MetadataManager implements IMetadataService {
|
|
|
195
195
|
* Convenience: list all object definitions
|
|
196
196
|
*/
|
|
197
197
|
listObjects(): Promise<unknown[]>;
|
|
198
|
+
/**
|
|
199
|
+
* Convenience: get a view definition by name
|
|
200
|
+
*/
|
|
201
|
+
getView(name: string): Promise<unknown | undefined>;
|
|
202
|
+
/**
|
|
203
|
+
* Convenience: list view definitions, optionally filtered by object
|
|
204
|
+
*/
|
|
205
|
+
listViews(object?: string): Promise<unknown[]>;
|
|
206
|
+
/**
|
|
207
|
+
* Convenience: get a dashboard definition by name
|
|
208
|
+
*/
|
|
209
|
+
getDashboard(name: string): Promise<unknown | undefined>;
|
|
210
|
+
/**
|
|
211
|
+
* Convenience: list all dashboard definitions
|
|
212
|
+
*/
|
|
213
|
+
listDashboards(): Promise<unknown[]>;
|
|
198
214
|
/**
|
|
199
215
|
* Unregister all metadata items from a specific package
|
|
200
216
|
*/
|
|
@@ -238,7 +254,12 @@ declare class MetadataManager implements IMetadataService {
|
|
|
238
254
|
* Get the effective (merged) metadata after applying all overlays.
|
|
239
255
|
* Resolution order: system ← merge(platform) ← merge(user)
|
|
240
256
|
*/
|
|
241
|
-
getEffective(type: string, name: string
|
|
257
|
+
getEffective(type: string, name: string, context?: {
|
|
258
|
+
userId?: string;
|
|
259
|
+
tenantId?: string;
|
|
260
|
+
roles?: string[];
|
|
261
|
+
permissions?: string[];
|
|
262
|
+
}): Promise<unknown | undefined>;
|
|
242
263
|
/**
|
|
243
264
|
* Watch for metadata changes (IMetadataService contract).
|
|
244
265
|
* Returns a handle for unsubscribing.
|
package/dist/index.d.ts
CHANGED
|
@@ -195,6 +195,22 @@ declare class MetadataManager implements IMetadataService {
|
|
|
195
195
|
* Convenience: list all object definitions
|
|
196
196
|
*/
|
|
197
197
|
listObjects(): Promise<unknown[]>;
|
|
198
|
+
/**
|
|
199
|
+
* Convenience: get a view definition by name
|
|
200
|
+
*/
|
|
201
|
+
getView(name: string): Promise<unknown | undefined>;
|
|
202
|
+
/**
|
|
203
|
+
* Convenience: list view definitions, optionally filtered by object
|
|
204
|
+
*/
|
|
205
|
+
listViews(object?: string): Promise<unknown[]>;
|
|
206
|
+
/**
|
|
207
|
+
* Convenience: get a dashboard definition by name
|
|
208
|
+
*/
|
|
209
|
+
getDashboard(name: string): Promise<unknown | undefined>;
|
|
210
|
+
/**
|
|
211
|
+
* Convenience: list all dashboard definitions
|
|
212
|
+
*/
|
|
213
|
+
listDashboards(): Promise<unknown[]>;
|
|
198
214
|
/**
|
|
199
215
|
* Unregister all metadata items from a specific package
|
|
200
216
|
*/
|
|
@@ -238,7 +254,12 @@ declare class MetadataManager implements IMetadataService {
|
|
|
238
254
|
* Get the effective (merged) metadata after applying all overlays.
|
|
239
255
|
* Resolution order: system ← merge(platform) ← merge(user)
|
|
240
256
|
*/
|
|
241
|
-
getEffective(type: string, name: string
|
|
257
|
+
getEffective(type: string, name: string, context?: {
|
|
258
|
+
userId?: string;
|
|
259
|
+
tenantId?: string;
|
|
260
|
+
roles?: string[];
|
|
261
|
+
permissions?: string[];
|
|
262
|
+
}): Promise<unknown | undefined>;
|
|
242
263
|
/**
|
|
243
264
|
* Watch for metadata changes (IMetadataService contract).
|
|
244
265
|
* Returns a handle for unsubscribing.
|