@biorate/migrations 2.2.4 → 2.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +213 -9
- package/package.json +20 -20
package/README.md
CHANGED
|
@@ -1,14 +1,27 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @biorate/migrations
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Database and infrastructure migration framework — manages schema and data migrations across Sequelize, MongoDB, Minio, Kafka, Clickhouse, AMQP, and Schema Registry.
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
- Sequelize
|
|
5
|
+
## Features
|
|
7
6
|
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
- **Multi-engine** — RDBMS (Sequelize), MongoDB, Minio/S3, Kafka Admin, Clickhouse, AMQP/RabbitMQ, Schema Registry.
|
|
8
|
+
- **Tracking** — tracks completed migrations per engine (Sequelize/MongoDB/Clickhouse store in a table/collection).
|
|
9
|
+
- **JavaScript/SQL** — JS scripts for complex logic, SQL files for schema DDL.
|
|
10
|
+
- **DI-orchestrated** — all connectors managed via `@biorate/inversion` container.
|
|
11
|
+
- **Side-effect import** — `@biorate/migrations` auto-runs when loaded (`node -r @biorate/migrations`).
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pnpm add @biorate/migrations
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Requires the underlying engine packages (`@biorate/sequelize`, `@biorate/mongodb`, `@biorate/minio`, `@biorate/kafkajs`, `@biorate/clickhouse`, `@biorate/amqp`, `@biorate/schema-registry`, `@biorate/vault`).
|
|
20
|
+
|
|
21
|
+
## Quick start
|
|
10
22
|
|
|
11
23
|
```json
|
|
24
|
+
// package.json
|
|
12
25
|
{
|
|
13
26
|
"scripts": {
|
|
14
27
|
"migrations": "node -r @biorate/migrations"
|
|
@@ -16,8 +29,199 @@ Merge this into your package.json file on server:
|
|
|
16
29
|
}
|
|
17
30
|
```
|
|
18
31
|
|
|
19
|
-
|
|
20
|
-
|
|
32
|
+
Run with environment config:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
MIGRATIONS_CONFIG=./config.ts npm run migrations
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Module reference
|
|
39
|
+
|
|
40
|
+
### `Root` — Orchestrator
|
|
41
|
+
|
|
42
|
+
`Root` is the main DI class that initialises all migration connectors. Binds every connector into the `@biorate/inversion` container and calls `process.exit(0)` on completion.
|
|
43
|
+
|
|
44
|
+
```ts
|
|
45
|
+
import { Root } from '@biorate/migrations';
|
|
46
|
+
|
|
47
|
+
// Root is auto-instantiated on package import.
|
|
48
|
+
// Manually:
|
|
49
|
+
import { container } from '@biorate/inversion';
|
|
50
|
+
const root = container.get<Root>(Root);
|
|
51
|
+
await root.$run();
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
### `Migration` — Abstract base
|
|
55
|
+
|
|
56
|
+
```ts
|
|
57
|
+
import { Migration } from '@biorate/migrations';
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Abstract base class for all migration types. Provides:
|
|
61
|
+
|
|
62
|
+
| Member | Signature | Description |
|
|
63
|
+
|--------------------|------------------------------------------------|-----------------------------------------------|
|
|
64
|
+
| `config` | `protected config: IConfig` | Injected config service. |
|
|
65
|
+
| `connector` | `protected abstract connector: IConnector` | Engine connector (must be overridden). |
|
|
66
|
+
| `type` | `protected get type(): string` | `this.constructor.name.toLowerCase()`. |
|
|
67
|
+
| `scan(...args)` | `async scan(...paths): Promise<string[]>` | Scans migration directory, returns file paths.|
|
|
68
|
+
| `path(...args)` | `path(...segments): string` | Builds path: `cwd/migrations/{type}/{...}`. |
|
|
69
|
+
| `log(...args)` | `log(...msg): void` | Console logging with type prefix. |
|
|
70
|
+
| `forEach(namespace, callback)` | `async forEach<T, C>(ns, fn)` | Iterates config entries and connections. |
|
|
71
|
+
| `forEachPath(paths, callback)` | `async forEachPath(paths, fn)` | Iterates files in a path list. |
|
|
72
|
+
| `process()` | `protected abstract process(): Promise<void>` | Override with migration logic. |
|
|
73
|
+
| `initialize()` | `@init() protected async initialize()` | Calls `this.process()` on lifecycle init. |
|
|
74
|
+
|
|
75
|
+
### `Sequelize` — RDBMS migrations
|
|
76
|
+
|
|
77
|
+
```ts
|
|
78
|
+
import { Sequelize } from '@biorate/migrations';
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Reads SQL files (e.g. `00001_create_users.sql`) from `migrations/sequelize/`, executes them in transactions, and tracks completion in a `migrations` table (`name CHAR PRIMARY KEY`).
|
|
82
|
+
|
|
83
|
+
**Config namespace:** `Sequelize` (from `@biorate/sequelize` base config).
|
|
84
|
+
|
|
85
|
+
**Directory:** `./migrations/sequelize/`
|
|
86
|
+
|
|
87
|
+
### `Mongodb` — MongoDB migrations
|
|
88
|
+
|
|
89
|
+
```ts
|
|
90
|
+
import { Mongodb } from '@biorate/migrations';
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Loads and executes JS scripts (`module.exports = async (connection, config, globalConfig) => {...}`) from `migrations/mongodb/`. Tracks completed scripts via documents in a `migrations` collection.
|
|
94
|
+
|
|
95
|
+
**Config namespace:** `MongoDB` (from `@biorate/mongodb` base config).
|
|
96
|
+
|
|
97
|
+
**Directory:** `./migrations/mongodb/`
|
|
98
|
+
|
|
99
|
+
### `Minio` — Minio/S3 migrations
|
|
100
|
+
|
|
101
|
+
```ts
|
|
102
|
+
import { Minio } from '@biorate/migrations';
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Loads and executes JS scripts (`module.exports = async (connection, config, globalConfig) => {...}`) from `migrations/minio/`. Tracks completed scripts via marker objects in a migrations bucket.
|
|
106
|
+
|
|
107
|
+
**Config namespace:** `Minio` (from `@biorate/minio` base config).
|
|
108
|
+
|
|
109
|
+
**Directory:** `./migrations/minio/`
|
|
110
|
+
|
|
111
|
+
### `Kafka` — Kafka Admin migrations
|
|
112
|
+
|
|
113
|
+
```ts
|
|
114
|
+
import { Kafka } from '@biorate/migrations';
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Loads and executes JS scripts (`module.exports = async (connection, config, globalConfig) => {...}`) from `migrations/kafka/`. Creates/alters topics, configs, etc. **No tracking** (every run re-applies).
|
|
118
|
+
|
|
119
|
+
**Config namespace:** `KafkaJSAdmin` (from `@biorate/kafkajs` `KafkaJSAdminConnector`).
|
|
120
|
+
|
|
121
|
+
**Directory:** `./migrations/kafka/`
|
|
122
|
+
|
|
123
|
+
### `Clickhouse` — Clickhouse migrations
|
|
124
|
+
|
|
125
|
+
```ts
|
|
126
|
+
import { Clickhouse } from '@biorate/migrations';
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Reads SQL files from `migrations/clickhouse/`, executes them via `connection.command()`, and tracks completion in a `migrations` MergeTree table with `name String` primary key.
|
|
130
|
+
|
|
131
|
+
**Config namespace:** `Clickhouse` (from `@biorate/clickhouse` base config).
|
|
132
|
+
|
|
133
|
+
**Directory:** `./migrations/clickhouse/`
|
|
134
|
+
|
|
135
|
+
### `Amqp` — AMQP/RabbitMQ migrations
|
|
136
|
+
|
|
137
|
+
```ts
|
|
138
|
+
import { Amqp } from '@biorate/migrations';
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Loads and executes JS scripts (`module.exports = async (channel, connection, config, globalConfig) => {...}`) from `migrations/amqp/`. The script receives the AMQP channel for declaring exchanges/queues/bindings. **No tracking**.
|
|
142
|
+
|
|
143
|
+
**Config namespace:** `Amqp` (from `@biorate/amqp` base config). Additional config: `migrations.Amqp.<name>.amqpChannelOptions`.
|
|
144
|
+
|
|
145
|
+
**Directory:** `./migrations/amqp/`
|
|
146
|
+
|
|
147
|
+
### `SchemaRegistry` — Schema Registry migrations
|
|
148
|
+
|
|
149
|
+
```ts
|
|
150
|
+
import { SchemaRegistry } from '@biorate/migrations';
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Reads `.avsc.json` / `.json` files named `00001_subjectname.avsc.json` from `migrations/schema-registry/`. Each file is registered as a new subject or a new version of an existing subject on the Schema Registry.
|
|
154
|
+
|
|
155
|
+
| Feature | Description |
|
|
156
|
+
|----------------------|--------------------------------------------------------|
|
|
157
|
+
| File pattern | `00001_{subject}.avsc.json` |
|
|
158
|
+
| Compatibility | Set via config, defaults to `FORWARD` |
|
|
159
|
+
| On conflict | Registers a new version for the subject |
|
|
160
|
+
| Errors | `SchemaRegistryWrongFileNameError` if pattern mismatch |
|
|
161
|
+
|
|
162
|
+
**Config namespace:** `SchemaRegistry` (from `@biorate/schema-registry` base config).
|
|
163
|
+
|
|
164
|
+
**Directory:** `./migrations/schema-registry/`
|
|
165
|
+
|
|
166
|
+
### Errors
|
|
167
|
+
|
|
168
|
+
| Error | Context |
|
|
169
|
+
|------------------------------------|--------------------------------------|
|
|
170
|
+
| `SchemaRegistryWrongFileNameError` | Schema Registry migration file name does not match `00001_name.avsc.json` pattern. |
|
|
171
|
+
|
|
172
|
+
## Config structure
|
|
173
|
+
|
|
174
|
+
Your config file (referenced by `MIGRATIONS_CONFIG` env var) must define the connection configs for all engines you use, for example:
|
|
175
|
+
|
|
176
|
+
```ts
|
|
177
|
+
export default {
|
|
178
|
+
Sequelize: [{ name: 'primary', options: { dialect: 'postgres', ... } }],
|
|
179
|
+
MongoDB: [{ name: 'primary', host: 'mongodb://...', options: { dbName: 'test' } }],
|
|
180
|
+
Minio: [{ name: 'primary', options: { endPoint: 'localhost', ... } }],
|
|
181
|
+
KafkaJSAdmin: [{ name: 'primary', options: { 'metadata.broker.list': 'localhost:9092' } }],
|
|
182
|
+
Clickhouse: [{ name: 'primary', options: { host: 'localhost', ... } }],
|
|
183
|
+
Amqp: [{ name: 'primary', options: { hostname: 'localhost', ... } }],
|
|
184
|
+
SchemaRegistry: [{ name: 'primary', options: { baseURL: 'http://localhost:8081' } }],
|
|
185
|
+
};
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
## Architecture
|
|
189
|
+
|
|
190
|
+
```
|
|
191
|
+
@biorate/migrations
|
|
192
|
+
|
|
193
|
+
MIGRATIONS_CONFIG / MIGRATIONS_ROOT env → requireCjs
|
|
194
|
+
│
|
|
195
|
+
└── Root (DI orchestrator)
|
|
196
|
+
├── container.bind(SequelizeConnector, MinioConnector, MongoDBConnector, ...)
|
|
197
|
+
│
|
|
198
|
+
└── @init() initialize()
|
|
199
|
+
├── Sequelize.process()
|
|
200
|
+
│ ├── forEach config → connection.query(sqlFile)
|
|
201
|
+
│ └── INSERT INTO migrations (name) VALUES (...)
|
|
202
|
+
│
|
|
203
|
+
├── Mongodb.process()
|
|
204
|
+
│ ├── forEach config → require(jsFile)(connection, config, globalConfig)
|
|
205
|
+
│ └── db.migrations.insertOne({ _id: name })
|
|
206
|
+
│
|
|
207
|
+
├── Minio.process()
|
|
208
|
+
│ ├── forEach config → require(jsFile)(connection, config, globalConfig)
|
|
209
|
+
│ └── client.putObject(migrationBucket, name, ...)
|
|
210
|
+
│
|
|
211
|
+
├── Kafka.process()
|
|
212
|
+
│ └── forEach config → require(jsFile)(adminClient, config, globalConfig)
|
|
213
|
+
│
|
|
214
|
+
├── Clickhouse.process()
|
|
215
|
+
│ ├── forEach config → connection.command(sqlFile)
|
|
216
|
+
│ └── INSERT INTO migrations (name) VALUES (...)
|
|
217
|
+
│
|
|
218
|
+
├── Amqp.process()
|
|
219
|
+
│ └── forEach config → require(jsFile)(channel, connection, config, globalConfig)
|
|
220
|
+
│
|
|
221
|
+
└── SchemaRegistry.process()
|
|
222
|
+
└── forEach config → POST /subjects/{subject}/versions
|
|
223
|
+
└── SchemaRegistryWrongFileNameError check
|
|
224
|
+
```
|
|
21
225
|
|
|
22
226
|
### Learn
|
|
23
227
|
|
|
@@ -27,7 +231,7 @@ Merge this into your package.json file on server:
|
|
|
27
231
|
|
|
28
232
|
See the [CHANGELOG](https://github.com/biorate/core/blob/master/packages/%40biorate/migrations/CHANGELOG.md)
|
|
29
233
|
|
|
30
|
-
|
|
234
|
+
## License
|
|
31
235
|
|
|
32
236
|
[MIT](https://github.com/biorate/core/blob/master/packages/%40biorate/migrations/LICENSE)
|
|
33
237
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@biorate/migrations",
|
|
3
|
-
"version": "2.2.
|
|
3
|
+
"version": "2.2.5",
|
|
4
4
|
"description": "Migrations tools",
|
|
5
5
|
"main": "./dist/cjs/index.js",
|
|
6
6
|
"module": "./dist/esm/index.js",
|
|
@@ -47,25 +47,25 @@
|
|
|
47
47
|
],
|
|
48
48
|
"author": "llevkin",
|
|
49
49
|
"license": "MIT",
|
|
50
|
-
"gitHead": "
|
|
50
|
+
"gitHead": "e17a03900999e261a0eb04c6afb09b9535cad952",
|
|
51
51
|
"dependencies": {
|
|
52
|
-
"@biorate/amqp": "^2.1.
|
|
53
|
-
"@biorate/clickhouse": "^2.4.
|
|
54
|
-
"@biorate/config": "^3.2.
|
|
55
|
-
"@biorate/config-loader": "^2.1.
|
|
56
|
-
"@biorate/config-loader-env": "^2.1.
|
|
57
|
-
"@biorate/config-loader-fs": "^2.1.
|
|
58
|
-
"@biorate/config-loader-vault": "^2.1.
|
|
59
|
-
"@biorate/connector": "^3.1.
|
|
60
|
-
"@biorate/errors": "^3.1.
|
|
61
|
-
"@biorate/inversion": "^3.1.
|
|
62
|
-
"@biorate/kafkajs": "^2.2.
|
|
63
|
-
"@biorate/minio": "^2.1.
|
|
64
|
-
"@biorate/mongodb": "^2.1.
|
|
65
|
-
"@biorate/proxy": "^2.1.
|
|
66
|
-
"@biorate/schema-registry": "^3.1.
|
|
67
|
-
"@biorate/sequelize": "^2.2.
|
|
68
|
-
"@biorate/tools": "^3.1.
|
|
69
|
-
"@biorate/vault": "^2.2.
|
|
52
|
+
"@biorate/amqp": "^2.1.2",
|
|
53
|
+
"@biorate/clickhouse": "^2.4.2",
|
|
54
|
+
"@biorate/config": "^3.2.2",
|
|
55
|
+
"@biorate/config-loader": "^2.1.2",
|
|
56
|
+
"@biorate/config-loader-env": "^2.1.2",
|
|
57
|
+
"@biorate/config-loader-fs": "^2.1.2",
|
|
58
|
+
"@biorate/config-loader-vault": "^2.1.2",
|
|
59
|
+
"@biorate/connector": "^3.1.2",
|
|
60
|
+
"@biorate/errors": "^3.1.2",
|
|
61
|
+
"@biorate/inversion": "^3.1.2",
|
|
62
|
+
"@biorate/kafkajs": "^2.2.2",
|
|
63
|
+
"@biorate/minio": "^2.1.2",
|
|
64
|
+
"@biorate/mongodb": "^2.1.2",
|
|
65
|
+
"@biorate/proxy": "^2.1.3",
|
|
66
|
+
"@biorate/schema-registry": "^3.1.4",
|
|
67
|
+
"@biorate/sequelize": "^2.2.2",
|
|
68
|
+
"@biorate/tools": "^3.1.2",
|
|
69
|
+
"@biorate/vault": "^2.2.2"
|
|
70
70
|
}
|
|
71
71
|
}
|