@oliasoft-open-source/node-json-migrator 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintcache +1 -0
- package/.eslintrc.js +28 -0
- package/.idea/codeStyles/Project.xml +65 -0
- package/.idea/codeStyles/codeStyleConfig.xml +5 -0
- package/.idea/inspectionProfiles/Project_Default.xml +6 -0
- package/.idea/jsLibraryMappings.xml +7 -0
- package/.idea/modules.xml +8 -0
- package/.idea/node-postgresql-migrator.iml +16 -0
- package/.idea/vcs.xml +6 -0
- package/.vscode/launch.json +16 -0
- package/LICENSE +21 -0
- package/README.md +169 -0
- package/dist/create/create.js +90 -0
- package/dist/database/database.js +119 -0
- package/dist/executed/executed.js +78 -0
- package/dist/git/git.js +83 -0
- package/dist/glob/glob.js +71 -0
- package/dist/hash/hash.js +22 -0
- package/dist/history/history.js +97 -0
- package/dist/index.js +61 -0
- package/dist/migrate/migrate.js +63 -0
- package/dist/pending/pending.js +68 -0
- package/dist/pipe/pipe.js +41 -0
- package/dist/plan/cached-planned-version.js +17 -0
- package/dist/plan/plan.js +266 -0
- package/dist/plan/plan.schema.json +20 -0
- package/dist/plan/planned-version.js +34 -0
- package/dist/plan/validator.js +254 -0
- package/index.js +1 -0
- package/package.json +78 -0
- package/scripts/send-mattermost-message.sh +21 -0
package/.eslintcache
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
[{"/Users/mark/Dev/node-json-migrator/src/database/database.js":"1"},{"size":2964,"mtime":1665142546460,"results":"2","hashOfConfig":"3"},{"filePath":"4","messages":"5","errorCount":0,"fatalErrorCount":0,"warningCount":0,"fixableErrorCount":0,"fixableWarningCount":0},"juro6x","/Users/mark/Dev/node-json-migrator/src/database/database.js",[]]
|
package/.eslintrc.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
module.exports = {
|
|
2
|
+
env: {
|
|
3
|
+
es2021: true,
|
|
4
|
+
node: true,
|
|
5
|
+
jest: true,
|
|
6
|
+
},
|
|
7
|
+
extends: ['airbnb/base', 'prettier'],
|
|
8
|
+
parser: '@babel/eslint-parser',
|
|
9
|
+
parserOptions: {
|
|
10
|
+
ecmaVersion: 12,
|
|
11
|
+
sourceType: 'module',
|
|
12
|
+
},
|
|
13
|
+
rules: {
|
|
14
|
+
'consistent-return': 0,
|
|
15
|
+
eqeqeq: 0,
|
|
16
|
+
'import/extensions': 0,
|
|
17
|
+
'import/prefer-default-export': 0,
|
|
18
|
+
'one-var': 0,
|
|
19
|
+
'spaced-comment': 0,
|
|
20
|
+
'arrow-body-style': 0,
|
|
21
|
+
'no-else-return': 0,
|
|
22
|
+
'no-return-assign': 0,
|
|
23
|
+
'no-restricted-syntax': 0,
|
|
24
|
+
'no-console': 0,
|
|
25
|
+
'import/no-extraneous-dependencies': 0,
|
|
26
|
+
'no-param-reassign': 0,
|
|
27
|
+
},
|
|
28
|
+
};
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
<component name="ProjectCodeStyleConfiguration">
|
|
2
|
+
<code_scheme name="Project" version="173">
|
|
3
|
+
<option name="OTHER_INDENT_OPTIONS">
|
|
4
|
+
<value>
|
|
5
|
+
<option name="INDENT_SIZE" value="2" />
|
|
6
|
+
<option name="TAB_SIZE" value="2" />
|
|
7
|
+
</value>
|
|
8
|
+
</option>
|
|
9
|
+
<HTMLCodeStyleSettings>
|
|
10
|
+
<option name="HTML_SPACE_INSIDE_EMPTY_TAG" value="true" />
|
|
11
|
+
<option name="HTML_ENFORCE_QUOTES" value="true" />
|
|
12
|
+
</HTMLCodeStyleSettings>
|
|
13
|
+
<JSCodeStyleSettings version="0">
|
|
14
|
+
<option name="FORCE_SEMICOLON_STYLE" value="true" />
|
|
15
|
+
<option name="SPACE_BEFORE_FUNCTION_LEFT_PARENTH" value="false" />
|
|
16
|
+
<option name="FORCE_QUOTE_STYlE" value="true" />
|
|
17
|
+
<option name="ENFORCE_TRAILING_COMMA" value="Remove" />
|
|
18
|
+
<option name="SPACES_WITHIN_OBJECT_LITERAL_BRACES" value="true" />
|
|
19
|
+
<option name="SPACES_WITHIN_IMPORTS" value="true" />
|
|
20
|
+
</JSCodeStyleSettings>
|
|
21
|
+
<TypeScriptCodeStyleSettings version="0">
|
|
22
|
+
<option name="FORCE_SEMICOLON_STYLE" value="true" />
|
|
23
|
+
<option name="SPACE_BEFORE_FUNCTION_LEFT_PARENTH" value="false" />
|
|
24
|
+
<option name="FORCE_QUOTE_STYlE" value="true" />
|
|
25
|
+
<option name="ENFORCE_TRAILING_COMMA" value="Remove" />
|
|
26
|
+
<option name="SPACES_WITHIN_OBJECT_LITERAL_BRACES" value="true" />
|
|
27
|
+
<option name="SPACES_WITHIN_IMPORTS" value="true" />
|
|
28
|
+
</TypeScriptCodeStyleSettings>
|
|
29
|
+
<VueCodeStyleSettings>
|
|
30
|
+
<option name="INTERPOLATION_NEW_LINE_AFTER_START_DELIMITER" value="false" />
|
|
31
|
+
<option name="INTERPOLATION_NEW_LINE_BEFORE_END_DELIMITER" value="false" />
|
|
32
|
+
</VueCodeStyleSettings>
|
|
33
|
+
<codeStyleSettings language="HTML">
|
|
34
|
+
<option name="SOFT_MARGINS" value="80" />
|
|
35
|
+
<indentOptions>
|
|
36
|
+
<option name="INDENT_SIZE" value="2" />
|
|
37
|
+
<option name="CONTINUATION_INDENT_SIZE" value="2" />
|
|
38
|
+
<option name="TAB_SIZE" value="2" />
|
|
39
|
+
</indentOptions>
|
|
40
|
+
</codeStyleSettings>
|
|
41
|
+
<codeStyleSettings language="JavaScript">
|
|
42
|
+
<option name="RIGHT_MARGIN" value="80" />
|
|
43
|
+
<option name="SOFT_MARGINS" value="80" />
|
|
44
|
+
<indentOptions>
|
|
45
|
+
<option name="INDENT_SIZE" value="2" />
|
|
46
|
+
<option name="CONTINUATION_INDENT_SIZE" value="2" />
|
|
47
|
+
<option name="TAB_SIZE" value="2" />
|
|
48
|
+
</indentOptions>
|
|
49
|
+
</codeStyleSettings>
|
|
50
|
+
<codeStyleSettings language="TypeScript">
|
|
51
|
+
<option name="SOFT_MARGINS" value="80" />
|
|
52
|
+
<indentOptions>
|
|
53
|
+
<option name="INDENT_SIZE" value="2" />
|
|
54
|
+
<option name="CONTINUATION_INDENT_SIZE" value="2" />
|
|
55
|
+
<option name="TAB_SIZE" value="2" />
|
|
56
|
+
</indentOptions>
|
|
57
|
+
</codeStyleSettings>
|
|
58
|
+
<codeStyleSettings language="Vue">
|
|
59
|
+
<option name="SOFT_MARGINS" value="80" />
|
|
60
|
+
<indentOptions>
|
|
61
|
+
<option name="CONTINUATION_INDENT_SIZE" value="2" />
|
|
62
|
+
</indentOptions>
|
|
63
|
+
</codeStyleSettings>
|
|
64
|
+
</code_scheme>
|
|
65
|
+
</component>
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<project version="4">
|
|
3
|
+
<component name="ProjectModuleManager">
|
|
4
|
+
<modules>
|
|
5
|
+
<module fileurl="file://$PROJECT_DIR$/.idea/node-postgresql-migrator.iml" filepath="$PROJECT_DIR$/.idea/node-postgresql-migrator.iml" />
|
|
6
|
+
</modules>
|
|
7
|
+
</component>
|
|
8
|
+
</project>
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<module type="WEB_MODULE" version="4">
|
|
3
|
+
<component name="NewModuleRootManager">
|
|
4
|
+
<content url="file://$MODULE_DIR$">
|
|
5
|
+
<excludeFolder url="file://$MODULE_DIR$/temp" />
|
|
6
|
+
<excludeFolder url="file://$MODULE_DIR$/.tmp" />
|
|
7
|
+
<excludeFolder url="file://$MODULE_DIR$/tmp" />
|
|
8
|
+
<excludeFolder url="file://$MODULE_DIR$/test/coverage" />
|
|
9
|
+
<excludeFolder url="file://$MODULE_DIR$/dist" />
|
|
10
|
+
</content>
|
|
11
|
+
<orderEntry type="inheritedJdk" />
|
|
12
|
+
<orderEntry type="sourceFolder" forTests="false" />
|
|
13
|
+
<orderEntry type="library" name="@types/jest" level="application" />
|
|
14
|
+
<orderEntry type="library" name="@types/lodash" level="application" />
|
|
15
|
+
</component>
|
|
16
|
+
</module>
|
package/.idea/vcs.xml
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
// Use IntelliSense to learn about possible attributes.
|
|
3
|
+
// Hover to view descriptions of existing attributes.
|
|
4
|
+
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
|
5
|
+
"version": "0.2.0",
|
|
6
|
+
"configurations": [
|
|
7
|
+
{
|
|
8
|
+
"name": "Jest Tests",
|
|
9
|
+
"program": "${workspaceRoot}/node_modules/jest/bin/jest.js",
|
|
10
|
+
"args": ["-i"],
|
|
11
|
+
"request": "launch",
|
|
12
|
+
"skipFiles": ["<node_internals>/**", "node_modules/**"],
|
|
13
|
+
"type": "node"
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
}
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020 Oliasoft AS and contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
# Node JSON Migrator
|
|
2
|
+
|
|
3
|
+
## A simple JSON change management tool for Node.js
|
|
4
|
+
|
|
5
|
+
- Lightweight JSON change management tool for `Node.js` applications
|
|
6
|
+
- Written in JavaScript
|
|
7
|
+
- Pre-transpiled (no extra build steps)
|
|
8
|
+
- Not tied to any framework
|
|
9
|
+
- Open source (MIT License)
|
|
10
|
+
|
|
11
|
+
## Features
|
|
12
|
+
|
|
13
|
+
- Versioning of JSON payloads
|
|
14
|
+
- Write change scripts (migrations) in JavaScript
|
|
15
|
+
- Apply changes to JSON payloads using a convenient `migrate()` function
|
|
16
|
+
- Works great in a team setting (multiple developers can track and handle changes)
|
|
17
|
+
- Supports automated change deployment across multiple environments
|
|
18
|
+
- Automatically tracks and applies only new (pending) change scripts
|
|
19
|
+
- Automatically detects and warns when previous scripts have been altered
|
|
20
|
+
- Optionally track version history in a PostgreSQL database (pg-promise interface)
|
|
21
|
+
|
|
22
|
+
# How to use
|
|
23
|
+
|
|
24
|
+
## Installation and setup
|
|
25
|
+
|
|
26
|
+
The database (if used) must already exist and have permissions granted.
|
|
27
|
+
|
|
28
|
+
Install the tool directly from Gitlab (published NPM packages may be available in future):
|
|
29
|
+
|
|
30
|
+
```
|
|
31
|
+
npm install --save git+https://gitlab.com/oliasoft-open-source/node-json-migrator.git
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Defining migrations (change scripts)
|
|
35
|
+
|
|
36
|
+
Use the `createMigration()` function to generate a skeleton migration script, for example:
|
|
37
|
+
|
|
38
|
+
```javascript
|
|
39
|
+
import {createMigration} from 'node-json-migrator';
|
|
40
|
+
|
|
41
|
+
const directory = 'json-migrations';
|
|
42
|
+
const description = 'add-panda';
|
|
43
|
+
await createMigration(directory, description);
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
This will create and populate skeleton files:
|
|
47
|
+
|
|
48
|
+
- `add-panda/add-panda.js` is the skeleton change script (alter this to describe the desired payload change)
|
|
49
|
+
- `plan.json` is the execution plan (defines the execution order / sequence for migration scripts)
|
|
50
|
+
|
|
51
|
+
Migration scripts export default functions that receive a payload and return the next payload.
|
|
52
|
+
They must use immutable update patterns, and [Immer](https://github.com/immerjs/immer) is recommended for this.
|
|
53
|
+
For example, after editing, the scripts might look like this:
|
|
54
|
+
|
|
55
|
+
**json-migrations/add-panda/add-panda.js** (adds a property to the payload)
|
|
56
|
+
|
|
57
|
+
```javascript
|
|
58
|
+
import produce from 'immer';
|
|
59
|
+
|
|
60
|
+
export default (payload) = produce(state, (draft) => {
|
|
61
|
+
draft.panda = 'Elliot';
|
|
62
|
+
});
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
**json-migrations/plan.json** (defines the migration execution plan)
|
|
66
|
+
|
|
67
|
+
```json
|
|
68
|
+
[
|
|
69
|
+
{
|
|
70
|
+
"fileHash": "70fd80fd19828435b8656b8d01f99b840381050ded4aec2dd126e2f0c9864c88",
|
|
71
|
+
"fileName": "add-panda.js",
|
|
72
|
+
"sequence": "1"
|
|
73
|
+
}
|
|
74
|
+
]
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Note: the `fileHash` property is the SHA256 hash of the migration file content. You don't have to manually maintain
|
|
78
|
+
that, the tool will sync it automatically for you.
|
|
79
|
+
|
|
80
|
+
## Running migrations
|
|
81
|
+
|
|
82
|
+
In your application source code, import and call the `migrate()` function. The simplest usage (runs all migrations)
|
|
83
|
+
is:
|
|
84
|
+
|
|
85
|
+
```javascript
|
|
86
|
+
import {migrate} from 'node-json-migrator';
|
|
87
|
+
|
|
88
|
+
const {nextPayload} = await migrate({
|
|
89
|
+
payload: {},
|
|
90
|
+
config: {
|
|
91
|
+
directory: 'json-migrations'
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Normally you want to track payload versions, and only execute pending migrations. To do this, you need to connect to a
|
|
97
|
+
PostgreSQL database and pass a
|
|
98
|
+
connected [pg-promise database object](http://vitaly-t.github.io/pg-promise/). Then it looks like this:
|
|
99
|
+
|
|
100
|
+
```javascript
|
|
101
|
+
import pgPromise from 'pg-promise';
|
|
102
|
+
import {migrate} from 'node-json-migrator';
|
|
103
|
+
|
|
104
|
+
//connect to the database somewhere in the database layer
|
|
105
|
+
const pgp = pgPromise({});
|
|
106
|
+
const pgpHelpers = pgp.helpers;
|
|
107
|
+
const database = pgp(/*connection string*/);
|
|
108
|
+
database.connect();
|
|
109
|
+
|
|
110
|
+
//if we know the current payload version, we can run only the newer, pending migration scripts
|
|
111
|
+
const currentPayloadVersion = undefined;
|
|
112
|
+
|
|
113
|
+
const {nextPayload, nextVersion} = await migrate({
|
|
114
|
+
payload: {},
|
|
115
|
+
config: {
|
|
116
|
+
directory: 'json-migrations',
|
|
117
|
+
database, //connected database object
|
|
118
|
+
pgpHelpers, //pg-promise helpers: https://vitaly-t.github.io/pg-promise/helpers.html
|
|
119
|
+
version: currentPayloadVersion
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
**Other config options**
|
|
125
|
+
|
|
126
|
+
```javascript
|
|
127
|
+
config = {
|
|
128
|
+
dry: true, //execute without writing to database tables or files
|
|
129
|
+
force: true, //execute bypassing some validation warnings
|
|
130
|
+
}
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## Usage Rules
|
|
134
|
+
|
|
135
|
+
- files in the plan must exist
|
|
136
|
+
- filenames must be unique and follow formatting rules
|
|
137
|
+
- sequences must be unique
|
|
138
|
+
- not allowed to change sequence numbers after releasing (merging) a migration
|
|
139
|
+
- not allowed to remove released files from the plan
|
|
140
|
+
- should release only one migration script at a time
|
|
141
|
+
- not allowed to alter/modify released migration files (instead, write a new migration to replace it)
|
|
142
|
+
- migration scripts should be repeatable (not fail if they run twice on the same payload)
|
|
143
|
+
|
|
144
|
+
## Backfitting bugfixes
|
|
145
|
+
|
|
146
|
+
If you release a bad migration script, there is an escape-hatch for backfitting bugfixes:
|
|
147
|
+
|
|
148
|
+
- rename the faulty script to `.skip.js` (or `.skip2.js` if it happened more than once)
|
|
149
|
+
- add a new migration file with the original filename, to replace it (fix bug in new file)
|
|
150
|
+
- the replacement script will *not* re-execute on payloads that already had the original script applied
|
|
151
|
+
|
|
152
|
+
Or, you can backfit a script earlier in the sequence
|
|
153
|
+
|
|
154
|
+
- add a new migration file
|
|
155
|
+
- set its sequence to for example `1.1` (subsequence number)
|
|
156
|
+
- the backfitted script *will* execute one time on all payloads
|
|
157
|
+
|
|
158
|
+
# How to contribute
|
|
159
|
+
|
|
160
|
+
Contribution is welcome via [issue tickets](https://gitlab.com/oliasoft-open-source/node-json-migrator/-/issues)
|
|
161
|
+
and [merge requests](https://gitlab.com/oliasoft-open-source/node-json-migrator/-/merge_requests).
|
|
162
|
+
|
|
163
|
+
- coding style (ESLint and prettier) is mandatory (applied via pre-commit hook)
|
|
164
|
+
- to test: `npm run test`
|
|
165
|
+
- database is mocked with [pg-mem](https://github.com/oguimbal/pg-mem) (in-memory database)
|
|
166
|
+
- filesystem is mocked [mock-fs](https://github.com/tschaub/mock-fs)
|
|
167
|
+
- coverage reports are in `test/coverage` (not committed)
|
|
168
|
+
- to build: `npm run buld` (this transpile a production build to the `dist` directory)
|
|
169
|
+
- the build `dist` directory should be committed
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
exports.createMigration = void 0;
|
|
7
|
+
|
|
8
|
+
var _path = _interopRequireDefault(require("path"));
|
|
9
|
+
|
|
10
|
+
var _fs = require("fs");
|
|
11
|
+
|
|
12
|
+
var _chalk = _interopRequireDefault(require("chalk"));
|
|
13
|
+
|
|
14
|
+
var _validator = require("../plan/validator");
|
|
15
|
+
|
|
16
|
+
var _plan = require("../plan/plan");
|
|
17
|
+
|
|
18
|
+
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
19
|
+
|
|
20
|
+
const templateMigrationFile = `import produce from 'immer';
|
|
21
|
+
//other imports not allowed
|
|
22
|
+
|
|
23
|
+
export default (dataset) => produce(dataset, (draft) => {
|
|
24
|
+
// https://immerjs.github.io/immer/produce#example
|
|
25
|
+
});
|
|
26
|
+
`;
|
|
27
|
+
|
|
28
|
+
const templateTestFile = fileName => `import migrate from './${fileName}';
|
|
29
|
+
|
|
30
|
+
describe('describe dataset change', () => {
|
|
31
|
+
test('describe test', () => {
|
|
32
|
+
//arrange
|
|
33
|
+
const dataset = {};
|
|
34
|
+
|
|
35
|
+
//act
|
|
36
|
+
const nextDataset = migrate(dataset);
|
|
37
|
+
|
|
38
|
+
//assert
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
`;
|
|
42
|
+
/**
|
|
43
|
+
* Generate a new dataset migration
|
|
44
|
+
*
|
|
45
|
+
* @param {String} directory path to migrations directory
|
|
46
|
+
* @param {String} description description of change-set / migration
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
const createMigration = async (directory, description) => {
|
|
51
|
+
if (!(0, _validator.validateFileDescription)(description)) {
|
|
52
|
+
throw new Error('Invalid migration description');
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
try {
|
|
56
|
+
const directoryFullPath = _path.default.resolve(directory, description);
|
|
57
|
+
|
|
58
|
+
const fileName = `${description}.js`;
|
|
59
|
+
const testFileName = `${description}.test.js`;
|
|
60
|
+
|
|
61
|
+
const filePath = _path.default.resolve(directoryFullPath, fileName);
|
|
62
|
+
|
|
63
|
+
const testFilePath = _path.default.resolve(directoryFullPath, testFileName);
|
|
64
|
+
|
|
65
|
+
await _fs.promises.mkdir(directoryFullPath, {
|
|
66
|
+
recursive: true
|
|
67
|
+
});
|
|
68
|
+
await _fs.promises.writeFile(filePath, templateMigrationFile, {
|
|
69
|
+
encoding: 'utf8'
|
|
70
|
+
});
|
|
71
|
+
await _fs.promises.writeFile(testFilePath, templateTestFile(fileName), {
|
|
72
|
+
encoding: 'utf8'
|
|
73
|
+
});
|
|
74
|
+
const rawPlan = await (0, _plan.readPlan)(directory);
|
|
75
|
+
const plannedMigrations = (0, _plan.parsePlan)(rawPlan);
|
|
76
|
+
const nextPlannedMigrations = plannedMigrations.concat({
|
|
77
|
+
fileHash: '',
|
|
78
|
+
fileName,
|
|
79
|
+
sequence: (0, _plan.getNextSequenceString)(plannedMigrations)
|
|
80
|
+
});
|
|
81
|
+
const nextPlan = JSON.stringify(nextPlannedMigrations, null, 2);
|
|
82
|
+
await (0, _plan.writePlan)(directory, nextPlan);
|
|
83
|
+
console.log(_chalk.default.green(`Created new dataset migration ${fileName} ✓`));
|
|
84
|
+
} catch (error) {
|
|
85
|
+
console.log(_chalk.default.red(error));
|
|
86
|
+
throw new Error('Unable to create migration file');
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
exports.createMigration = createMigration;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
exports.replaceMigrationsRecords = exports.migrationTablesExist = exports.insertVersions = exports.insertMigrationsRecords = exports.getVersions = exports.getPlanFromVersion = exports.getMigrationRecords = exports.createMigrationsTables = void 0;
|
|
7
|
+
|
|
8
|
+
const migrationTablesExist = (db, entity) => {
|
|
9
|
+
const query = `
|
|
10
|
+
SELECT EXISTS (
|
|
11
|
+
SELECT FROM information_schema.tables
|
|
12
|
+
WHERE table_schema = 'public'
|
|
13
|
+
AND table_name = '${entity}_migrations'
|
|
14
|
+
)
|
|
15
|
+
`;
|
|
16
|
+
return db.any(query);
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
exports.migrationTablesExist = migrationTablesExist;
|
|
20
|
+
|
|
21
|
+
const createMigrationsTables = (db, entity) => {
|
|
22
|
+
const query = `
|
|
23
|
+
CREATE TABLE IF NOT EXISTS ${entity}_migrations (
|
|
24
|
+
file_hash TEXT NOT NULL PRIMARY KEY,
|
|
25
|
+
file_name TEXT NOT NULL,
|
|
26
|
+
sequence TEXT NOT NULL,
|
|
27
|
+
created_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
28
|
+
UNIQUE (file_name),
|
|
29
|
+
UNIQUE (sequence)
|
|
30
|
+
);
|
|
31
|
+
CREATE TABLE IF NOT EXISTS ${entity}_versions (
|
|
32
|
+
version TEXT NOT NULL PRIMARY KEY,
|
|
33
|
+
plan TEXT NOT NULL,
|
|
34
|
+
created_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
35
|
+
);
|
|
36
|
+
`;
|
|
37
|
+
return db.any(query);
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
exports.createMigrationsTables = createMigrationsTables;
|
|
41
|
+
|
|
42
|
+
const getMigrationRecords = (db, entity) => {
|
|
43
|
+
const query = `
|
|
44
|
+
SELECT file_hash, file_name, sequence FROM ${entity}_migrations
|
|
45
|
+
ORDER BY sequence
|
|
46
|
+
`;
|
|
47
|
+
return db.manyOrNone(query).then(data => data.map(r => ({
|
|
48
|
+
fileHash: r.file_hash,
|
|
49
|
+
fileName: r.file_name,
|
|
50
|
+
sequence: r.sequence
|
|
51
|
+
})));
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
exports.getMigrationRecords = getMigrationRecords;
|
|
55
|
+
|
|
56
|
+
const insertMigrationsRecords = (db, pgpHelpers, entity, migrations) => {
|
|
57
|
+
if (migrations.length) {
|
|
58
|
+
const toInsert = migrations.map(m => ({
|
|
59
|
+
file_name: m.fileName,
|
|
60
|
+
file_hash: m.fileHash,
|
|
61
|
+
sequence: m.sequence
|
|
62
|
+
})).filter(m => m.file_hash !== null); //can't insert null file hash
|
|
63
|
+
|
|
64
|
+
const cs = new pgpHelpers.ColumnSet(['file_name', 'file_hash', 'sequence'], {
|
|
65
|
+
table: `${entity}_migrations`
|
|
66
|
+
});
|
|
67
|
+
const onConflict = ' ON CONFLICT(file_name) DO NOTHING';
|
|
68
|
+
const query = pgpHelpers.insert(toInsert, cs) + onConflict;
|
|
69
|
+
return db.none(query);
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
exports.insertMigrationsRecords = insertMigrationsRecords;
|
|
74
|
+
|
|
75
|
+
const replaceMigrationsRecords = async (db, pgpHelpers, entity, migrations) => {
|
|
76
|
+
/*
|
|
77
|
+
OW-9043: Table is used for change detection/tracking. DELETE ALL not
|
|
78
|
+
TRUNCATE here so that fewer permissions are needed in downstream
|
|
79
|
+
environments.
|
|
80
|
+
*/
|
|
81
|
+
const truncate = `DELETE FROM ${entity}_migrations`;
|
|
82
|
+
await db.none(truncate);
|
|
83
|
+
await insertMigrationsRecords(db, pgpHelpers, entity, migrations);
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
exports.replaceMigrationsRecords = replaceMigrationsRecords;
|
|
87
|
+
|
|
88
|
+
const insertVersions = (db, pgpHelpers, entity, records) => {
|
|
89
|
+
const cs = new pgpHelpers.ColumnSet(['version', 'plan'], {
|
|
90
|
+
table: `${entity}_versions`
|
|
91
|
+
});
|
|
92
|
+
const onConflict = ' ON CONFLICT(version) DO NOTHING';
|
|
93
|
+
const query = pgpHelpers.insert(records, cs) + onConflict;
|
|
94
|
+
return db.none(query);
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
exports.insertVersions = insertVersions;
|
|
98
|
+
|
|
99
|
+
const getPlanFromVersion = (db, entity, version) => {
|
|
100
|
+
const query = `
|
|
101
|
+
SELECT plan FROM ${entity}_versions
|
|
102
|
+
WHERE version=$1;
|
|
103
|
+
`;
|
|
104
|
+
return db.oneOrNone(query, [version], r => (r === null || r === void 0 ? void 0 : r.plan) || null);
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
exports.getPlanFromVersion = getPlanFromVersion;
|
|
108
|
+
|
|
109
|
+
const getVersions = async (db, entity) => {
|
|
110
|
+
const query = `
|
|
111
|
+
SELECT
|
|
112
|
+
version,
|
|
113
|
+
plan
|
|
114
|
+
FROM ${entity}_versions ORDER BY created_on DESC
|
|
115
|
+
`;
|
|
116
|
+
return db.any(query);
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
exports.getVersions = getVersions;
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
exports.getExecutedMigrationsFromVersion = void 0;
|
|
7
|
+
|
|
8
|
+
var _chalk = _interopRequireDefault(require("chalk"));
|
|
9
|
+
|
|
10
|
+
var _database = require("../database/database");
|
|
11
|
+
|
|
12
|
+
var _validator = require("../plan/validator");
|
|
13
|
+
|
|
14
|
+
var _git = require("../git/git");
|
|
15
|
+
|
|
16
|
+
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Get the executed migrations for a given payload version
|
|
20
|
+
*
|
|
21
|
+
* @param {Object} database database connection object
|
|
22
|
+
* @param {Object} pgpHelpers pg-promise helpers
|
|
23
|
+
* @param {String} entity entity name for payload
|
|
24
|
+
* @param {String} directory path to migrations directory
|
|
25
|
+
* @param {String} currentVersion
|
|
26
|
+
* @param {String} plannedVersion
|
|
27
|
+
* @param {Array<Object>} plannedMigrations planned migration entries
|
|
28
|
+
* returns Promise<Array<Object>> executed migration entries
|
|
29
|
+
*/
|
|
30
|
+
const getExecutedMigrationsFromVersion = async (database, pgpHelpers, entity, directory, currentVersion, plannedVersion, plannedMigrations) => {
|
|
31
|
+
if (!currentVersion) {
|
|
32
|
+
//when there is no current version, payload has never been migrated
|
|
33
|
+
//so executed entries is empty
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
if (currentVersion === plannedVersion) {
|
|
38
|
+
//when planned version matches current version, there is nothing to migrate
|
|
39
|
+
return plannedMigrations;
|
|
40
|
+
} else {
|
|
41
|
+
//otherwise we look up the historical plan file for that version from the DB
|
|
42
|
+
let planFromVersion = await (0, _database.getPlanFromVersion)(database, entity, currentVersion);
|
|
43
|
+
|
|
44
|
+
if (planFromVersion === null) {
|
|
45
|
+
var _historicalPlans$find;
|
|
46
|
+
|
|
47
|
+
//if not in database, try to get it from the git history
|
|
48
|
+
console.warn(_chalk.default.yellow(`Version not found in ${entity}_version table, trying git history instead`));
|
|
49
|
+
const planFilePath = `${directory}/plan.json`;
|
|
50
|
+
const {
|
|
51
|
+
historicalPlans
|
|
52
|
+
} = await (0, _git.getHistoricalPlansFromGit)(planFilePath); //cache the response in the entity versions table
|
|
53
|
+
|
|
54
|
+
await (0, _database.insertVersions)(database, pgpHelpers, entity, historicalPlans);
|
|
55
|
+
planFromVersion = ((_historicalPlans$find = historicalPlans.find(p => p.version === currentVersion)) === null || _historicalPlans$find === void 0 ? void 0 : _historicalPlans$find.plan) || null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (planFromVersion !== null) {
|
|
59
|
+
let migrationsFromHistory = null;
|
|
60
|
+
|
|
61
|
+
try {
|
|
62
|
+
migrationsFromHistory = JSON.parse(planFromVersion);
|
|
63
|
+
} catch {
|
|
64
|
+
throw new Error(`Invalid record in ${entity}_version table (cannot parse JSON)`);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (!(0, _validator.validatePlan)(migrationsFromHistory)) {
|
|
68
|
+
throw new Error(`Invalid plan in ${entity}_version table (does not match schema)`);
|
|
69
|
+
} else {
|
|
70
|
+
return migrationsFromHistory;
|
|
71
|
+
}
|
|
72
|
+
} else {
|
|
73
|
+
throw new Error(`Unable to migrate, could not find version history`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
exports.getExecutedMigrationsFromVersion = getExecutedMigrationsFromVersion;
|