@oliasoft-open-source/node-json-migrator 5.3.0 → 5.5.0-beta-1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/index.d.mts +7 -3
- package/dist/index.mjs +39 -23
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -167,3 +167,13 @@ and [merge requests](https://gitlab.com/oliasoft-open-source/node-json-migrator/
|
|
|
167
167
|
- coverage reports are in `test/coverage` (not committed)
|
|
168
168
|
- to build: `npm run buld` (this transpile a production build to the `dist` directory)
|
|
169
169
|
- the build `dist` directory should be committed
|
|
170
|
+
|
|
171
|
+
## Concurrent database migrations
|
|
172
|
+
|
|
173
|
+
`migrateAll()` accepts `config.concurrency`, a positive integer that defaults to `1` (sequential processing).
|
|
174
|
+
Set it to a higher value to process independent records concurrently. Each record keeps its own transaction and retries,
|
|
175
|
+
and all workers reuse the plan prepared at the start of the run. Record completion and log order may differ.
|
|
176
|
+
|
|
177
|
+
Keep concurrency below the database pool size and leave capacity for other queries. Version history is currently loaded
|
|
178
|
+
outside the record transaction, and callbacks may also need extra connections. Using every connection for record
|
|
179
|
+
transactions can leave the run waiting indefinitely for a free connection.
|
package/dist/index.d.mts
CHANGED
|
@@ -55,7 +55,8 @@ type TConfig = {
|
|
|
55
55
|
printPendingFileNames?: boolean;
|
|
56
56
|
force?: boolean;
|
|
57
57
|
dry?: boolean;
|
|
58
|
-
autoInitializeTable?: boolean;
|
|
58
|
+
autoInitializeTable?: boolean; /** Maximum concurrent records. Defaults to 1. Must leave DB pool capacity for queries outside transactions. */
|
|
59
|
+
concurrency?: number;
|
|
59
60
|
};
|
|
60
61
|
type TBeforeMigrateRecordPayload = {
|
|
61
62
|
currentRecord: TRecord;
|
|
@@ -100,6 +101,7 @@ declare const getPlannedMigrations: ({
|
|
|
100
101
|
}[];
|
|
101
102
|
nextVersion: string;
|
|
102
103
|
}>;
|
|
104
|
+
type TPreparedPlan = Awaited<ReturnType<typeof getPlannedMigrations>>;
|
|
103
105
|
//#endregion
|
|
104
106
|
//#region src/plan/planned-version.d.ts
|
|
105
107
|
declare const getPlannedVersion: ({
|
|
@@ -114,16 +116,18 @@ declare const getPlannedVersion: ({
|
|
|
114
116
|
*
|
|
115
117
|
* There are two ways to use this function:
|
|
116
118
|
* i) function looks up the pending migrations for you (default)
|
|
117
|
-
* ii)
|
|
119
|
+
* ii) reuse a prepared plan while resolving pending migrations for the current payload version
|
|
118
120
|
*/
|
|
119
121
|
declare const migrate: ({
|
|
120
122
|
payload,
|
|
121
123
|
config,
|
|
122
|
-
metaData
|
|
124
|
+
metaData,
|
|
125
|
+
preparedPlan
|
|
123
126
|
}: {
|
|
124
127
|
payload: TPayload;
|
|
125
128
|
config: TConfig;
|
|
126
129
|
metaData?: TMetaData;
|
|
130
|
+
preparedPlan?: TPreparedPlan;
|
|
127
131
|
}) => Promise<{
|
|
128
132
|
nextPayload: TPayload;
|
|
129
133
|
nextVersion: string;
|
package/dist/index.mjs
CHANGED
|
@@ -4749,9 +4749,9 @@ const getPendingMigrations = ({ plannedMigrations, executedMigrations, printPend
|
|
|
4749
4749
|
migrator: m.migrator
|
|
4750
4750
|
}));
|
|
4751
4751
|
};
|
|
4752
|
-
const getPendingMigrators = async ({ config }) => {
|
|
4752
|
+
const getPendingMigrators = async ({ config, preparedPlan }) => {
|
|
4753
4753
|
const { directory, database, pgpHelpers, entity, version, printPendingFileNames, force } = config;
|
|
4754
|
-
const { plannedMigrations, nextVersion } = await PlanModule.getPlannedMigrations({ config });
|
|
4754
|
+
const { plannedMigrations, nextVersion } = preparedPlan ?? await PlanModule.getPlannedMigrations({ config });
|
|
4755
4755
|
const executedMigrations = database ? await getExecutedMigrationsFromVersion(database, pgpHelpers, entity, directory, version, nextVersion, plannedMigrations) : [];
|
|
4756
4756
|
return {
|
|
4757
4757
|
pendingMigrators: getPendingMigrations({
|
|
@@ -4770,10 +4770,13 @@ const getPendingMigrators = async ({ config }) => {
|
|
|
4770
4770
|
*
|
|
4771
4771
|
* There are two ways to use this function:
|
|
4772
4772
|
* i) function looks up the pending migrations for you (default)
|
|
4773
|
-
* ii)
|
|
4773
|
+
* ii) reuse a prepared plan while resolving pending migrations for the current payload version
|
|
4774
4774
|
*/
|
|
4775
|
-
const migrate = async ({ payload, config, metaData }) => {
|
|
4776
|
-
const { pendingMigrators, nextVersion } = await getPendingMigrators({
|
|
4775
|
+
const migrate = async ({ payload, config, metaData, preparedPlan }) => {
|
|
4776
|
+
const { pendingMigrators, nextVersion } = await getPendingMigrators({
|
|
4777
|
+
config,
|
|
4778
|
+
preparedPlan
|
|
4779
|
+
});
|
|
4777
4780
|
return {
|
|
4778
4781
|
nextPayload: pendingMigrators.length ? PipeModule.pipe(pendingMigrators, payload, metaData) : payload,
|
|
4779
4782
|
nextVersion
|
|
@@ -4782,10 +4785,13 @@ const migrate = async ({ payload, config, metaData }) => {
|
|
|
4782
4785
|
//#endregion
|
|
4783
4786
|
//#region src/migrate-all/migrate-all.ts
|
|
4784
4787
|
const MAX_ATTEMPTS = 3;
|
|
4788
|
+
const validateConcurrency = (concurrency) => {
|
|
4789
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1) throw new Error("Migration concurrency must be a positive integer");
|
|
4790
|
+
};
|
|
4785
4791
|
/**
|
|
4786
4792
|
* Migrate an entity record in the database
|
|
4787
4793
|
*/
|
|
4788
|
-
const migrateRecord = async ({ record, config, beforeMigrateRecord, afterMigrateRecord }) => {
|
|
4794
|
+
const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord, afterMigrateRecord }) => {
|
|
4789
4795
|
const { dry } = config;
|
|
4790
4796
|
console.log(chalk.gray(` ${record.name}`));
|
|
4791
4797
|
try {
|
|
@@ -4799,6 +4805,7 @@ const migrateRecord = async ({ record, config, beforeMigrateRecord, afterMigrate
|
|
|
4799
4805
|
});
|
|
4800
4806
|
const { nextPayload: nextData, nextVersion } = await migrate({
|
|
4801
4807
|
payload,
|
|
4808
|
+
preparedPlan,
|
|
4802
4809
|
config: {
|
|
4803
4810
|
...config,
|
|
4804
4811
|
version: currentVersion,
|
|
@@ -4830,26 +4837,32 @@ const migrateRecord = async ({ record, config, beforeMigrateRecord, afterMigrate
|
|
|
4830
4837
|
return migrationErrorPayload;
|
|
4831
4838
|
}
|
|
4832
4839
|
};
|
|
4833
|
-
const migrateRecords = async ({ recordList, config, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
|
|
4834
|
-
const { dry } = config;
|
|
4840
|
+
const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
|
|
4841
|
+
const { dry, concurrency = 1 } = config;
|
|
4842
|
+
validateConcurrency(concurrency);
|
|
4835
4843
|
const migrationErrors = [];
|
|
4836
4844
|
if (recordList.length) {
|
|
4837
4845
|
console.log(chalk.gray("Executing record migration scripts..."));
|
|
4838
|
-
|
|
4839
|
-
|
|
4840
|
-
|
|
4841
|
-
attempt
|
|
4842
|
-
|
|
4843
|
-
|
|
4844
|
-
|
|
4845
|
-
|
|
4846
|
-
|
|
4847
|
-
|
|
4848
|
-
|
|
4849
|
-
|
|
4850
|
-
|
|
4846
|
+
const records = recordList.values();
|
|
4847
|
+
const worker = async () => {
|
|
4848
|
+
for (const record of records) {
|
|
4849
|
+
let attempt = 0;
|
|
4850
|
+
while (attempt < MAX_ATTEMPTS) {
|
|
4851
|
+
attempt += 1;
|
|
4852
|
+
if (attempt > 1) console.log(chalk.gray("Retrying migration..."));
|
|
4853
|
+
const migrationErrorPayload = await migrateRecord({
|
|
4854
|
+
record,
|
|
4855
|
+
config,
|
|
4856
|
+
preparedPlan,
|
|
4857
|
+
beforeMigrateRecord,
|
|
4858
|
+
afterMigrateRecord
|
|
4859
|
+
});
|
|
4860
|
+
if (!migrationErrorPayload) break;
|
|
4861
|
+
if (migrationErrorPayload && attempt === MAX_ATTEMPTS) migrationErrors.push(migrationErrorPayload);
|
|
4862
|
+
}
|
|
4851
4863
|
}
|
|
4852
|
-
}
|
|
4864
|
+
};
|
|
4865
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, recordList.length) }, worker));
|
|
4853
4866
|
}
|
|
4854
4867
|
if (migrationErrors.length) {
|
|
4855
4868
|
try {
|
|
@@ -4867,9 +4880,12 @@ const migrateRecords = async ({ recordList, config, beforeMigrateRecord, afterMi
|
|
|
4867
4880
|
* Migrate all records (only use via command line interface)
|
|
4868
4881
|
*/
|
|
4869
4882
|
const migrateAll = async ({ config, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
|
|
4870
|
-
|
|
4883
|
+
validateConcurrency(config.concurrency ?? 1);
|
|
4884
|
+
const preparedPlan = await getPlannedMigrations({ config });
|
|
4885
|
+
const { validatedPlan: latestPlan, nextVersion: latestVersion } = preparedPlan;
|
|
4871
4886
|
await migrateRecords({
|
|
4872
4887
|
config,
|
|
4888
|
+
preparedPlan,
|
|
4873
4889
|
recordList: await getRecordList(config?.database, config?.entity, config?.entityTableName, config?.entityColumnNames, latestVersion, latestPlan),
|
|
4874
4890
|
beforeMigrateRecord,
|
|
4875
4891
|
afterMigrateRecord,
|
package/package.json
CHANGED