@oliasoft-open-source/node-json-migrator 5.4.0 → 5.5.0-beta-2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/index.d.mts +2 -1
- package/dist/index.mjs +65 -27
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -167,3 +167,19 @@ and [merge requests](https://gitlab.com/oliasoft-open-source/node-json-migrator/
|
|
|
167
167
|
- coverage reports are in `test/coverage` (not committed)
|
|
168
168
|
- to build: `npm run buld` (this transpile a production build to the `dist` directory)
|
|
169
169
|
- the build `dist` directory should be committed
|
|
170
|
+
|
|
171
|
+
## Concurrent database migrations
|
|
172
|
+
|
|
173
|
+
`migrateAll()` accepts `config.concurrency`, a positive integer that defaults to `1` (sequential processing).
|
|
174
|
+
Set it to a higher value to process independent records concurrently. Each record keeps its own transaction and retries,
|
|
175
|
+
and all workers reuse the plan prepared at the start of the run. Record completion and log order may differ.
|
|
176
|
+
|
|
177
|
+
Each record uses one transaction for its database reads, version-history lookup/cache writes, and update.
|
|
178
|
+
Callbacks `beforeMigrateRecord` and `afterMigrateRecord` must use their provided `transaction` for database operations,
|
|
179
|
+
rather than acquiring another connection from the pool.
|
|
180
|
+
|
|
181
|
+
Choose concurrency to fit the database pool and leave capacity for other application work. Callbacks that acquire
|
|
182
|
+
additional connections can still exhaust the pool and leave migrations waiting for a free connection.
|
|
183
|
+
|
|
184
|
+
When version history is missing from the database, records share a cached Git history lookup per migration run.
|
|
185
|
+
The lookup is lazy. Failed lookups are removed from the cache so retries can try again. A new run starts with an empty cache.
|
package/dist/index.d.mts
CHANGED
|
@@ -55,7 +55,8 @@ type TConfig = {
|
|
|
55
55
|
printPendingFileNames?: boolean;
|
|
56
56
|
force?: boolean;
|
|
57
57
|
dry?: boolean;
|
|
58
|
-
autoInitializeTable?: boolean;
|
|
58
|
+
autoInitializeTable?: boolean; /** Maximum concurrent records. Defaults to 1. Leave DB pool capacity for other application work. */
|
|
59
|
+
concurrency?: number;
|
|
59
60
|
};
|
|
60
61
|
type TBeforeMigrateRecordPayload = {
|
|
61
62
|
currentRecord: TRecord;
|
package/dist/index.mjs
CHANGED
|
@@ -464,6 +464,20 @@ function xor(arr1, arr2) {
|
|
|
464
464
|
return difference(union(arr1, arr2), intersection(arr1, arr2));
|
|
465
465
|
}
|
|
466
466
|
//#endregion
|
|
467
|
+
//#region node_modules/.pnpm/es-toolkit@1.45.1/node_modules/es-toolkit/dist/function/memoize.mjs
|
|
468
|
+
function memoize(fn, options = {}) {
|
|
469
|
+
const { cache = /* @__PURE__ */ new Map(), getCacheKey } = options;
|
|
470
|
+
const memoizedFn = function(arg) {
|
|
471
|
+
const key = getCacheKey ? getCacheKey(arg) : arg;
|
|
472
|
+
if (cache.has(key)) return cache.get(key);
|
|
473
|
+
const result = fn.call(this, arg);
|
|
474
|
+
cache.set(key, result);
|
|
475
|
+
return result;
|
|
476
|
+
};
|
|
477
|
+
memoizedFn.cache = cache;
|
|
478
|
+
return memoizedFn;
|
|
479
|
+
}
|
|
480
|
+
//#endregion
|
|
467
481
|
//#region node_modules/.pnpm/es-toolkit@1.45.1/node_modules/es-toolkit/dist/function/noop.mjs
|
|
468
482
|
function noop() {}
|
|
469
483
|
//#endregion
|
|
@@ -4607,7 +4621,7 @@ const pipe = (migrations, payload, metaData) => structuredClone(migrations.reduc
|
|
|
4607
4621
|
try {
|
|
4608
4622
|
return m.migrator(v, metaData);
|
|
4609
4623
|
} catch (error) {
|
|
4610
|
-
throw new Error(errorMessage(m?.fileName, error.message, error
|
|
4624
|
+
throw new Error(errorMessage(m?.fileName, error.message, error?.stack));
|
|
4611
4625
|
}
|
|
4612
4626
|
}, payload));
|
|
4613
4627
|
const PipeModule = { pipe };
|
|
@@ -4699,19 +4713,20 @@ const getHistoricalPlansFromGit = async (planFilePath) => {
|
|
|
4699
4713
|
throw new Error("Unable to fetch plan.json history from git");
|
|
4700
4714
|
}
|
|
4701
4715
|
};
|
|
4716
|
+
const GitModule = { getHistoricalPlansFromGit };
|
|
4702
4717
|
//#endregion
|
|
4703
4718
|
//#region src/executed/executed.ts
|
|
4704
4719
|
/**
|
|
4705
4720
|
* Get the executed migrations for a given payload version
|
|
4706
4721
|
*/
|
|
4707
|
-
const getExecutedMigrationsFromVersion = async (database, pgpHelpers, entity, directory, currentVersion, plannedVersion, plannedMigrations) => {
|
|
4722
|
+
const getExecutedMigrationsFromVersion = async (database, pgpHelpers, entity, directory, currentVersion, plannedVersion, plannedMigrations, loadHistoricalPlans = getHistoricalPlansFromGit) => {
|
|
4708
4723
|
if (!currentVersion) return [];
|
|
4709
4724
|
if (currentVersion === plannedVersion) return plannedMigrations;
|
|
4710
4725
|
else {
|
|
4711
4726
|
let planFromVersion = await getPlanFromVersion(database, entity, currentVersion);
|
|
4712
4727
|
if (planFromVersion === null) {
|
|
4713
4728
|
console.warn(chalk.yellow(`Version not found in ${entity}_version table, trying git history instead`));
|
|
4714
|
-
const { historicalPlans } = await
|
|
4729
|
+
const { historicalPlans } = await loadHistoricalPlans(`${directory}/plan.json`);
|
|
4715
4730
|
if (historicalPlans.length) {
|
|
4716
4731
|
await insertVersions(database, pgpHelpers, entity, historicalPlans);
|
|
4717
4732
|
planFromVersion = historicalPlans.find((p) => p.version === currentVersion)?.plan || null;
|
|
@@ -4749,10 +4764,11 @@ const getPendingMigrations = ({ plannedMigrations, executedMigrations, printPend
|
|
|
4749
4764
|
migrator: m.migrator
|
|
4750
4765
|
}));
|
|
4751
4766
|
};
|
|
4752
|
-
const getPendingMigrators = async ({ config, preparedPlan }) => {
|
|
4767
|
+
const getPendingMigrators = async ({ config, preparedPlan, transaction, loadHistoricalPlans }) => {
|
|
4753
4768
|
const { directory, database, pgpHelpers, entity, version, printPendingFileNames, force } = config;
|
|
4754
4769
|
const { plannedMigrations, nextVersion } = preparedPlan ?? await PlanModule.getPlannedMigrations({ config });
|
|
4755
|
-
const
|
|
4770
|
+
const connection = transaction ?? database;
|
|
4771
|
+
const executedMigrations = connection ? await getExecutedMigrationsFromVersion(connection, pgpHelpers, entity, directory, version, nextVersion, plannedMigrations, loadHistoricalPlans) : [];
|
|
4756
4772
|
return {
|
|
4757
4773
|
pendingMigrators: getPendingMigrations({
|
|
4758
4774
|
plannedMigrations,
|
|
@@ -4775,10 +4791,12 @@ const migrate = async ({ payload, config, metaData }) => {
|
|
|
4775
4791
|
metaData
|
|
4776
4792
|
});
|
|
4777
4793
|
};
|
|
4778
|
-
const migrateWithPlan = async ({ payload, config, metaData, preparedPlan }) => {
|
|
4794
|
+
const migrateWithPlan = async ({ payload, config, metaData, preparedPlan, transaction, loadHistoricalPlans }) => {
|
|
4779
4795
|
const { pendingMigrators, nextVersion } = await getPendingMigrators({
|
|
4780
4796
|
config,
|
|
4781
|
-
preparedPlan
|
|
4797
|
+
preparedPlan,
|
|
4798
|
+
transaction,
|
|
4799
|
+
loadHistoricalPlans
|
|
4782
4800
|
});
|
|
4783
4801
|
return {
|
|
4784
4802
|
nextPayload: pendingMigrators.length ? PipeModule.pipe(pendingMigrators, payload, metaData) : payload,
|
|
@@ -4788,10 +4806,13 @@ const migrateWithPlan = async ({ payload, config, metaData, preparedPlan }) => {
|
|
|
4788
4806
|
//#endregion
|
|
4789
4807
|
//#region src/migrate-all/migrate-all.ts
|
|
4790
4808
|
const MAX_ATTEMPTS = 3;
|
|
4809
|
+
const validateConcurrency = (concurrency) => {
|
|
4810
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1) throw new Error("Migration concurrency must be a positive integer");
|
|
4811
|
+
};
|
|
4791
4812
|
/**
|
|
4792
4813
|
* Migrate an entity record in the database
|
|
4793
4814
|
*/
|
|
4794
|
-
const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord, afterMigrateRecord }) => {
|
|
4815
|
+
const migrateRecord = async ({ record, config, preparedPlan, loadHistoricalPlans, beforeMigrateRecord, afterMigrateRecord }) => {
|
|
4795
4816
|
const { dry } = config;
|
|
4796
4817
|
console.log(chalk.gray(` ${record.name}`));
|
|
4797
4818
|
try {
|
|
@@ -4805,7 +4826,9 @@ const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord
|
|
|
4805
4826
|
});
|
|
4806
4827
|
const { nextPayload: nextData, nextVersion } = await migrateWithPlan({
|
|
4807
4828
|
payload,
|
|
4829
|
+
transaction,
|
|
4808
4830
|
preparedPlan,
|
|
4831
|
+
loadHistoricalPlans,
|
|
4809
4832
|
config: {
|
|
4810
4833
|
...config,
|
|
4811
4834
|
version: currentVersion,
|
|
@@ -4831,33 +4854,47 @@ const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord
|
|
|
4831
4854
|
[config.entityColumnNames.name]: record.name,
|
|
4832
4855
|
[config.entityColumnNames.created]: record.created,
|
|
4833
4856
|
error,
|
|
4834
|
-
stackTrace: error
|
|
4857
|
+
stackTrace: error?.stack
|
|
4835
4858
|
};
|
|
4836
|
-
console.log(chalk.red(` ${capitalize(config.entity)} migration with id ${record.id} failed:\n ${error
|
|
4859
|
+
console.log(chalk.red(` ${capitalize(config.entity)} migration with id ${record.id} failed:\n ${error?.stack}`));
|
|
4837
4860
|
return migrationErrorPayload;
|
|
4838
4861
|
}
|
|
4839
4862
|
};
|
|
4840
4863
|
const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
|
|
4841
|
-
const { dry } = config;
|
|
4864
|
+
const { dry, concurrency = 1 } = config;
|
|
4865
|
+
validateConcurrency(concurrency);
|
|
4842
4866
|
const migrationErrors = [];
|
|
4867
|
+
const loadHistoricalPlans = memoize(async (planFilePath) => {
|
|
4868
|
+
try {
|
|
4869
|
+
return await GitModule.getHistoricalPlansFromGit(planFilePath);
|
|
4870
|
+
} catch (error) {
|
|
4871
|
+
loadHistoricalPlans.cache.delete(planFilePath);
|
|
4872
|
+
throw error;
|
|
4873
|
+
}
|
|
4874
|
+
});
|
|
4843
4875
|
if (recordList.length) {
|
|
4844
4876
|
console.log(chalk.gray("Executing record migration scripts..."));
|
|
4845
|
-
|
|
4846
|
-
|
|
4847
|
-
|
|
4848
|
-
attempt
|
|
4849
|
-
|
|
4850
|
-
|
|
4851
|
-
record
|
|
4852
|
-
|
|
4853
|
-
|
|
4854
|
-
|
|
4855
|
-
|
|
4856
|
-
|
|
4857
|
-
|
|
4858
|
-
|
|
4877
|
+
const records = recordList.values();
|
|
4878
|
+
const worker = async () => {
|
|
4879
|
+
for (const record of records) {
|
|
4880
|
+
let attempt = 0;
|
|
4881
|
+
while (attempt < MAX_ATTEMPTS) {
|
|
4882
|
+
attempt += 1;
|
|
4883
|
+
if (attempt > 1) console.log(chalk.gray(`Retrying migration of ${record.name} (attempt ${attempt}/${MAX_ATTEMPTS}).`));
|
|
4884
|
+
const migrationErrorPayload = await migrateRecord({
|
|
4885
|
+
record,
|
|
4886
|
+
config,
|
|
4887
|
+
preparedPlan,
|
|
4888
|
+
loadHistoricalPlans,
|
|
4889
|
+
beforeMigrateRecord,
|
|
4890
|
+
afterMigrateRecord
|
|
4891
|
+
});
|
|
4892
|
+
if (!migrationErrorPayload) break;
|
|
4893
|
+
if (migrationErrorPayload && attempt === MAX_ATTEMPTS) migrationErrors.push(migrationErrorPayload);
|
|
4894
|
+
}
|
|
4859
4895
|
}
|
|
4860
|
-
}
|
|
4896
|
+
};
|
|
4897
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, recordList.length) }, worker));
|
|
4861
4898
|
}
|
|
4862
4899
|
if (migrationErrors.length) {
|
|
4863
4900
|
try {
|
|
@@ -4866,7 +4903,7 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
|
|
|
4866
4903
|
dry
|
|
4867
4904
|
});
|
|
4868
4905
|
} catch (error) {
|
|
4869
|
-
console.log(chalk.red(`Handling errors for ${config.entity} migrations failed:\n ${error
|
|
4906
|
+
console.log(chalk.red(`Handling errors for ${config.entity} migrations failed:\n ${error?.stack}`));
|
|
4870
4907
|
}
|
|
4871
4908
|
console.log(chalk.yellow(`Completed ${config?.entity ?? "entity"} migrations (some failed)${config?.dry ? " [dry-run, no output written]" : ""} ✓`));
|
|
4872
4909
|
} else console.log(chalk.green(`Completed ${config?.entity ?? "entity"} migrations (success)${config?.dry ? " [dry-run, no output written]" : ""} ✓`));
|
|
@@ -4875,6 +4912,7 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
|
|
|
4875
4912
|
* Migrate all records (only use via command line interface)
|
|
4876
4913
|
*/
|
|
4877
4914
|
const migrateAll = async ({ config, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
|
|
4915
|
+
validateConcurrency(config.concurrency ?? 1);
|
|
4878
4916
|
const preparedPlan = await PlanModule.getPlannedMigrations({ config });
|
|
4879
4917
|
const { validatedPlan: latestPlan, nextVersion: latestVersion } = preparedPlan;
|
|
4880
4918
|
await migrateRecords({
|
package/package.json
CHANGED