@oliasoft-open-source/node-json-migrator 5.4.0 → 5.5.0-beta-2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -167,3 +167,19 @@ and [merge requests](https://gitlab.com/oliasoft-open-source/node-json-migrator/
167
167
  - coverage reports are in `test/coverage` (not committed)
168
168
  - to build: `npm run buld` (this transpile a production build to the `dist` directory)
169
169
  - the build `dist` directory should be committed
170
+
171
+ ## Concurrent database migrations
172
+
173
+ `migrateAll()` accepts `config.concurrency`, a positive integer that defaults to `1` (sequential processing).
174
+ Set it to a higher value to process independent records concurrently. Each record keeps its own transaction and retries,
175
+ and all workers reuse the plan prepared at the start of the run. Record completion and log order may differ.
176
+
177
+ Each record uses one transaction for its database reads, version-history lookup/cache writes, and update.
178
+ Callbacks `beforeMigrateRecord` and `afterMigrateRecord` must use their provided `transaction` for database operations,
179
+ rather than acquiring another connection from the pool.
180
+
181
+ Choose concurrency to fit the database pool and leave capacity for other application work. Callbacks that acquire
182
+ additional connections can still exhaust the pool and leave migrations waiting for a free connection.
183
+
184
+ When version history is missing from the database, records share a cached Git history lookup per migration run.
185
+ The lookup is lazy. Failed lookups are removed from the cache so retries can try again. A new run starts with an empty cache.
package/dist/index.d.mts CHANGED
@@ -55,7 +55,8 @@ type TConfig = {
55
55
  printPendingFileNames?: boolean;
56
56
  force?: boolean;
57
57
  dry?: boolean;
58
- autoInitializeTable?: boolean;
58
+ autoInitializeTable?: boolean; /** Maximum concurrent records. Defaults to 1. Leave DB pool capacity for other application work. */
59
+ concurrency?: number;
59
60
  };
60
61
  type TBeforeMigrateRecordPayload = {
61
62
  currentRecord: TRecord;
package/dist/index.mjs CHANGED
@@ -464,6 +464,20 @@ function xor(arr1, arr2) {
464
464
  return difference(union(arr1, arr2), intersection(arr1, arr2));
465
465
  }
466
466
  //#endregion
467
+ //#region node_modules/.pnpm/es-toolkit@1.45.1/node_modules/es-toolkit/dist/function/memoize.mjs
468
+ function memoize(fn, options = {}) {
469
+ const { cache = /* @__PURE__ */ new Map(), getCacheKey } = options;
470
+ const memoizedFn = function(arg) {
471
+ const key = getCacheKey ? getCacheKey(arg) : arg;
472
+ if (cache.has(key)) return cache.get(key);
473
+ const result = fn.call(this, arg);
474
+ cache.set(key, result);
475
+ return result;
476
+ };
477
+ memoizedFn.cache = cache;
478
+ return memoizedFn;
479
+ }
480
+ //#endregion
467
481
  //#region node_modules/.pnpm/es-toolkit@1.45.1/node_modules/es-toolkit/dist/function/noop.mjs
468
482
  function noop() {}
469
483
  //#endregion
@@ -4607,7 +4621,7 @@ const pipe = (migrations, payload, metaData) => structuredClone(migrations.reduc
4607
4621
  try {
4608
4622
  return m.migrator(v, metaData);
4609
4623
  } catch (error) {
4610
- throw new Error(errorMessage(m?.fileName, error.message, error.stack));
4624
+ throw new Error(errorMessage(m?.fileName, error.message, error?.stack));
4611
4625
  }
4612
4626
  }, payload));
4613
4627
  const PipeModule = { pipe };
@@ -4699,19 +4713,20 @@ const getHistoricalPlansFromGit = async (planFilePath) => {
4699
4713
  throw new Error("Unable to fetch plan.json history from git");
4700
4714
  }
4701
4715
  };
4716
+ const GitModule = { getHistoricalPlansFromGit };
4702
4717
  //#endregion
4703
4718
  //#region src/executed/executed.ts
4704
4719
  /**
4705
4720
  * Get the executed migrations for a given payload version
4706
4721
  */
4707
- const getExecutedMigrationsFromVersion = async (database, pgpHelpers, entity, directory, currentVersion, plannedVersion, plannedMigrations) => {
4722
+ const getExecutedMigrationsFromVersion = async (database, pgpHelpers, entity, directory, currentVersion, plannedVersion, plannedMigrations, loadHistoricalPlans = getHistoricalPlansFromGit) => {
4708
4723
  if (!currentVersion) return [];
4709
4724
  if (currentVersion === plannedVersion) return plannedMigrations;
4710
4725
  else {
4711
4726
  let planFromVersion = await getPlanFromVersion(database, entity, currentVersion);
4712
4727
  if (planFromVersion === null) {
4713
4728
  console.warn(chalk.yellow(`Version not found in ${entity}_version table, trying git history instead`));
4714
- const { historicalPlans } = await getHistoricalPlansFromGit(`${directory}/plan.json`);
4729
+ const { historicalPlans } = await loadHistoricalPlans(`${directory}/plan.json`);
4715
4730
  if (historicalPlans.length) {
4716
4731
  await insertVersions(database, pgpHelpers, entity, historicalPlans);
4717
4732
  planFromVersion = historicalPlans.find((p) => p.version === currentVersion)?.plan || null;
@@ -4749,10 +4764,11 @@ const getPendingMigrations = ({ plannedMigrations, executedMigrations, printPend
4749
4764
  migrator: m.migrator
4750
4765
  }));
4751
4766
  };
4752
- const getPendingMigrators = async ({ config, preparedPlan }) => {
4767
+ const getPendingMigrators = async ({ config, preparedPlan, transaction, loadHistoricalPlans }) => {
4753
4768
  const { directory, database, pgpHelpers, entity, version, printPendingFileNames, force } = config;
4754
4769
  const { plannedMigrations, nextVersion } = preparedPlan ?? await PlanModule.getPlannedMigrations({ config });
4755
- const executedMigrations = database ? await getExecutedMigrationsFromVersion(database, pgpHelpers, entity, directory, version, nextVersion, plannedMigrations) : [];
4770
+ const connection = transaction ?? database;
4771
+ const executedMigrations = connection ? await getExecutedMigrationsFromVersion(connection, pgpHelpers, entity, directory, version, nextVersion, plannedMigrations, loadHistoricalPlans) : [];
4756
4772
  return {
4757
4773
  pendingMigrators: getPendingMigrations({
4758
4774
  plannedMigrations,
@@ -4775,10 +4791,12 @@ const migrate = async ({ payload, config, metaData }) => {
4775
4791
  metaData
4776
4792
  });
4777
4793
  };
4778
- const migrateWithPlan = async ({ payload, config, metaData, preparedPlan }) => {
4794
+ const migrateWithPlan = async ({ payload, config, metaData, preparedPlan, transaction, loadHistoricalPlans }) => {
4779
4795
  const { pendingMigrators, nextVersion } = await getPendingMigrators({
4780
4796
  config,
4781
- preparedPlan
4797
+ preparedPlan,
4798
+ transaction,
4799
+ loadHistoricalPlans
4782
4800
  });
4783
4801
  return {
4784
4802
  nextPayload: pendingMigrators.length ? PipeModule.pipe(pendingMigrators, payload, metaData) : payload,
@@ -4788,10 +4806,13 @@ const migrateWithPlan = async ({ payload, config, metaData, preparedPlan }) => {
4788
4806
  //#endregion
4789
4807
  //#region src/migrate-all/migrate-all.ts
4790
4808
  const MAX_ATTEMPTS = 3;
4809
+ const validateConcurrency = (concurrency) => {
4810
+ if (!Number.isSafeInteger(concurrency) || concurrency < 1) throw new Error("Migration concurrency must be a positive integer");
4811
+ };
4791
4812
  /**
4792
4813
  * Migrate an entity record in the database
4793
4814
  */
4794
- const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord, afterMigrateRecord }) => {
4815
+ const migrateRecord = async ({ record, config, preparedPlan, loadHistoricalPlans, beforeMigrateRecord, afterMigrateRecord }) => {
4795
4816
  const { dry } = config;
4796
4817
  console.log(chalk.gray(` ${record.name}`));
4797
4818
  try {
@@ -4805,7 +4826,9 @@ const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord
4805
4826
  });
4806
4827
  const { nextPayload: nextData, nextVersion } = await migrateWithPlan({
4807
4828
  payload,
4829
+ transaction,
4808
4830
  preparedPlan,
4831
+ loadHistoricalPlans,
4809
4832
  config: {
4810
4833
  ...config,
4811
4834
  version: currentVersion,
@@ -4831,33 +4854,47 @@ const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord
4831
4854
  [config.entityColumnNames.name]: record.name,
4832
4855
  [config.entityColumnNames.created]: record.created,
4833
4856
  error,
4834
- stackTrace: error.stack
4857
+ stackTrace: error?.stack
4835
4858
  };
4836
- console.log(chalk.red(` ${capitalize(config.entity)} migration with id ${record.id} failed:\n ${error.stack}`));
4859
+ console.log(chalk.red(` ${capitalize(config.entity)} migration with id ${record.id} failed:\n ${error?.stack}`));
4837
4860
  return migrationErrorPayload;
4838
4861
  }
4839
4862
  };
4840
4863
  const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
4841
- const { dry } = config;
4864
+ const { dry, concurrency = 1 } = config;
4865
+ validateConcurrency(concurrency);
4842
4866
  const migrationErrors = [];
4867
+ const loadHistoricalPlans = memoize(async (planFilePath) => {
4868
+ try {
4869
+ return await GitModule.getHistoricalPlansFromGit(planFilePath);
4870
+ } catch (error) {
4871
+ loadHistoricalPlans.cache.delete(planFilePath);
4872
+ throw error;
4873
+ }
4874
+ });
4843
4875
  if (recordList.length) {
4844
4876
  console.log(chalk.gray("Executing record migration scripts..."));
4845
- for await (const record of recordList) {
4846
- let attempt = 0;
4847
- while (attempt < MAX_ATTEMPTS) {
4848
- attempt += 1;
4849
- if (attempt > 1) console.log(chalk.gray("Retrying migration..."));
4850
- const migrationErrorPayload = await migrateRecord({
4851
- record,
4852
- config,
4853
- preparedPlan,
4854
- beforeMigrateRecord,
4855
- afterMigrateRecord
4856
- });
4857
- if (!migrationErrorPayload) break;
4858
- if (migrationErrorPayload && attempt === MAX_ATTEMPTS) migrationErrors.push(migrationErrorPayload);
4877
+ const records = recordList.values();
4878
+ const worker = async () => {
4879
+ for (const record of records) {
4880
+ let attempt = 0;
4881
+ while (attempt < MAX_ATTEMPTS) {
4882
+ attempt += 1;
4883
+ if (attempt > 1) console.log(chalk.gray(`Retrying migration of ${record.name} (attempt ${attempt}/${MAX_ATTEMPTS}).`));
4884
+ const migrationErrorPayload = await migrateRecord({
4885
+ record,
4886
+ config,
4887
+ preparedPlan,
4888
+ loadHistoricalPlans,
4889
+ beforeMigrateRecord,
4890
+ afterMigrateRecord
4891
+ });
4892
+ if (!migrationErrorPayload) break;
4893
+ if (migrationErrorPayload && attempt === MAX_ATTEMPTS) migrationErrors.push(migrationErrorPayload);
4894
+ }
4859
4895
  }
4860
- }
4896
+ };
4897
+ await Promise.all(Array.from({ length: Math.min(concurrency, recordList.length) }, worker));
4861
4898
  }
4862
4899
  if (migrationErrors.length) {
4863
4900
  try {
@@ -4866,7 +4903,7 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
4866
4903
  dry
4867
4904
  });
4868
4905
  } catch (error) {
4869
- console.log(chalk.red(`Handling errors for ${config.entity} migrations failed:\n ${error.stack}`));
4906
+ console.log(chalk.red(`Handling errors for ${config.entity} migrations failed:\n ${error?.stack}`));
4870
4907
  }
4871
4908
  console.log(chalk.yellow(`Completed ${config?.entity ?? "entity"} migrations (some failed)${config?.dry ? " [dry-run, no output written]" : ""} ✓`));
4872
4909
  } else console.log(chalk.green(`Completed ${config?.entity ?? "entity"} migrations (success)${config?.dry ? " [dry-run, no output written]" : ""} ✓`));
@@ -4875,6 +4912,7 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
4875
4912
  * Migrate all records (only use via command line interface)
4876
4913
  */
4877
4914
  const migrateAll = async ({ config, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
4915
+ validateConcurrency(config.concurrency ?? 1);
4878
4916
  const preparedPlan = await PlanModule.getPlannedMigrations({ config });
4879
4917
  const { validatedPlan: latestPlan, nextVersion: latestVersion } = preparedPlan;
4880
4918
  await migrateRecords({
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@oliasoft-open-source/node-json-migrator",
3
- "version": "5.4.0",
3
+ "version": "5.5.0-beta-2",
4
4
  "description": "A library for JSON migrations",
5
5
  "homepage": "https://oliasoft-open-source.gitlab.io/node-json-migrator",
6
6
  "bugs": {