@oliasoft-open-source/node-json-migrator 5.3.0 → 5.5.0-beta-1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -167,3 +167,13 @@ and [merge requests](https://gitlab.com/oliasoft-open-source/node-json-migrator/
167
167
  - coverage reports are in `test/coverage` (not committed)
168
168
  - to build: `npm run buld` (this transpile a production build to the `dist` directory)
169
169
  - the build `dist` directory should be committed
170
+
171
+ ## Concurrent database migrations
172
+
173
+ `migrateAll()` accepts `config.concurrency`, a positive integer that defaults to `1` (sequential processing).
174
+ Set it to a higher value to process independent records concurrently. Each record keeps its own transaction and retries,
175
+ and all workers reuse the plan prepared at the start of the run. Record completion and log order may differ.
176
+
177
+ Keep concurrency below the database pool size and leave capacity for other queries. Version history is currently loaded
178
+ outside the record transaction, and callbacks may also need extra connections. Using every connection for record
179
+ transactions can leave the run waiting indefinitely for a free connection.
package/dist/index.d.mts CHANGED
@@ -55,7 +55,8 @@ type TConfig = {
55
55
  printPendingFileNames?: boolean;
56
56
  force?: boolean;
57
57
  dry?: boolean;
58
- autoInitializeTable?: boolean;
58
+ autoInitializeTable?: boolean; /** Maximum concurrent records. Defaults to 1. Must leave DB pool capacity for queries outside transactions. */
59
+ concurrency?: number;
59
60
  };
60
61
  type TBeforeMigrateRecordPayload = {
61
62
  currentRecord: TRecord;
@@ -100,6 +101,7 @@ declare const getPlannedMigrations: ({
100
101
  }[];
101
102
  nextVersion: string;
102
103
  }>;
104
+ type TPreparedPlan = Awaited<ReturnType<typeof getPlannedMigrations>>;
103
105
  //#endregion
104
106
  //#region src/plan/planned-version.d.ts
105
107
  declare const getPlannedVersion: ({
@@ -114,16 +116,18 @@ declare const getPlannedVersion: ({
114
116
  *
115
117
  * There are two ways to use this function:
116
118
  * i) function looks up the pending migrations for you (default)
117
- * ii) pass in pre-fetched pending migrators via config.pending
119
+ * ii) reuse a prepared plan while resolving pending migrations for the current payload version
118
120
  */
119
121
  declare const migrate: ({
120
122
  payload,
121
123
  config,
122
- metaData
124
+ metaData,
125
+ preparedPlan
123
126
  }: {
124
127
  payload: TPayload;
125
128
  config: TConfig;
126
129
  metaData?: TMetaData;
130
+ preparedPlan?: TPreparedPlan;
127
131
  }) => Promise<{
128
132
  nextPayload: TPayload;
129
133
  nextVersion: string;
package/dist/index.mjs CHANGED
@@ -4749,9 +4749,9 @@ const getPendingMigrations = ({ plannedMigrations, executedMigrations, printPend
4749
4749
  migrator: m.migrator
4750
4750
  }));
4751
4751
  };
4752
- const getPendingMigrators = async ({ config }) => {
4752
+ const getPendingMigrators = async ({ config, preparedPlan }) => {
4753
4753
  const { directory, database, pgpHelpers, entity, version, printPendingFileNames, force } = config;
4754
- const { plannedMigrations, nextVersion } = await PlanModule.getPlannedMigrations({ config });
4754
+ const { plannedMigrations, nextVersion } = preparedPlan ?? await PlanModule.getPlannedMigrations({ config });
4755
4755
  const executedMigrations = database ? await getExecutedMigrationsFromVersion(database, pgpHelpers, entity, directory, version, nextVersion, plannedMigrations) : [];
4756
4756
  return {
4757
4757
  pendingMigrators: getPendingMigrations({
@@ -4770,10 +4770,13 @@ const getPendingMigrators = async ({ config }) => {
4770
4770
  *
4771
4771
  * There are two ways to use this function:
4772
4772
  * i) function looks up the pending migrations for you (default)
4773
- * ii) pass in pre-fetched pending migrators via config.pending
4773
+ * ii) reuse a prepared plan while resolving pending migrations for the current payload version
4774
4774
  */
4775
- const migrate = async ({ payload, config, metaData }) => {
4776
- const { pendingMigrators, nextVersion } = await getPendingMigrators({ config });
4775
+ const migrate = async ({ payload, config, metaData, preparedPlan }) => {
4776
+ const { pendingMigrators, nextVersion } = await getPendingMigrators({
4777
+ config,
4778
+ preparedPlan
4779
+ });
4777
4780
  return {
4778
4781
  nextPayload: pendingMigrators.length ? PipeModule.pipe(pendingMigrators, payload, metaData) : payload,
4779
4782
  nextVersion
@@ -4782,10 +4785,13 @@ const migrate = async ({ payload, config, metaData }) => {
4782
4785
  //#endregion
4783
4786
  //#region src/migrate-all/migrate-all.ts
4784
4787
  const MAX_ATTEMPTS = 3;
4788
+ const validateConcurrency = (concurrency) => {
4789
+ if (!Number.isSafeInteger(concurrency) || concurrency < 1) throw new Error("Migration concurrency must be a positive integer");
4790
+ };
4785
4791
  /**
4786
4792
  * Migrate an entity record in the database
4787
4793
  */
4788
- const migrateRecord = async ({ record, config, beforeMigrateRecord, afterMigrateRecord }) => {
4794
+ const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord, afterMigrateRecord }) => {
4789
4795
  const { dry } = config;
4790
4796
  console.log(chalk.gray(` ${record.name}`));
4791
4797
  try {
@@ -4799,6 +4805,7 @@ const migrateRecord = async ({ record, config, beforeMigrateRecord, afterMigrate
4799
4805
  });
4800
4806
  const { nextPayload: nextData, nextVersion } = await migrate({
4801
4807
  payload,
4808
+ preparedPlan,
4802
4809
  config: {
4803
4810
  ...config,
4804
4811
  version: currentVersion,
@@ -4830,26 +4837,32 @@ const migrateRecord = async ({ record, config, beforeMigrateRecord, afterMigrate
4830
4837
  return migrationErrorPayload;
4831
4838
  }
4832
4839
  };
4833
- const migrateRecords = async ({ recordList, config, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
4834
- const { dry } = config;
4840
+ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
4841
+ const { dry, concurrency = 1 } = config;
4842
+ validateConcurrency(concurrency);
4835
4843
  const migrationErrors = [];
4836
4844
  if (recordList.length) {
4837
4845
  console.log(chalk.gray("Executing record migration scripts..."));
4838
- for await (const record of recordList) {
4839
- let attempt = 0;
4840
- while (attempt < MAX_ATTEMPTS) {
4841
- attempt += 1;
4842
- if (attempt > 1) console.log(chalk.gray("Retrying migration..."));
4843
- const migrationErrorPayload = await migrateRecord({
4844
- record,
4845
- config,
4846
- beforeMigrateRecord,
4847
- afterMigrateRecord
4848
- });
4849
- if (!migrationErrorPayload) break;
4850
- if (migrationErrorPayload && attempt === MAX_ATTEMPTS) migrationErrors.push(migrationErrorPayload);
4846
+ const records = recordList.values();
4847
+ const worker = async () => {
4848
+ for (const record of records) {
4849
+ let attempt = 0;
4850
+ while (attempt < MAX_ATTEMPTS) {
4851
+ attempt += 1;
4852
+ if (attempt > 1) console.log(chalk.gray("Retrying migration..."));
4853
+ const migrationErrorPayload = await migrateRecord({
4854
+ record,
4855
+ config,
4856
+ preparedPlan,
4857
+ beforeMigrateRecord,
4858
+ afterMigrateRecord
4859
+ });
4860
+ if (!migrationErrorPayload) break;
4861
+ if (migrationErrorPayload && attempt === MAX_ATTEMPTS) migrationErrors.push(migrationErrorPayload);
4862
+ }
4851
4863
  }
4852
- }
4864
+ };
4865
+ await Promise.all(Array.from({ length: Math.min(concurrency, recordList.length) }, worker));
4853
4866
  }
4854
4867
  if (migrationErrors.length) {
4855
4868
  try {
@@ -4867,9 +4880,12 @@ const migrateRecords = async ({ recordList, config, beforeMigrateRecord, afterMi
4867
4880
  * Migrate all records (only use via command line interface)
4868
4881
  */
4869
4882
  const migrateAll = async ({ config, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
4870
- const { validatedPlan: latestPlan, nextVersion: latestVersion } = await getPlannedMigrations({ config });
4883
+ validateConcurrency(config.concurrency ?? 1);
4884
+ const preparedPlan = await getPlannedMigrations({ config });
4885
+ const { validatedPlan: latestPlan, nextVersion: latestVersion } = preparedPlan;
4871
4886
  await migrateRecords({
4872
4887
  config,
4888
+ preparedPlan,
4873
4889
  recordList: await getRecordList(config?.database, config?.entity, config?.entityTableName, config?.entityColumnNames, latestVersion, latestPlan),
4874
4890
  beforeMigrateRecord,
4875
4891
  afterMigrateRecord,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@oliasoft-open-source/node-json-migrator",
3
- "version": "5.3.0",
3
+ "version": "5.5.0-beta-1",
4
4
  "description": "A library for JSON migrations",
5
5
  "homepage": "https://oliasoft-open-source.gitlab.io/node-json-migrator",
6
6
  "bugs": {