@oliasoft-open-source/node-json-migrator 5.5.0-beta-1 → 5.5.0-beta-2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -174,6 +174,12 @@ and [merge requests](https://gitlab.com/oliasoft-open-source/node-json-migrator/
174
174
  Set it to a higher value to process independent records concurrently. Each record keeps its own transaction and retries,
175
175
  and all workers reuse the plan prepared at the start of the run. Record completion and log order may differ.
176
176
 
177
- Keep concurrency below the database pool size and leave capacity for other queries. Version history is currently loaded
178
- outside the record transaction, and callbacks may also need extra connections. Using every connection for record
179
- transactions can leave the run waiting indefinitely for a free connection.
177
+ Each record uses one transaction for its database reads, version-history lookup/cache writes, and update.
178
+ Callbacks `beforeMigrateRecord` and `afterMigrateRecord` must use their provided `transaction` for database operations,
179
+ rather than acquiring another connection from the pool.
180
+
181
+ Choose concurrency to fit the database pool and leave capacity for other application work. Callbacks that acquire
182
+ additional connections can still exhaust the pool and leave migrations waiting for a free connection.
183
+
184
+ When version history is missing from the database, records share a cached Git history lookup per migration run.
185
+ The lookup is lazy. Failed lookups are removed from the cache so retries can try again. A new run starts with an empty cache.
package/dist/index.d.mts CHANGED
@@ -55,7 +55,7 @@ type TConfig = {
55
55
  printPendingFileNames?: boolean;
56
56
  force?: boolean;
57
57
  dry?: boolean;
58
- autoInitializeTable?: boolean; /** Maximum concurrent records. Defaults to 1. Must leave DB pool capacity for queries outside transactions. */
58
+ autoInitializeTable?: boolean; /** Maximum concurrent records. Defaults to 1. Leave DB pool capacity for other application work. */
59
59
  concurrency?: number;
60
60
  };
61
61
  type TBeforeMigrateRecordPayload = {
@@ -101,7 +101,6 @@ declare const getPlannedMigrations: ({
101
101
  }[];
102
102
  nextVersion: string;
103
103
  }>;
104
- type TPreparedPlan = Awaited<ReturnType<typeof getPlannedMigrations>>;
105
104
  //#endregion
106
105
  //#region src/plan/planned-version.d.ts
107
106
  declare const getPlannedVersion: ({
@@ -113,21 +112,15 @@ declare const getPlannedVersion: ({
113
112
  //#region src/migrate/migrate.d.ts
114
113
  /**
115
114
  * Migrates a payload
116
- *
117
- * There are two ways to use this function:
118
- * i) function looks up the pending migrations for you (default)
119
- * ii) reuse a prepared plan while resolving pending migrations for the current payload version
120
115
  */
121
116
  declare const migrate: ({
122
117
  payload,
123
118
  config,
124
- metaData,
125
- preparedPlan
119
+ metaData
126
120
  }: {
127
121
  payload: TPayload;
128
122
  config: TConfig;
129
123
  metaData?: TMetaData;
130
- preparedPlan?: TPreparedPlan;
131
124
  }) => Promise<{
132
125
  nextPayload: TPayload;
133
126
  nextVersion: string;
package/dist/index.mjs CHANGED
@@ -464,6 +464,20 @@ function xor(arr1, arr2) {
464
464
  return difference(union(arr1, arr2), intersection(arr1, arr2));
465
465
  }
466
466
  //#endregion
467
+ //#region node_modules/.pnpm/es-toolkit@1.45.1/node_modules/es-toolkit/dist/function/memoize.mjs
468
+ function memoize(fn, options = {}) {
469
+ const { cache = /* @__PURE__ */ new Map(), getCacheKey } = options;
470
+ const memoizedFn = function(arg) {
471
+ const key = getCacheKey ? getCacheKey(arg) : arg;
472
+ if (cache.has(key)) return cache.get(key);
473
+ const result = fn.call(this, arg);
474
+ cache.set(key, result);
475
+ return result;
476
+ };
477
+ memoizedFn.cache = cache;
478
+ return memoizedFn;
479
+ }
480
+ //#endregion
467
481
  //#region node_modules/.pnpm/es-toolkit@1.45.1/node_modules/es-toolkit/dist/function/noop.mjs
468
482
  function noop() {}
469
483
  //#endregion
@@ -4607,7 +4621,7 @@ const pipe = (migrations, payload, metaData) => structuredClone(migrations.reduc
4607
4621
  try {
4608
4622
  return m.migrator(v, metaData);
4609
4623
  } catch (error) {
4610
- throw new Error(errorMessage(m?.fileName, error.message, error.stack));
4624
+ throw new Error(errorMessage(m?.fileName, error.message, error?.stack));
4611
4625
  }
4612
4626
  }, payload));
4613
4627
  const PipeModule = { pipe };
@@ -4699,19 +4713,20 @@ const getHistoricalPlansFromGit = async (planFilePath) => {
4699
4713
  throw new Error("Unable to fetch plan.json history from git");
4700
4714
  }
4701
4715
  };
4716
+ const GitModule = { getHistoricalPlansFromGit };
4702
4717
  //#endregion
4703
4718
  //#region src/executed/executed.ts
4704
4719
  /**
4705
4720
  * Get the executed migrations for a given payload version
4706
4721
  */
4707
- const getExecutedMigrationsFromVersion = async (database, pgpHelpers, entity, directory, currentVersion, plannedVersion, plannedMigrations) => {
4722
+ const getExecutedMigrationsFromVersion = async (database, pgpHelpers, entity, directory, currentVersion, plannedVersion, plannedMigrations, loadHistoricalPlans = getHistoricalPlansFromGit) => {
4708
4723
  if (!currentVersion) return [];
4709
4724
  if (currentVersion === plannedVersion) return plannedMigrations;
4710
4725
  else {
4711
4726
  let planFromVersion = await getPlanFromVersion(database, entity, currentVersion);
4712
4727
  if (planFromVersion === null) {
4713
4728
  console.warn(chalk.yellow(`Version not found in ${entity}_version table, trying git history instead`));
4714
- const { historicalPlans } = await getHistoricalPlansFromGit(`${directory}/plan.json`);
4729
+ const { historicalPlans } = await loadHistoricalPlans(`${directory}/plan.json`);
4715
4730
  if (historicalPlans.length) {
4716
4731
  await insertVersions(database, pgpHelpers, entity, historicalPlans);
4717
4732
  planFromVersion = historicalPlans.find((p) => p.version === currentVersion)?.plan || null;
@@ -4749,10 +4764,11 @@ const getPendingMigrations = ({ plannedMigrations, executedMigrations, printPend
4749
4764
  migrator: m.migrator
4750
4765
  }));
4751
4766
  };
4752
- const getPendingMigrators = async ({ config, preparedPlan }) => {
4767
+ const getPendingMigrators = async ({ config, preparedPlan, transaction, loadHistoricalPlans }) => {
4753
4768
  const { directory, database, pgpHelpers, entity, version, printPendingFileNames, force } = config;
4754
4769
  const { plannedMigrations, nextVersion } = preparedPlan ?? await PlanModule.getPlannedMigrations({ config });
4755
- const executedMigrations = database ? await getExecutedMigrationsFromVersion(database, pgpHelpers, entity, directory, version, nextVersion, plannedMigrations) : [];
4770
+ const connection = transaction ?? database;
4771
+ const executedMigrations = connection ? await getExecutedMigrationsFromVersion(connection, pgpHelpers, entity, directory, version, nextVersion, plannedMigrations, loadHistoricalPlans) : [];
4756
4772
  return {
4757
4773
  pendingMigrators: getPendingMigrations({
4758
4774
  plannedMigrations,
@@ -4767,15 +4783,20 @@ const getPendingMigrators = async ({ config, preparedPlan }) => {
4767
4783
  //#region src/migrate/migrate.ts
4768
4784
  /**
4769
4785
  * Migrates a payload
4770
- *
4771
- * There are two ways to use this function:
4772
- * i) function looks up the pending migrations for you (default)
4773
- * ii) reuse a prepared plan while resolving pending migrations for the current payload version
4774
4786
  */
4775
- const migrate = async ({ payload, config, metaData, preparedPlan }) => {
4787
+ const migrate = async ({ payload, config, metaData }) => {
4788
+ return migrateWithPlan({
4789
+ payload,
4790
+ config,
4791
+ metaData
4792
+ });
4793
+ };
4794
+ const migrateWithPlan = async ({ payload, config, metaData, preparedPlan, transaction, loadHistoricalPlans }) => {
4776
4795
  const { pendingMigrators, nextVersion } = await getPendingMigrators({
4777
4796
  config,
4778
- preparedPlan
4797
+ preparedPlan,
4798
+ transaction,
4799
+ loadHistoricalPlans
4779
4800
  });
4780
4801
  return {
4781
4802
  nextPayload: pendingMigrators.length ? PipeModule.pipe(pendingMigrators, payload, metaData) : payload,
@@ -4791,7 +4812,7 @@ const validateConcurrency = (concurrency) => {
4791
4812
  /**
4792
4813
  * Migrate an entity record in the database
4793
4814
  */
4794
- const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord, afterMigrateRecord }) => {
4815
+ const migrateRecord = async ({ record, config, preparedPlan, loadHistoricalPlans, beforeMigrateRecord, afterMigrateRecord }) => {
4795
4816
  const { dry } = config;
4796
4817
  console.log(chalk.gray(` ${record.name}`));
4797
4818
  try {
@@ -4803,9 +4824,11 @@ const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord
4803
4824
  transaction,
4804
4825
  dry
4805
4826
  });
4806
- const { nextPayload: nextData, nextVersion } = await migrate({
4827
+ const { nextPayload: nextData, nextVersion } = await migrateWithPlan({
4807
4828
  payload,
4829
+ transaction,
4808
4830
  preparedPlan,
4831
+ loadHistoricalPlans,
4809
4832
  config: {
4810
4833
  ...config,
4811
4834
  version: currentVersion,
@@ -4831,9 +4854,9 @@ const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord
4831
4854
  [config.entityColumnNames.name]: record.name,
4832
4855
  [config.entityColumnNames.created]: record.created,
4833
4856
  error,
4834
- stackTrace: error.stack
4857
+ stackTrace: error?.stack
4835
4858
  };
4836
- console.log(chalk.red(` ${capitalize(config.entity)} migration with id ${record.id} failed:\n ${error.stack}`));
4859
+ console.log(chalk.red(` ${capitalize(config.entity)} migration with id ${record.id} failed:\n ${error?.stack}`));
4837
4860
  return migrationErrorPayload;
4838
4861
  }
4839
4862
  };
@@ -4841,6 +4864,14 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
4841
4864
  const { dry, concurrency = 1 } = config;
4842
4865
  validateConcurrency(concurrency);
4843
4866
  const migrationErrors = [];
4867
+ const loadHistoricalPlans = memoize(async (planFilePath) => {
4868
+ try {
4869
+ return await GitModule.getHistoricalPlansFromGit(planFilePath);
4870
+ } catch (error) {
4871
+ loadHistoricalPlans.cache.delete(planFilePath);
4872
+ throw error;
4873
+ }
4874
+ });
4844
4875
  if (recordList.length) {
4845
4876
  console.log(chalk.gray("Executing record migration scripts..."));
4846
4877
  const records = recordList.values();
@@ -4849,11 +4880,12 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
4849
4880
  let attempt = 0;
4850
4881
  while (attempt < MAX_ATTEMPTS) {
4851
4882
  attempt += 1;
4852
- if (attempt > 1) console.log(chalk.gray("Retrying migration..."));
4883
+ if (attempt > 1) console.log(chalk.gray(`Retrying migration of ${record.name} (attempt ${attempt}/${MAX_ATTEMPTS}).`));
4853
4884
  const migrationErrorPayload = await migrateRecord({
4854
4885
  record,
4855
4886
  config,
4856
4887
  preparedPlan,
4888
+ loadHistoricalPlans,
4857
4889
  beforeMigrateRecord,
4858
4890
  afterMigrateRecord
4859
4891
  });
@@ -4871,7 +4903,7 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
4871
4903
  dry
4872
4904
  });
4873
4905
  } catch (error) {
4874
- console.log(chalk.red(`Handling errors for ${config.entity} migrations failed:\n ${error.stack}`));
4906
+ console.log(chalk.red(`Handling errors for ${config.entity} migrations failed:\n ${error?.stack}`));
4875
4907
  }
4876
4908
  console.log(chalk.yellow(`Completed ${config?.entity ?? "entity"} migrations (some failed)${config?.dry ? " [dry-run, no output written]" : ""} ✓`));
4877
4909
  } else console.log(chalk.green(`Completed ${config?.entity ?? "entity"} migrations (success)${config?.dry ? " [dry-run, no output written]" : ""} ✓`));
@@ -4881,7 +4913,7 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
4881
4913
  */
4882
4914
  const migrateAll = async ({ config, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
4883
4915
  validateConcurrency(config.concurrency ?? 1);
4884
- const preparedPlan = await getPlannedMigrations({ config });
4916
+ const preparedPlan = await PlanModule.getPlannedMigrations({ config });
4885
4917
  const { validatedPlan: latestPlan, nextVersion: latestVersion } = preparedPlan;
4886
4918
  await migrateRecords({
4887
4919
  config,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@oliasoft-open-source/node-json-migrator",
3
- "version": "5.5.0-beta-1",
3
+ "version": "5.5.0-beta-2",
4
4
  "description": "A library for JSON migrations",
5
5
  "homepage": "https://oliasoft-open-source.gitlab.io/node-json-migrator",
6
6
  "bugs": {