@oliasoft-open-source/node-json-migrator 5.5.0-beta-1 → 5.5.0-beta-2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -3
- package/dist/index.d.mts +2 -9
- package/dist/index.mjs +50 -18
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -174,6 +174,12 @@ and [merge requests](https://gitlab.com/oliasoft-open-source/node-json-migrator/
|
|
|
174
174
|
Set it to a higher value to process independent records concurrently. Each record keeps its own transaction and retries,
|
|
175
175
|
and all workers reuse the plan prepared at the start of the run. Record completion and log order may differ.
|
|
176
176
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
177
|
+
Each record uses one transaction for its database reads, version-history lookup/cache writes, and update.
|
|
178
|
+
Callbacks `beforeMigrateRecord` and `afterMigrateRecord` must use their provided `transaction` for database operations,
|
|
179
|
+
rather than acquiring another connection from the pool.
|
|
180
|
+
|
|
181
|
+
Choose concurrency to fit the database pool and leave capacity for other application work. Callbacks that acquire
|
|
182
|
+
additional connections can still exhaust the pool and leave migrations waiting for a free connection.
|
|
183
|
+
|
|
184
|
+
When version history is missing from the database, records share a cached Git history lookup per migration run.
|
|
185
|
+
The lookup is lazy. Failed lookups are removed from the cache so retries can try again. A new run starts with an empty cache.
|
package/dist/index.d.mts
CHANGED
|
@@ -55,7 +55,7 @@ type TConfig = {
|
|
|
55
55
|
printPendingFileNames?: boolean;
|
|
56
56
|
force?: boolean;
|
|
57
57
|
dry?: boolean;
|
|
58
|
-
autoInitializeTable?: boolean; /** Maximum concurrent records. Defaults to 1.
|
|
58
|
+
autoInitializeTable?: boolean; /** Maximum concurrent records. Defaults to 1. Leave DB pool capacity for other application work. */
|
|
59
59
|
concurrency?: number;
|
|
60
60
|
};
|
|
61
61
|
type TBeforeMigrateRecordPayload = {
|
|
@@ -101,7 +101,6 @@ declare const getPlannedMigrations: ({
|
|
|
101
101
|
}[];
|
|
102
102
|
nextVersion: string;
|
|
103
103
|
}>;
|
|
104
|
-
type TPreparedPlan = Awaited<ReturnType<typeof getPlannedMigrations>>;
|
|
105
104
|
//#endregion
|
|
106
105
|
//#region src/plan/planned-version.d.ts
|
|
107
106
|
declare const getPlannedVersion: ({
|
|
@@ -113,21 +112,15 @@ declare const getPlannedVersion: ({
|
|
|
113
112
|
//#region src/migrate/migrate.d.ts
|
|
114
113
|
/**
|
|
115
114
|
* Migrates a payload
|
|
116
|
-
*
|
|
117
|
-
* There are two ways to use this function:
|
|
118
|
-
* i) function looks up the pending migrations for you (default)
|
|
119
|
-
* ii) reuse a prepared plan while resolving pending migrations for the current payload version
|
|
120
115
|
*/
|
|
121
116
|
declare const migrate: ({
|
|
122
117
|
payload,
|
|
123
118
|
config,
|
|
124
|
-
metaData
|
|
125
|
-
preparedPlan
|
|
119
|
+
metaData
|
|
126
120
|
}: {
|
|
127
121
|
payload: TPayload;
|
|
128
122
|
config: TConfig;
|
|
129
123
|
metaData?: TMetaData;
|
|
130
|
-
preparedPlan?: TPreparedPlan;
|
|
131
124
|
}) => Promise<{
|
|
132
125
|
nextPayload: TPayload;
|
|
133
126
|
nextVersion: string;
|
package/dist/index.mjs
CHANGED
|
@@ -464,6 +464,20 @@ function xor(arr1, arr2) {
|
|
|
464
464
|
return difference(union(arr1, arr2), intersection(arr1, arr2));
|
|
465
465
|
}
|
|
466
466
|
//#endregion
|
|
467
|
+
//#region node_modules/.pnpm/es-toolkit@1.45.1/node_modules/es-toolkit/dist/function/memoize.mjs
|
|
468
|
+
function memoize(fn, options = {}) {
|
|
469
|
+
const { cache = /* @__PURE__ */ new Map(), getCacheKey } = options;
|
|
470
|
+
const memoizedFn = function(arg) {
|
|
471
|
+
const key = getCacheKey ? getCacheKey(arg) : arg;
|
|
472
|
+
if (cache.has(key)) return cache.get(key);
|
|
473
|
+
const result = fn.call(this, arg);
|
|
474
|
+
cache.set(key, result);
|
|
475
|
+
return result;
|
|
476
|
+
};
|
|
477
|
+
memoizedFn.cache = cache;
|
|
478
|
+
return memoizedFn;
|
|
479
|
+
}
|
|
480
|
+
//#endregion
|
|
467
481
|
//#region node_modules/.pnpm/es-toolkit@1.45.1/node_modules/es-toolkit/dist/function/noop.mjs
|
|
468
482
|
function noop() {}
|
|
469
483
|
//#endregion
|
|
@@ -4607,7 +4621,7 @@ const pipe = (migrations, payload, metaData) => structuredClone(migrations.reduc
|
|
|
4607
4621
|
try {
|
|
4608
4622
|
return m.migrator(v, metaData);
|
|
4609
4623
|
} catch (error) {
|
|
4610
|
-
throw new Error(errorMessage(m?.fileName, error.message, error
|
|
4624
|
+
throw new Error(errorMessage(m?.fileName, error.message, error?.stack));
|
|
4611
4625
|
}
|
|
4612
4626
|
}, payload));
|
|
4613
4627
|
const PipeModule = { pipe };
|
|
@@ -4699,19 +4713,20 @@ const getHistoricalPlansFromGit = async (planFilePath) => {
|
|
|
4699
4713
|
throw new Error("Unable to fetch plan.json history from git");
|
|
4700
4714
|
}
|
|
4701
4715
|
};
|
|
4716
|
+
const GitModule = { getHistoricalPlansFromGit };
|
|
4702
4717
|
//#endregion
|
|
4703
4718
|
//#region src/executed/executed.ts
|
|
4704
4719
|
/**
|
|
4705
4720
|
* Get the executed migrations for a given payload version
|
|
4706
4721
|
*/
|
|
4707
|
-
const getExecutedMigrationsFromVersion = async (database, pgpHelpers, entity, directory, currentVersion, plannedVersion, plannedMigrations) => {
|
|
4722
|
+
const getExecutedMigrationsFromVersion = async (database, pgpHelpers, entity, directory, currentVersion, plannedVersion, plannedMigrations, loadHistoricalPlans = getHistoricalPlansFromGit) => {
|
|
4708
4723
|
if (!currentVersion) return [];
|
|
4709
4724
|
if (currentVersion === plannedVersion) return plannedMigrations;
|
|
4710
4725
|
else {
|
|
4711
4726
|
let planFromVersion = await getPlanFromVersion(database, entity, currentVersion);
|
|
4712
4727
|
if (planFromVersion === null) {
|
|
4713
4728
|
console.warn(chalk.yellow(`Version not found in ${entity}_version table, trying git history instead`));
|
|
4714
|
-
const { historicalPlans } = await
|
|
4729
|
+
const { historicalPlans } = await loadHistoricalPlans(`${directory}/plan.json`);
|
|
4715
4730
|
if (historicalPlans.length) {
|
|
4716
4731
|
await insertVersions(database, pgpHelpers, entity, historicalPlans);
|
|
4717
4732
|
planFromVersion = historicalPlans.find((p) => p.version === currentVersion)?.plan || null;
|
|
@@ -4749,10 +4764,11 @@ const getPendingMigrations = ({ plannedMigrations, executedMigrations, printPend
|
|
|
4749
4764
|
migrator: m.migrator
|
|
4750
4765
|
}));
|
|
4751
4766
|
};
|
|
4752
|
-
const getPendingMigrators = async ({ config, preparedPlan }) => {
|
|
4767
|
+
const getPendingMigrators = async ({ config, preparedPlan, transaction, loadHistoricalPlans }) => {
|
|
4753
4768
|
const { directory, database, pgpHelpers, entity, version, printPendingFileNames, force } = config;
|
|
4754
4769
|
const { plannedMigrations, nextVersion } = preparedPlan ?? await PlanModule.getPlannedMigrations({ config });
|
|
4755
|
-
const
|
|
4770
|
+
const connection = transaction ?? database;
|
|
4771
|
+
const executedMigrations = connection ? await getExecutedMigrationsFromVersion(connection, pgpHelpers, entity, directory, version, nextVersion, plannedMigrations, loadHistoricalPlans) : [];
|
|
4756
4772
|
return {
|
|
4757
4773
|
pendingMigrators: getPendingMigrations({
|
|
4758
4774
|
plannedMigrations,
|
|
@@ -4767,15 +4783,20 @@ const getPendingMigrators = async ({ config, preparedPlan }) => {
|
|
|
4767
4783
|
//#region src/migrate/migrate.ts
|
|
4768
4784
|
/**
|
|
4769
4785
|
* Migrates a payload
|
|
4770
|
-
*
|
|
4771
|
-
* There are two ways to use this function:
|
|
4772
|
-
* i) function looks up the pending migrations for you (default)
|
|
4773
|
-
* ii) reuse a prepared plan while resolving pending migrations for the current payload version
|
|
4774
4786
|
*/
|
|
4775
|
-
const migrate = async ({ payload, config, metaData
|
|
4787
|
+
const migrate = async ({ payload, config, metaData }) => {
|
|
4788
|
+
return migrateWithPlan({
|
|
4789
|
+
payload,
|
|
4790
|
+
config,
|
|
4791
|
+
metaData
|
|
4792
|
+
});
|
|
4793
|
+
};
|
|
4794
|
+
const migrateWithPlan = async ({ payload, config, metaData, preparedPlan, transaction, loadHistoricalPlans }) => {
|
|
4776
4795
|
const { pendingMigrators, nextVersion } = await getPendingMigrators({
|
|
4777
4796
|
config,
|
|
4778
|
-
preparedPlan
|
|
4797
|
+
preparedPlan,
|
|
4798
|
+
transaction,
|
|
4799
|
+
loadHistoricalPlans
|
|
4779
4800
|
});
|
|
4780
4801
|
return {
|
|
4781
4802
|
nextPayload: pendingMigrators.length ? PipeModule.pipe(pendingMigrators, payload, metaData) : payload,
|
|
@@ -4791,7 +4812,7 @@ const validateConcurrency = (concurrency) => {
|
|
|
4791
4812
|
/**
|
|
4792
4813
|
* Migrate an entity record in the database
|
|
4793
4814
|
*/
|
|
4794
|
-
const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord, afterMigrateRecord }) => {
|
|
4815
|
+
const migrateRecord = async ({ record, config, preparedPlan, loadHistoricalPlans, beforeMigrateRecord, afterMigrateRecord }) => {
|
|
4795
4816
|
const { dry } = config;
|
|
4796
4817
|
console.log(chalk.gray(` ${record.name}`));
|
|
4797
4818
|
try {
|
|
@@ -4803,9 +4824,11 @@ const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord
|
|
|
4803
4824
|
transaction,
|
|
4804
4825
|
dry
|
|
4805
4826
|
});
|
|
4806
|
-
const { nextPayload: nextData, nextVersion } = await
|
|
4827
|
+
const { nextPayload: nextData, nextVersion } = await migrateWithPlan({
|
|
4807
4828
|
payload,
|
|
4829
|
+
transaction,
|
|
4808
4830
|
preparedPlan,
|
|
4831
|
+
loadHistoricalPlans,
|
|
4809
4832
|
config: {
|
|
4810
4833
|
...config,
|
|
4811
4834
|
version: currentVersion,
|
|
@@ -4831,9 +4854,9 @@ const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord
|
|
|
4831
4854
|
[config.entityColumnNames.name]: record.name,
|
|
4832
4855
|
[config.entityColumnNames.created]: record.created,
|
|
4833
4856
|
error,
|
|
4834
|
-
stackTrace: error
|
|
4857
|
+
stackTrace: error?.stack
|
|
4835
4858
|
};
|
|
4836
|
-
console.log(chalk.red(` ${capitalize(config.entity)} migration with id ${record.id} failed:\n ${error
|
|
4859
|
+
console.log(chalk.red(` ${capitalize(config.entity)} migration with id ${record.id} failed:\n ${error?.stack}`));
|
|
4837
4860
|
return migrationErrorPayload;
|
|
4838
4861
|
}
|
|
4839
4862
|
};
|
|
@@ -4841,6 +4864,14 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
|
|
|
4841
4864
|
const { dry, concurrency = 1 } = config;
|
|
4842
4865
|
validateConcurrency(concurrency);
|
|
4843
4866
|
const migrationErrors = [];
|
|
4867
|
+
const loadHistoricalPlans = memoize(async (planFilePath) => {
|
|
4868
|
+
try {
|
|
4869
|
+
return await GitModule.getHistoricalPlansFromGit(planFilePath);
|
|
4870
|
+
} catch (error) {
|
|
4871
|
+
loadHistoricalPlans.cache.delete(planFilePath);
|
|
4872
|
+
throw error;
|
|
4873
|
+
}
|
|
4874
|
+
});
|
|
4844
4875
|
if (recordList.length) {
|
|
4845
4876
|
console.log(chalk.gray("Executing record migration scripts..."));
|
|
4846
4877
|
const records = recordList.values();
|
|
@@ -4849,11 +4880,12 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
|
|
|
4849
4880
|
let attempt = 0;
|
|
4850
4881
|
while (attempt < MAX_ATTEMPTS) {
|
|
4851
4882
|
attempt += 1;
|
|
4852
|
-
if (attempt > 1) console.log(chalk.gray(
|
|
4883
|
+
if (attempt > 1) console.log(chalk.gray(`Retrying migration of ${record.name} (attempt ${attempt}/${MAX_ATTEMPTS}).`));
|
|
4853
4884
|
const migrationErrorPayload = await migrateRecord({
|
|
4854
4885
|
record,
|
|
4855
4886
|
config,
|
|
4856
4887
|
preparedPlan,
|
|
4888
|
+
loadHistoricalPlans,
|
|
4857
4889
|
beforeMigrateRecord,
|
|
4858
4890
|
afterMigrateRecord
|
|
4859
4891
|
});
|
|
@@ -4871,7 +4903,7 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
|
|
|
4871
4903
|
dry
|
|
4872
4904
|
});
|
|
4873
4905
|
} catch (error) {
|
|
4874
|
-
console.log(chalk.red(`Handling errors for ${config.entity} migrations failed:\n ${error
|
|
4906
|
+
console.log(chalk.red(`Handling errors for ${config.entity} migrations failed:\n ${error?.stack}`));
|
|
4875
4907
|
}
|
|
4876
4908
|
console.log(chalk.yellow(`Completed ${config?.entity ?? "entity"} migrations (some failed)${config?.dry ? " [dry-run, no output written]" : ""} ✓`));
|
|
4877
4909
|
} else console.log(chalk.green(`Completed ${config?.entity ?? "entity"} migrations (success)${config?.dry ? " [dry-run, no output written]" : ""} ✓`));
|
|
@@ -4881,7 +4913,7 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
|
|
|
4881
4913
|
*/
|
|
4882
4914
|
const migrateAll = async ({ config, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
|
|
4883
4915
|
validateConcurrency(config.concurrency ?? 1);
|
|
4884
|
-
const preparedPlan = await getPlannedMigrations({ config });
|
|
4916
|
+
const preparedPlan = await PlanModule.getPlannedMigrations({ config });
|
|
4885
4917
|
const { validatedPlan: latestPlan, nextVersion: latestVersion } = preparedPlan;
|
|
4886
4918
|
await migrateRecords({
|
|
4887
4919
|
config,
|
package/package.json
CHANGED