@oliasoft-open-source/node-json-migrator 5.4.0-beta-1 → 5.5.0-beta-1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/index.d.mts +2 -1
- package/dist/index.mjs +25 -16
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -167,3 +167,13 @@ and [merge requests](https://gitlab.com/oliasoft-open-source/node-json-migrator/
|
|
|
167
167
|
- coverage reports are in `test/coverage` (not committed)
|
|
168
168
|
- to build: `npm run buld` (this transpile a production build to the `dist` directory)
|
|
169
169
|
- the build `dist` directory should be committed
|
|
170
|
+
|
|
171
|
+
## Concurrent database migrations
|
|
172
|
+
|
|
173
|
+
`migrateAll()` accepts `config.concurrency`, a positive integer that defaults to `1` (sequential processing).
|
|
174
|
+
Set it to a higher value to process independent records concurrently. Each record keeps its own transaction and retries,
|
|
175
|
+
and all workers reuse the plan prepared at the start of the run. Record completion and log order may differ.
|
|
176
|
+
|
|
177
|
+
Keep concurrency below the database pool size and leave capacity for other queries. Version history is currently loaded
|
|
178
|
+
outside the record transaction, and callbacks may also need extra connections. Using every connection for record
|
|
179
|
+
transactions can leave the run waiting indefinitely for a free connection.
|
package/dist/index.d.mts
CHANGED
|
@@ -55,7 +55,8 @@ type TConfig = {
|
|
|
55
55
|
printPendingFileNames?: boolean;
|
|
56
56
|
force?: boolean;
|
|
57
57
|
dry?: boolean;
|
|
58
|
-
autoInitializeTable?: boolean;
|
|
58
|
+
autoInitializeTable?: boolean; /** Maximum concurrent records. Defaults to 1. Must leave DB pool capacity for queries outside transactions. */
|
|
59
|
+
concurrency?: number;
|
|
59
60
|
};
|
|
60
61
|
type TBeforeMigrateRecordPayload = {
|
|
61
62
|
currentRecord: TRecord;
|
package/dist/index.mjs
CHANGED
|
@@ -4785,6 +4785,9 @@ const migrate = async ({ payload, config, metaData, preparedPlan }) => {
|
|
|
4785
4785
|
//#endregion
|
|
4786
4786
|
//#region src/migrate-all/migrate-all.ts
|
|
4787
4787
|
const MAX_ATTEMPTS = 3;
|
|
4788
|
+
const validateConcurrency = (concurrency) => {
|
|
4789
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1) throw new Error("Migration concurrency must be a positive integer");
|
|
4790
|
+
};
|
|
4788
4791
|
/**
|
|
4789
4792
|
* Migrate an entity record in the database
|
|
4790
4793
|
*/
|
|
@@ -4835,26 +4838,31 @@ const migrateRecord = async ({ record, config, preparedPlan, beforeMigrateRecord
|
|
|
4835
4838
|
}
|
|
4836
4839
|
};
|
|
4837
4840
|
const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
|
|
4838
|
-
const { dry } = config;
|
|
4841
|
+
const { dry, concurrency = 1 } = config;
|
|
4842
|
+
validateConcurrency(concurrency);
|
|
4839
4843
|
const migrationErrors = [];
|
|
4840
4844
|
if (recordList.length) {
|
|
4841
4845
|
console.log(chalk.gray("Executing record migration scripts..."));
|
|
4842
|
-
|
|
4843
|
-
|
|
4844
|
-
|
|
4845
|
-
attempt
|
|
4846
|
-
|
|
4847
|
-
|
|
4848
|
-
|
|
4849
|
-
|
|
4850
|
-
|
|
4851
|
-
|
|
4852
|
-
|
|
4853
|
-
|
|
4854
|
-
|
|
4855
|
-
|
|
4846
|
+
const records = recordList.values();
|
|
4847
|
+
const worker = async () => {
|
|
4848
|
+
for (const record of records) {
|
|
4849
|
+
let attempt = 0;
|
|
4850
|
+
while (attempt < MAX_ATTEMPTS) {
|
|
4851
|
+
attempt += 1;
|
|
4852
|
+
if (attempt > 1) console.log(chalk.gray("Retrying migration..."));
|
|
4853
|
+
const migrationErrorPayload = await migrateRecord({
|
|
4854
|
+
record,
|
|
4855
|
+
config,
|
|
4856
|
+
preparedPlan,
|
|
4857
|
+
beforeMigrateRecord,
|
|
4858
|
+
afterMigrateRecord
|
|
4859
|
+
});
|
|
4860
|
+
if (!migrationErrorPayload) break;
|
|
4861
|
+
if (migrationErrorPayload && attempt === MAX_ATTEMPTS) migrationErrors.push(migrationErrorPayload);
|
|
4862
|
+
}
|
|
4856
4863
|
}
|
|
4857
|
-
}
|
|
4864
|
+
};
|
|
4865
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, recordList.length) }, worker));
|
|
4858
4866
|
}
|
|
4859
4867
|
if (migrationErrors.length) {
|
|
4860
4868
|
try {
|
|
@@ -4872,6 +4880,7 @@ const migrateRecords = async ({ recordList, config, preparedPlan, beforeMigrateR
|
|
|
4872
4880
|
* Migrate all records (only use via command line interface)
|
|
4873
4881
|
*/
|
|
4874
4882
|
const migrateAll = async ({ config, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors }) => {
|
|
4883
|
+
validateConcurrency(config.concurrency ?? 1);
|
|
4875
4884
|
const preparedPlan = await getPlannedMigrations({ config });
|
|
4876
4885
|
const { validatedPlan: latestPlan, nextVersion: latestVersion } = preparedPlan;
|
|
4877
4886
|
await migrateRecords({
|
package/package.json
CHANGED