@oliasoft-open-source/node-json-migrator 3.1.0-beta-1 → 3.1.0-beta-3

package/dist/index.cjs

@@ -9027,6 +9027,77 @@ const getVersions = async (db, entity) => {
   };
   return db.any(query, params);
 };
+const getRecordList = async (db, entity, entityTableName, entityColumnNames, latestVersion, latestVersionPlanLength) => {
+  const query = `
+    SELECT
+      $(entityIdColumn:name) AS id,
+      name,
+      $(entityCreatedColumn:name) AS created,
+      version
+    FROM $(entityTable:name)
+    WHERE (
+      version IS NULL
+      OR (
+        version <> $(latestVersion)
+        AND (
+          SELECT
+            CASE
+              WHEN IS_JSON(plan) THEN JSON_ARRAY_LENGTH(plan::json)
+            ELSE 0
+            END AS length
+          FROM $(entityVersionsTable:name)
+          WHERE $(entityVersionsTable:name).version = version
+          LIMIT 1
+        )::integer <= $(latestVersionPlanLength)
+      )
+    )
+  `;
+  const params = {
+    entityTable: entityTableName,
+    entityVersionsTable: `${entity}_versions`,
+    entityIdColumn: entityColumnNames.id,
+    entityCreatedColumn: entityColumnNames.created,
+    latestVersion,
+    latestVersionPlanLength
+  };
+  return db.any(query, params);
+};
+const getRecord = async (t, entityTableName, entityColumnNames, entityId) => {
+  const query = `
+    SELECT *
+    FROM $(entityTable:name)
+    WHERE $(entityIdColumn:name) = $(entityId)
+  `;
+  const params = {
+    entityTable: entityTableName,
+    entityId,
+    entityIdColumn: entityColumnNames.id
+  };
+  return t.one(query, params);
+};
+const updateRecord = async (t, entityTableName, entityColumnNames, entityId, currentVersion, nextData, nextVersion, etag) => {
+  const useEtag = typeof etag === "number";
+  const query = `
+    UPDATE $(entityTable:name)
+    SET data = $(nextData),
+        version = $(nextVersion)
+        ${useEtag ? ", etag = etag + 1" : ""}
+    WHERE $(entityIdColumn:name) = $(entityId)
+      AND version = $(currentVersion)
+      ${useEtag ? "AND etag = $(etag)" : ""}
+    RETURNING *
+  `;
+  const params = {
+    entityTable: entityTableName,
+    entityId,
+    entityIdColumn: entityColumnNames.id,
+    currentVersion,
+    nextData,
+    nextVersion,
+    etag
+  };
+  return t.oneOrNone(query, params);
+};
 
 let cachedPlannedVersion = null;
 let cachedMigratorFunctions = [];
@@ -9589,6 +9660,146 @@ const migrate = async ({
   };
 };
 
+const migrateRecord = async ({
+  record,
+  config,
+  beforeMigrateRecord,
+  afterMigrateRecord
+}) => {
+  const { dry } = config;
+  console.log(chalk.gray(` ${record.name}`));
+  try {
+    await config.database.tx(async (transaction) => {
+      const currentRecord = await getRecord(
+        transaction,
+        config.entityTableName,
+        config.entityColumnNames,
+        record.id
+      );
+      const { data, etag, version: currentVersion } = currentRecord;
+      await beforeMigrateRecord({
+        currentRecord,
+        dry
+      });
+      const { nextPayload: nextData, nextVersion } = await migrate({
+        payload: data,
+        config: {
+          ...config,
+          version: currentVersion,
+          printPendingFileNames: true
+        }
+      });
+      if (!dry) {
+        const nextRecord = await updateRecord(
+          transaction,
+          config?.entityTableName,
+          config?.entityColumnNames,
+          record.id,
+          currentVersion,
+          nextData,
+          nextVersion,
+          etag
+        );
+        if (!nextRecord) {
+          throw new Error(
+            "Failed to patch record with newest migration - The record was probably updated while migrating..."
+          );
+        }
+        await afterMigrateRecord({
+          currentRecord,
+          nextRecord,
+          transaction,
+          dry
+        });
+      }
+    });
+    return null;
+  } catch (error) {
+    const migrationErrorPayload = {
+      id: record.id,
+      name: record.name,
+      created: record.created,
+      error,
+      stackTrace: error.stack
+    };
+    console.log(
+      chalk.red(
+        ` Dataset migration with id ${record.id} failed:
+ ${error.stack}`
+      )
+    );
+    return migrationErrorPayload;
+  }
+};
+const migrateRecords = async ({
+  recordList,
+  config,
+  beforeMigrateRecord,
+  afterMigrateRecord,
+  onMigrationErrors
+}) => {
+  const { dry } = config;
+  const migrationErrors = [];
+  if (recordList.length) {
+    console.log(chalk.gray("Executing record migration scripts..."));
+    for await (const record of recordList) {
+      const migrationErrorPayload = await migrateRecord({
+        record,
+        config,
+        beforeMigrateRecord,
+        afterMigrateRecord
+      });
+      if (migrationErrorPayload) {
+        migrationErrors.push(migrationErrorPayload);
+      }
+    }
+  }
+  if (migrationErrors.length) {
+    console.log(
+      chalk.yellow(
+        `Completed ${config?.entity ?? "entity"} migrations (some failed)${config?.dry ? " [dry-run, no output written]" : ""} \u2713`
+      )
+    );
+    await onMigrationErrors({
+      migrationErrors,
+      dry
+    });
+  } else {
+    console.log(
+      chalk.green(
+        `Completed ${config?.entity ?? "entity"} migrations (success)${config?.dry ? " [dry-run, no output written]" : ""} \u2713`
+      )
+    );
+  }
+};
+const migrateAll = async ({
+  config,
+  beforeMigrateRecord,
+  afterMigrateRecord,
+  onMigrationErrors
+}) => {
+  const plan = await getPlannedMigrations({
+    config
+  });
+  const latestVersion = plan.nextVersion;
+  const latestVersionPlanLength = plan.plannedMigrations.length;
+  const recordList = await getRecordList(
+    config?.database,
+    config?.entity,
+    config?.entityTableName,
+    config?.entityColumnNames,
+    latestVersion,
+    latestVersionPlanLength
+  );
+  await migrateRecords({
+    config,
+    recordList,
+    beforeMigrateRecord,
+    afterMigrateRecord,
+    onMigrationErrors
+  });
+};
+
 const parsePlan = (plan) => {
   return (() => {
     try {
@@ -9642,5 +9853,6 @@ exports.getPlannedMigrations = getPlannedMigrations;
 exports.getPlannedVersion = getPlannedVersion;
 exports.getVersions = getVersions;
 exports.migrate = migrate;
+exports.migrateAll = migrateAll;
 exports.pipe = pipe;
 exports.printVersionHistory = printVersionHistory;

package/dist/index.d.cts

@@ -1,4 +1,4 @@
-import { IDatabase, IHelpers } from 'pg-promise';
+import { IDatabase, IHelpers, ITask } from 'pg-promise';
 
 /**
  * Generate a new dataset migration
@@ -21,13 +21,21 @@ type TEntityColumnNames = {
     id: string;
     created: string;
 };
+type TRecord<T extends TEntityColumnNames = TEntityColumnNames> = {
+    name: string;
+    version: string;
+} & {
+    [K in T['id']]: number | string;
+} & {
+    [K in T['created']]: string | Date;
+};
 type TConfig = {
     directory: string;
     database?: IDatabase<unknown>;
     pgpHelpers?: IHelpers;
     entity?: string;
-    entityColumnNames: TEntityColumnNames;
-    entityIdColumnName?: string;
+    entityTableName?: string;
+    entityColumnNames?: TEntityColumnNames;
     version?: string;
     printPendingFileNames?: boolean;
     force?: boolean;
@@ -35,6 +43,30 @@ type TConfig = {
     importModule?: boolean;
     autoInitializeTable?: boolean;
 };
+type TBeforeMigrateRecordPayload = {
+    currentRecord: TRecord;
+    dry: boolean;
+};
+type TBeforeMigrateRecord = (params: TBeforeMigrateRecordPayload) => Promise<void>;
+type TAfterMigrateRecordPayload = {
+    currentRecord: TRecord;
+    nextRecord: TRecord;
+    transaction?: ITask<unknown>;
+    dry: boolean;
+};
+type TAfterMigrateRecord = (params: TAfterMigrateRecordPayload) => Promise<void>;
+type TMigrationErrorPayload = {
+    id: string;
+    name: string;
+    created: string | Date;
+    error: unknown;
+    stackTrace?: string;
+};
+type TOnMigrationErrorsPayload = {
+    migrationErrors: TMigrationErrorPayload[];
+    dry: boolean;
+};
+type TOnMigrationErrors = (params: TOnMigrationErrorsPayload) => Promise<void>;
 
 declare const getPlannedMigrations: ({ config }: {
     config: TConfig;
@@ -62,6 +94,16 @@ declare const migrate: ({ payload, config, }: {
     nextVersion: string;
 }>;
 
+/**
+ * Migrate all records (only use via command line interface)
+ */
+declare const migrateAll: ({ config, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors, }: {
+    config: TConfig;
+    beforeMigrateRecord: TBeforeMigrateRecord;
+    afterMigrateRecord: TAfterMigrateRecord;
+    onMigrationErrors: TOnMigrationErrors;
+}) => Promise<void>;
+
 /**
  * Executes the pending migrators on a payload, returns modified payload
  *
@@ -84,4 +126,4 @@ declare const printVersionHistory: ({ config }: {
     config: TConfig;
 }) => Promise<void>;
 
-export { createMigration, getPlannedMigrations, getPlannedVersion, getVersions, migrate, pipe, printVersionHistory };
+export { type TAfterMigrateRecord, type TAfterMigrateRecordPayload, type TBeforeMigrateRecord, type TBeforeMigrateRecordPayload, type TConfig, type TEntityColumnNames, type TMigration, type TMigrationErrorPayload, type TOnMigrationErrors, type TOnMigrationErrorsPayload, type TRecord, createMigration, getPlannedMigrations, getPlannedVersion, getVersions, migrate, migrateAll, pipe, printVersionHistory };

package/dist/index.d.mts

@@ -1,4 +1,4 @@
-import { IDatabase, IHelpers } from 'pg-promise';
+import { IDatabase, IHelpers, ITask } from 'pg-promise';
 
 /**
  * Generate a new dataset migration
@@ -21,13 +21,21 @@ type TEntityColumnNames = {
     id: string;
     created: string;
 };
+type TRecord<T extends TEntityColumnNames = TEntityColumnNames> = {
+    name: string;
+    version: string;
+} & {
+    [K in T['id']]: number | string;
+} & {
+    [K in T['created']]: string | Date;
+};
 type TConfig = {
     directory: string;
     database?: IDatabase<unknown>;
     pgpHelpers?: IHelpers;
     entity?: string;
-    entityColumnNames: TEntityColumnNames;
-    entityIdColumnName?: string;
+    entityTableName?: string;
+    entityColumnNames?: TEntityColumnNames;
     version?: string;
     printPendingFileNames?: boolean;
     force?: boolean;
@@ -35,6 +43,30 @@ type TConfig = {
     importModule?: boolean;
     autoInitializeTable?: boolean;
 };
+type TBeforeMigrateRecordPayload = {
+    currentRecord: TRecord;
+    dry: boolean;
+};
+type TBeforeMigrateRecord = (params: TBeforeMigrateRecordPayload) => Promise<void>;
+type TAfterMigrateRecordPayload = {
+    currentRecord: TRecord;
+    nextRecord: TRecord;
+    transaction?: ITask<unknown>;
+    dry: boolean;
+};
+type TAfterMigrateRecord = (params: TAfterMigrateRecordPayload) => Promise<void>;
+type TMigrationErrorPayload = {
+    id: string;
+    name: string;
+    created: string | Date;
+    error: unknown;
+    stackTrace?: string;
+};
+type TOnMigrationErrorsPayload = {
+    migrationErrors: TMigrationErrorPayload[];
+    dry: boolean;
+};
+type TOnMigrationErrors = (params: TOnMigrationErrorsPayload) => Promise<void>;
 
 declare const getPlannedMigrations: ({ config }: {
     config: TConfig;
@@ -62,6 +94,16 @@ declare const migrate: ({ payload, config, }: {
     nextVersion: string;
 }>;
 
+/**
+ * Migrate all records (only use via command line interface)
+ */
+declare const migrateAll: ({ config, beforeMigrateRecord, afterMigrateRecord, onMigrationErrors, }: {
+    config: TConfig;
+    beforeMigrateRecord: TBeforeMigrateRecord;
+    afterMigrateRecord: TAfterMigrateRecord;
+    onMigrationErrors: TOnMigrationErrors;
+}) => Promise<void>;
+
 /**
  * Executes the pending migrators on a payload, returns modified payload
  *
@@ -84,4 +126,4 @@ declare const printVersionHistory: ({ config }: {
     config: TConfig;
 }) => Promise<void>;
 
-export { createMigration, getPlannedMigrations, getPlannedVersion, getVersions, migrate, pipe, printVersionHistory };
+export { type TAfterMigrateRecord, type TAfterMigrateRecordPayload, type TBeforeMigrateRecord, type TBeforeMigrateRecordPayload, type TConfig, type TEntityColumnNames, type TMigration, type TMigrationErrorPayload, type TOnMigrationErrors, type TOnMigrationErrorsPayload, type TRecord, createMigration, getPlannedMigrations, getPlannedVersion, getVersions, migrate, migrateAll, pipe, printVersionHistory };

package/dist/index.mjs

@@ -9006,6 +9006,77 @@ const getVersions = async (db, entity) => {
   };
   return db.any(query, params);
 };
+const getRecordList = async (db, entity, entityTableName, entityColumnNames, latestVersion, latestVersionPlanLength) => {
+  const query = `
+    SELECT
+      $(entityIdColumn:name) AS id,
+      name,
+      $(entityCreatedColumn:name) AS created,
+      version
+    FROM $(entityTable:name)
+    WHERE (
+      version IS NULL
+      OR (
+        version <> $(latestVersion)
+        AND (
+          SELECT
+            CASE
+              WHEN IS_JSON(plan) THEN JSON_ARRAY_LENGTH(plan::json)
+            ELSE 0
+            END AS length
+          FROM $(entityVersionsTable:name)
+          WHERE $(entityVersionsTable:name).version = version
+          LIMIT 1
+        )::integer <= $(latestVersionPlanLength)
+      )
+    )
+  `;
+  const params = {
+    entityTable: entityTableName,
+    entityVersionsTable: `${entity}_versions`,
+    entityIdColumn: entityColumnNames.id,
+    entityCreatedColumn: entityColumnNames.created,
+    latestVersion,
+    latestVersionPlanLength
+  };
+  return db.any(query, params);
+};
+const getRecord = async (t, entityTableName, entityColumnNames, entityId) => {
+  const query = `
+    SELECT *
+    FROM $(entityTable:name)
+    WHERE $(entityIdColumn:name) = $(entityId)
+  `;
+  const params = {
+    entityTable: entityTableName,
+    entityId,
+    entityIdColumn: entityColumnNames.id
+  };
+  return t.one(query, params);
+};
+const updateRecord = async (t, entityTableName, entityColumnNames, entityId, currentVersion, nextData, nextVersion, etag) => {
+  const useEtag = typeof etag === "number";
+  const query = `
+    UPDATE $(entityTable:name)
+    SET data = $(nextData),
+        version = $(nextVersion)
+        ${useEtag ? ", etag = etag + 1" : ""}
+    WHERE $(entityIdColumn:name) = $(entityId)
+      AND version = $(currentVersion)
+      ${useEtag ? "AND etag = $(etag)" : ""}
+    RETURNING *
+  `;
+  const params = {
+    entityTable: entityTableName,
+    entityId,
+    entityIdColumn: entityColumnNames.id,
+    currentVersion,
+    nextData,
+    nextVersion,
+    etag
+  };
+  return t.oneOrNone(query, params);
+};
 
 let cachedPlannedVersion = null;
 let cachedMigratorFunctions = [];
@@ -9568,6 +9639,146 @@ const migrate = async ({
   };
 };
 
+const migrateRecord = async ({
+  record,
+  config,
+  beforeMigrateRecord,
+  afterMigrateRecord
+}) => {
+  const { dry } = config;
+  console.log(chalk.gray(` ${record.name}`));
+  try {
+    await config.database.tx(async (transaction) => {
+      const currentRecord = await getRecord(
+        transaction,
+        config.entityTableName,
+        config.entityColumnNames,
+        record.id
+      );
+      const { data, etag, version: currentVersion } = currentRecord;
+      await beforeMigrateRecord({
+        currentRecord,
+        dry
+      });
+      const { nextPayload: nextData, nextVersion } = await migrate({
+        payload: data,
+        config: {
+          ...config,
+          version: currentVersion,
+          printPendingFileNames: true
+        }
+      });
+      if (!dry) {
+        const nextRecord = await updateRecord(
+          transaction,
+          config?.entityTableName,
+          config?.entityColumnNames,
+          record.id,
+          currentVersion,
+          nextData,
+          nextVersion,
+          etag
+        );
+        if (!nextRecord) {
+          throw new Error(
+            "Failed to patch record with newest migration - The record was probably updated while migrating..."
+          );
+        }
+        await afterMigrateRecord({
+          currentRecord,
+          nextRecord,
+          transaction,
+          dry
+        });
+      }
+    });
+    return null;
+  } catch (error) {
+    const migrationErrorPayload = {
+      id: record.id,
+      name: record.name,
+      created: record.created,
+      error,
+      stackTrace: error.stack
+    };
+    console.log(
+      chalk.red(
+        ` Dataset migration with id ${record.id} failed:
+ ${error.stack}`
+      )
+    );
+    return migrationErrorPayload;
+  }
+};
+const migrateRecords = async ({
+  recordList,
+  config,
+  beforeMigrateRecord,
+  afterMigrateRecord,
+  onMigrationErrors
+}) => {
+  const { dry } = config;
+  const migrationErrors = [];
+  if (recordList.length) {
+    console.log(chalk.gray("Executing record migration scripts..."));
+    for await (const record of recordList) {
+      const migrationErrorPayload = await migrateRecord({
+        record,
+        config,
+        beforeMigrateRecord,
+        afterMigrateRecord
+      });
+      if (migrationErrorPayload) {
+        migrationErrors.push(migrationErrorPayload);
+      }
+    }
+  }
+  if (migrationErrors.length) {
+    console.log(
+      chalk.yellow(
+        `Completed ${config?.entity ?? "entity"} migrations (some failed)${config?.dry ? " [dry-run, no output written]" : ""} \u2713`
+      )
+    );
+    await onMigrationErrors({
+      migrationErrors,
+      dry
+    });
+  } else {
+    console.log(
+      chalk.green(
+        `Completed ${config?.entity ?? "entity"} migrations (success)${config?.dry ? " [dry-run, no output written]" : ""} \u2713`
+      )
+    );
+  }
+};
+const migrateAll = async ({
+  config,
+  beforeMigrateRecord,
+  afterMigrateRecord,
+  onMigrationErrors
+}) => {
+  const plan = await getPlannedMigrations({
+    config
+  });
+  const latestVersion = plan.nextVersion;
+  const latestVersionPlanLength = plan.plannedMigrations.length;
+  const recordList = await getRecordList(
+    config?.database,
+    config?.entity,
+    config?.entityTableName,
+    config?.entityColumnNames,
+    latestVersion,
+    latestVersionPlanLength
+  );
+  await migrateRecords({
+    config,
+    recordList,
+    beforeMigrateRecord,
+    afterMigrateRecord,
+    onMigrationErrors
+  });
+};
+
 const parsePlan = (plan) => {
   return (() => {
     try {
@@ -9616,4 +9827,4 @@ const printVersionHistory = async ({ config }) => {
   }
 };
 
-export { createMigration, getPlannedMigrations, getPlannedVersion, getVersions, migrate, pipe, printVersionHistory };
+export { createMigration, getPlannedMigrations, getPlannedVersion, getVersions, migrate, migrateAll, pipe, printVersionHistory };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@oliasoft-open-source/node-json-migrator",
-  "version": "3.1.0-beta-1",
+  "version": "3.1.0-beta-3",
   "description": "A library for JSON migrations",
   "homepage": "https://oliasoft-open-source.gitlab.io/node-postgresql-migrator",
   "bugs": {