@webiny/migrations 5.39.7-beta.2

package/migrations/5.39.6/001/ddb-es/worker.js

@@ -0,0 +1,352 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
+var _utils = require("@webiny/utils");
+var _logger = require("@webiny/logger");
+var _dataMigration = require("@webiny/data-migration");
+var _clientDynamodb = require("@webiny/aws-sdk/client-dynamodb");
+var _apiElasticsearch = require("@webiny/api-elasticsearch");
+var _yargs = _interopRequireDefault(require("yargs/yargs"));
+var _helpers = require("yargs/helpers");
+var _isMigratedEntry = require("../../../5.39.0/001/utils/isMigratedEntry");
+var _hasValidTypeFieldValue = require("../../../5.39.0/001/utils/hasValidTypeFieldValue");
+var _hasAllNonNullableValues = require("../../../5.39.0/001/utils/hasAllNonNullableValues");
+var _getOldestRevisionCreatedOn = require("../../../5.39.0/001/utils/getOldestRevisionCreatedOn");
+var _getFirstLastPublishedOn = require("../../../5.39.0/001/utils/getFirstLastPublishedOn");
+var _assignNewMetaFields = require("../../../5.39.0/001/utils/assignNewMetaFields");
+var _fixTypeFieldValue = require("../../../5.39.0/001/utils/fixTypeFieldValue");
+var _getFallbackIdentity = require("../../../5.39.0/001/utils/getFallbackIdentity");
+var _ensureAllNonNullableValues = require("../../../5.39.0/001/utils/ensureAllNonNullableValues");
+var _getDecompressedData = require("../../../5.39.0/001/utils/getDecompressedData");
+var _getCompressedData = require("../../../5.39.0/001/utils/getCompressedData");
+var _createEntryEntity = require("../../../5.39.0/001/entities/createEntryEntity");
+var _utils2 = require("../../../../utils");
+var _waitUntilHealthy = require("@webiny/api-elasticsearch/utils/waitUntilHealthy");
+var _pinoPretty = _interopRequireDefault(require("pino-pretty"));
+var _path = _interopRequireDefault(require("path"));
+var _os = _interopRequireDefault(require("os"));
+var _fs = _interopRequireDefault(require("fs"));
+var _getNonNullableFieldsWithMissingValues = require("../../../5.39.0/001/utils/getNonNullableFieldsWithMissingValues");
+const argv = (0, _yargs.default)((0, _helpers.hideBin)(process.argv)).options({
+  runId: {
+    type: "string",
+    demandOption: true
+  },
+  ddbTable: {
+    type: "string",
+    demandOption: true
+  },
+  ddbEsTable: {
+    type: "string",
+    demandOption: true
+  },
+  esEndpoint: {
+    type: "string",
+    demandOption: true
+  },
+  segmentIndex: {
+    type: "number",
+    demandOption: true
+  },
+  totalSegments: {
+    type: "number",
+    demandOption: true
+  },
+  // Elasticsearch health check options.
+  esHealthMinClusterHealthStatus: {
+    type: "string",
+    demandOption: true
+  },
+  esHealthMaxProcessorPercent: {
+    type: "number",
+    demandOption: true
+  },
+  esHealthMaxRamPercent: {
+    type: "number",
+    demandOption: true
+  },
+  esHealthMaxWaitingTime: {
+    type: "number",
+    demandOption: true
+  },
+  esHealthWaitingTimeStep: {
+    type: "number",
+    demandOption: true
+  }
+}).parseSync();
+const createInitialStatus = () => {
+  return {
+    lastEvaluatedKey: null,
+    stats: {
+      iterationsCount: 0,
+      recordsScanned: 0,
+      recordsUpdated: 0,
+      recordsSkipped: 0,
+      esHealthChecks: {
+        timeSpentWaiting: 0,
+        checksCount: 0,
+        unhealthyReasons: {}
+      }
+    }
+  };
+};
+(async () => {
+  const logger = (0, _logger.createPinoLogger)({
+    level: (0, _logger.getLogLevel)(process.env.MIGRATIONS_LOG_LEVEL, "trace"),
+    msgPrefix: `[segment #${argv.segmentIndex}] `
+  }, (0, _pinoPretty.default)({
+    ignore: "pid,hostname"
+  }));
+  const documentClient = (0, _clientDynamodb.getDocumentClient)();
+  const elasticsearchClient = (0, _apiElasticsearch.createElasticsearchClient)({
+    endpoint: `https://${argv.esEndpoint}`
+  });
+  const primaryTable = (0, _dataMigration.createTable)({
+    name: argv.ddbTable,
+    documentClient
+  });
+  const dynamoToEsTable = (0, _dataMigration.createTable)({
+    name: argv.ddbEsTable,
+    documentClient
+  });
+  const ddbEntryEntity = (0, _createEntryEntity.createDdbEntryEntity)(primaryTable);
+  const ddbEsEntryEntity = (0, _createEntryEntity.createDdbEsEntryEntity)(dynamoToEsTable);
+  const status = createInitialStatus();
+  const waitUntilHealthy = (0, _waitUntilHealthy.createWaitUntilHealthy)(elasticsearchClient, {
+    minClusterHealthStatus: argv.esHealthMinClusterHealthStatus,
+    maxProcessorPercent: argv.esHealthMaxProcessorPercent,
+    maxRamPercent: argv.esHealthMaxRamPercent,
+    maxWaitingTime: argv.esHealthMaxWaitingTime,
+    waitingTimeStep: argv.esHealthWaitingTimeStep
+  });
+  await (0, _utils2.ddbScanWithCallback)({
+    entity: ddbEntryEntity,
+    options: {
+      segment: argv.segmentIndex,
+      segments: argv.totalSegments,
+      filters: [{
+        attr: "_et",
+        eq: "CmsEntries"
+      }],
+      startKey: status.lastEvaluatedKey || undefined,
+      limit: 100
+    }
+  }, async result => {
+    status.stats.iterationsCount++;
+    status.stats.recordsScanned += result.items.length;
+    if (status.stats.iterationsCount % 5 === 0) {
+      // We log every 5th iteration.
+      logger.trace(`[iteration #${status.stats.iterationsCount}] Reading ${result.items.length} record(s)...`);
+    }
+    const ddbItemsToBatchWrite = [];
+    const ddbEsItemsToBatchWrite = [];
+    const ddbEsItemsToBatchRead = {};
+    const fallbackDateTime = new Date().toISOString();
+
+    // Update records in primary DynamoDB table. Also do preparations for
+    // subsequent updates on DDB-ES DynamoDB table, and in Elasticsearch.
+    for (const item of result.items) {
+      const isFullyMigrated = (0, _isMigratedEntry.isMigratedEntry)(item) && (0, _hasValidTypeFieldValue.hasValidTypeFieldValue)(item) && (0, _hasAllNonNullableValues.hasAllNonNullableValues)(item);
+      if (isFullyMigrated) {
+        status.stats.recordsSkipped++;
+        continue;
+      }
+
+      // 1. Check if the data migration was ever performed. If not, let's perform it.
+      if (!(0, _isMigratedEntry.isMigratedEntry)(item)) {
+        // Get the oldest revision's `createdOn` value. We use that to set the entry-level `createdOn` value.
+        const createdOn = await (0, _getOldestRevisionCreatedOn.getOldestRevisionCreatedOn)({
+          entry: item,
+          entryEntity: ddbEntryEntity
+        });
+        const firstLastPublishedOnByFields = await (0, _getFirstLastPublishedOn.getFirstLastPublishedOnBy)({
+          entry: item,
+          entryEntity: ddbEntryEntity
+        });
+        (0, _assignNewMetaFields.assignNewMetaFields)(item, {
+          createdOn,
+          ...firstLastPublishedOnByFields
+        });
+      }
+
+      // 2. We've noticed some of the records had an invalid `TYPE` field value
+      //    in the database. This step addresses this issue.
+      if (!(0, _hasValidTypeFieldValue.hasValidTypeFieldValue)(item)) {
+        // Fixes the value of the `TYPE` field, if it's not valid.
+        (0, _fixTypeFieldValue.fixTypeFieldValue)(item);
+      }
+
+      // 3. Finally, once both of the steps were performed, ensure that all
+      //    new non-nullable meta fields have a value and nothing is missing.
+      if (!(0, _hasAllNonNullableValues.hasAllNonNullableValues)(item)) {
+        logger.trace((0, _getNonNullableFieldsWithMissingValues.getNonNullableFieldsWithMissingValues)(item), `Detected an entry with missing values for non-nullable meta fields (${item.modelId}/${item.id}).`);
+        try {
+          const fallbackIdentity = await (0, _getFallbackIdentity.getFallbackIdentity)({
+            entity: ddbEntryEntity,
+            tenant: item.tenant
+          });
+          (0, _ensureAllNonNullableValues.ensureAllNonNullableValues)(item, {
+            dateTime: fallbackDateTime,
+            identity: fallbackIdentity
+          });
+          logger.trace(`Successfully ensured all non-nullable meta fields have values (${item.modelId}/${item.id}). Will be saving into the database soon.`);
+        } catch (e) {
+          logger.debug(`Failed to ensure all non-nullable meta fields have values (${item.modelId}/${item.id}): ${e.message}`);
+        }
+      }
+      ddbItemsToBatchWrite.push(ddbEntryEntity.putBatch(item));
+
+      /**
+       * Prepare the loading of DynamoDB Elasticsearch part of the records.
+       */
+
+      const ddbEsLatestRecordKey = `${item.entryId}:L`;
+      if (ddbEsItemsToBatchRead[ddbEsLatestRecordKey]) {
+        continue;
+      }
+      ddbEsItemsToBatchRead[ddbEsLatestRecordKey] = ddbEsEntryEntity.getBatch({
+        PK: item.PK,
+        SK: "L"
+      });
+      const ddbEsPublishedRecordKey = `${item.entryId}:P`;
+      if (item.status === "published" || !!item.locked) {
+        ddbEsItemsToBatchRead[ddbEsPublishedRecordKey] = ddbEsEntryEntity.getBatch({
+          PK: item.PK,
+          SK: "P"
+        });
+      }
+    }
+    if (Object.keys(ddbEsItemsToBatchRead).length > 0) {
+      /**
+       * Get all the records from DynamoDB Elasticsearch.
+       */
+      const ddbEsRecords = await (0, _utils2.batchReadAll)({
+        table: ddbEsEntryEntity.table,
+        items: Object.values(ddbEsItemsToBatchRead)
+      });
+      for (const ddbEsRecord of ddbEsRecords) {
+        const decompressedData = await (0, _getDecompressedData.getDecompressedData)(ddbEsRecord.data);
+        if (!decompressedData) {
+          logger.trace(`[DDB-ES Table] Skipping record "${ddbEsRecord.PK}" as it is not a valid CMS entry...`);
+          continue;
+        }
+
+        // 1. Check if the data migration was ever performed. If not, let's perform it.
+        if (!(0, _isMigratedEntry.isMigratedEntry)(decompressedData)) {
+          // Get the oldest revision's `createdOn` value. We use that to set the entry-level `createdOn` value.
+          const createdOn = await (0, _getOldestRevisionCreatedOn.getOldestRevisionCreatedOn)({
+            entry: {
+              ...decompressedData,
+              PK: ddbEsRecord.PK
+            },
+            entryEntity: ddbEntryEntity
+          });
+          const firstLastPublishedOnByFields = await (0, _getFirstLastPublishedOn.getFirstLastPublishedOnBy)({
+            entry: {
+              ...decompressedData,
+              PK: ddbEsRecord.PK
+            },
+            entryEntity: ddbEntryEntity
+          });
+          (0, _assignNewMetaFields.assignNewMetaFields)(decompressedData, {
+            createdOn,
+            ...firstLastPublishedOnByFields
+          });
+        }
+
+        // 2. Ensure new non-nullable meta fields have a value and nothing is missing.
+        if (!(0, _hasAllNonNullableValues.hasAllNonNullableValues)(decompressedData)) {
+          logger.trace((0, _getNonNullableFieldsWithMissingValues.getNonNullableFieldsWithMissingValues)(decompressedData), [`[DDB-ES Table] Detected an entry with missing values for non-nullable meta fields`, `(${decompressedData.modelId}/${decompressedData.id}).`].join(" "));
+          try {
+            const fallbackIdentity = await (0, _getFallbackIdentity.getFallbackIdentity)({
+              entity: ddbEntryEntity,
+              tenant: decompressedData.tenant
+            });
+            (0, _ensureAllNonNullableValues.ensureAllNonNullableValues)(decompressedData, {
+              dateTime: fallbackDateTime,
+              identity: fallbackIdentity
+            });
+            logger.trace([`[DDB-ES Table] Successfully ensured all non-nullable meta fields`, `have values (${decompressedData.modelId}/${decompressedData.id}).`, "Will be saving the changes soon."].join(" "));
+          } catch (e) {
+            logger.error(["[DDB-ES Table] Failed to ensure all non-nullable meta fields have values", `(${decompressedData.modelId}/${decompressedData.id}): ${e.message}`].join(" "));
+          }
+        }
+        const compressedData = await (0, _getCompressedData.getCompressedData)(decompressedData);
+        ddbEsItemsToBatchWrite.push(ddbEsEntryEntity.putBatch({
+          ...ddbEsRecord,
+          data: compressedData
+        }));
+      }
+    }
+    if (ddbItemsToBatchWrite.length) {
+      // Store data in primary DynamoDB table.
+      const execute = () => {
+        return (0, _utils2.batchWriteAll)({
+          table: ddbEntryEntity.table,
+          items: ddbItemsToBatchWrite
+        });
+      };
+      logger.trace(`Storing ${ddbItemsToBatchWrite.length} record(s) in primary DynamoDB table...`);
+      await (0, _utils.executeWithRetry)(execute, {
+        onFailedAttempt: error => {
+          logger.warn(`Batch write attempt #${error.attemptNumber} failed: ${error.message}`);
+        }
+      });
+      if (ddbEsItemsToBatchWrite.length) {
+        logger.trace(`Storing ${ddbEsItemsToBatchWrite.length} record(s) in DDB-ES DynamoDB table...`);
+        const results = await waitUntilHealthy.wait({
+          async onUnhealthy(params) {
+            const shouldWaitReason = params.waitingReason.name;
+            logger.warn(`Cluster is unhealthy (${shouldWaitReason}). Waiting for the cluster to become healthy...`, params);
+            if (status.stats.esHealthChecks.unhealthyReasons[shouldWaitReason]) {
+              status.stats.esHealthChecks.unhealthyReasons[shouldWaitReason]++;
+            } else {
+              status.stats.esHealthChecks.unhealthyReasons[shouldWaitReason] = 1;
+            }
+          }
+        });
+        status.stats.esHealthChecks.checksCount++;
+        status.stats.esHealthChecks.timeSpentWaiting += results.runningTime;
+
+        // Store data in DDB-ES DynamoDB table.
+        const executeDdbEs = () => {
+          return (0, _utils2.batchWriteAll)({
+            table: ddbEsEntryEntity.table,
+            items: ddbEsItemsToBatchWrite
+          });
+        };
+        await (0, _utils.executeWithRetry)(executeDdbEs, {
+          onFailedAttempt: error => {
+            logger.warn(`[DDB-ES Table] Batch write attempt #${error.attemptNumber} failed: ${error.message}`);
+          }
+        });
+      }
+      status.stats.recordsUpdated += ddbItemsToBatchWrite.length;
+    }
+
+    // Update checkpoint after every batch.
+    let lastEvaluatedKey = true;
+    if (result.lastEvaluatedKey) {
+      lastEvaluatedKey = result.lastEvaluatedKey;
+    }
+    status.lastEvaluatedKey = lastEvaluatedKey;
+    if (lastEvaluatedKey === true) {
+      return false;
+    }
+
+    // Continue further scanning.
+    return true;
+  });
+
+  // Store status in tmp file.
+  logger.trace({
+    status
+  }, "Segment processing completed. Saving status to tmp file...");
+  const logFilePath = _path.default.join(_os.default.tmpdir(), `webiny-5-39-6-meta-fields-data-migration-log-${argv.runId}-${argv.segmentIndex}.log`);
+
+  // Save segment processing stats to a file.
+  _fs.default.writeFileSync(logFilePath, JSON.stringify(status.stats, null, 2));
+  logger.trace(`Segment processing stats saved in ${logFilePath}.`);
+})();
+
+//# sourceMappingURL=worker.js.map

package/migrations/5.39.6/001/ddb-es/worker.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_utils","require","_logger","_dataMigration","_clientDynamodb","_apiElasticsearch","_yargs","_interopRequireDefault","_helpers","_isMigratedEntry","_hasValidTypeFieldValue","_hasAllNonNullableValues","_getOldestRevisionCreatedOn","_getFirstLastPublishedOn","_assignNewMetaFields","_fixTypeFieldValue","_getFallbackIdentity","_ensureAllNonNullableValues","_getDecompressedData","_getCompressedData","_createEntryEntity","_utils2","_waitUntilHealthy","_pinoPretty","_path","_os","_fs","_getNonNullableFieldsWithMissingValues","argv","yargs","hideBin","process","options","runId","type","demandOption","ddbTable","ddbEsTable","esEndpoint","segmentIndex","totalSegments","esHealthMinClusterHealthStatus","esHealthMaxProcessorPercent","esHealthMaxRamPercent","esHealthMaxWaitingTime","esHealthWaitingTimeStep","parseSync","createInitialStatus","lastEvaluatedKey","stats","iterationsCount","recordsScanned","recordsUpdated","recordsSkipped","esHealthChecks","timeSpentWaiting","checksCount","unhealthyReasons","logger","createPinoLogger","level","getLogLevel","env","MIGRATIONS_LOG_LEVEL","msgPrefix","pinoPretty","ignore","documentClient","getDocumentClient","elasticsearchClient","createElasticsearchClient","endpoint","primaryTable","createTable","name","dynamoToEsTable","ddbEntryEntity","createDdbEntryEntity","ddbEsEntryEntity","createDdbEsEntryEntity","status","waitUntilHealthy","createWaitUntilHealthy","minClusterHealthStatus","maxProcessorPercent","maxRamPercent","maxWaitingTime","waitingTimeStep","ddbScanWithCallback","entity","segment","segments","filters","attr","eq","startKey","undefined","limit","result","items","length","trace","ddbItemsToBatchWrite","ddbEsItemsToBatchWrite","ddbEsItemsToBatchRead","fallbackDateTime","Date","toISOString","item","isFullyMigrated","isMigratedEntry","hasValidTypeFieldValue","hasAllNonNullableValues","createdOn","getOldestRevisionCreatedOn","entry","entryEntity","firstLastPublishedOnByFields","getFirstLastPublishedOnBy","assignNewMetaFields","fixTypeFieldValue","getNonNullableFieldsWithMissingValues","modelId","id","fallbackIdentity","getFallbackIdentity","tenant","ensureAllNonNullableValues","dateTime","identity","e","debug","message","push","putBatch","ddbEsLatestRecordKey","entryId","getBatch","PK","SK","ddbEsPublishedRecordKey","locked","Object","keys","ddbEsRecords","batchReadAll","table","values","ddbEsRecord","decompressedData","getDecompressedData","data","join","error","compressedData","getCompressedData","execute","batchWriteAll","executeWithRetry","onFailedAttempt","warn","attemptNumber","results","wait","onUnhealthy","params","shouldWaitReason","waitingReason","runningTime","executeDdbEs","logFilePath","path","os","tmpdir","fs","writeFileSync","JSON","stringify"],"sources":["worker.ts"],"sourcesContent":["import { executeWithRetry } from \"@webiny/utils\";\nimport { createPinoLogger, getLogLevel } from \"@webiny/logger\";\nimport { createTable } from \"@webiny/data-migration\";\nimport { getDocumentClient } from \"@webiny/aws-sdk/client-dynamodb\";\nimport { createElasticsearchClient } from \"@webiny/api-elasticsearch\";\nimport yargs from \"yargs/yargs\";\nimport { hideBin } from \"yargs/helpers\";\nimport { isMigratedEntry } from \"~/migrations/5.39.0/001/utils/isMigratedEntry\";\nimport { hasValidTypeFieldValue } from \"~/migrations/5.39.0/001/utils/hasValidTypeFieldValue\";\nimport { hasAllNonNullableValues } from \"~/migrations/5.39.0/001/utils/hasAllNonNullableValues\";\nimport { getOldestRevisionCreatedOn } from \"~/migrations/5.39.0/001/utils/getOldestRevisionCreatedOn\";\nimport { getFirstLastPublishedOnBy } from \"~/migrations/5.39.0/001/utils/getFirstLastPublishedOn\";\nimport { assignNewMetaFields } from \"~/migrations/5.39.0/001/utils/assignNewMetaFields\";\nimport { fixTypeFieldValue } from \"~/migrations/5.39.0/001/utils/fixTypeFieldValue\";\nimport { getFallbackIdentity } from \"~/migrations/5.39.0/001/utils/getFallbackIdentity\";\nimport { ensureAllNonNullableValues } from \"~/migrations/5.39.0/001/utils/ensureAllNonNullableValues\";\nimport { getDecompressedData } from \"~/migrations/5.39.0/001/utils/getDecompressedData\";\nimport { getCompressedData } from \"~/migrations/5.39.0/001/utils/getCompressedData\";\nimport { CmsEntry } from \"~/migrations/5.39.0/001/types\";\nimport {\n    createDdbEntryEntity,\n    createDdbEsEntryEntity\n} from \"~/migrations/5.39.0/001/entities/createEntryEntity\";\nimport {\n    batchReadAll,\n    BatchReadItem,\n    batchWriteAll,\n    BatchWriteItem,\n    ddbScanWithCallback\n} from \"~/utils\";\nimport { createWaitUntilHealthy } from \"@webiny/api-elasticsearch/utils/waitUntilHealthy\";\nimport pinoPretty from \"pino-pretty\";\nimport { EsHealthChecksParams } from \"~/migrations/5.39.6/001/ddb-es/utils\";\nimport path from \"path\";\nimport os from \"os\";\nimport fs from \"fs\";\nimport { getNonNullableFieldsWithMissingValues } from \"~/migrations/5.39.0/001/utils/getNonNullableFieldsWithMissingValues\";\n\nconst argv = yargs(hideBin(process.argv))\n    .options({\n        runId: { type: \"string\", demandOption: true },\n        ddbTable: { type: \"string\", demandOption: true },\n        ddbEsTable: { type: \"string\", demandOption: true },\n        esEndpoint: { type: \"string\", demandOption: true },\n        segmentIndex: { type: \"number\", demandOption: true },\n        totalSegments: { type: \"number\", demandOption: true },\n\n        // Elasticsearch health check options.\n        esHealthMinClusterHealthStatus: { type: \"string\", demandOption: true },\n        esHealthMaxProcessorPercent: { type: \"number\", demandOption: true },\n        esHealthMaxRamPercent: { type: \"number\", demandOption: true },\n        esHealthMaxWaitingTime: { type: \"number\", demandOption: true },\n        esHealthWaitingTimeStep: { type: \"number\", demandOption: true }\n    })\n    .parseSync();\n\ninterface LastEvaluatedKeyObject {\n    PK: string;\n    SK: string;\n    GSI1_PK: string;\n    GSI1_SK: string;\n}\n\ntype LastEvaluatedKey = LastEvaluatedKeyObject | true | null;\n\ninterface MigrationStatus {\n    lastEvaluatedKey: LastEvaluatedKey;\n    stats: {\n        iterationsCount: number;\n        recordsScanned: number;\n        recordsUpdated: number;\n        recordsSkipped: number;\n        esHealthChecks: {\n            timeSpentWaiting: number;\n            checksCount: number;\n            unhealthyReasons: {\n                [key: string]: number;\n            };\n        };\n    };\n}\n\ninterface DynamoDbElasticsearchRecord {\n    PK: string;\n    SK: string;\n    data: string;\n}\n\nconst createInitialStatus = (): MigrationStatus => {\n    return {\n        lastEvaluatedKey: null,\n        stats: {\n            iterationsCount: 0,\n            recordsScanned: 0,\n            recordsUpdated: 0,\n            recordsSkipped: 0,\n            esHealthChecks: {\n                timeSpentWaiting: 0,\n                checksCount: 0,\n                unhealthyReasons: {}\n            }\n        }\n    };\n};\n\n(async () => {\n    const logger = createPinoLogger(\n        {\n            level: getLogLevel(process.env.MIGRATIONS_LOG_LEVEL, \"trace\"),\n            msgPrefix: `[segment #${argv.segmentIndex}] `\n        },\n        pinoPretty({ ignore: \"pid,hostname\" })\n    );\n\n    const documentClient = getDocumentClient();\n    const elasticsearchClient = createElasticsearchClient({\n        endpoint: `https://${argv.esEndpoint}`\n    });\n\n    const primaryTable = createTable({\n        name: argv.ddbTable,\n        documentClient\n    });\n    const dynamoToEsTable = createTable({\n        name: argv.ddbEsTable,\n        documentClient\n    });\n\n    const ddbEntryEntity = createDdbEntryEntity(primaryTable);\n    const ddbEsEntryEntity = createDdbEsEntryEntity(dynamoToEsTable);\n\n    const status = createInitialStatus();\n\n    const waitUntilHealthy = createWaitUntilHealthy(elasticsearchClient, {\n        minClusterHealthStatus:\n            argv.esHealthMinClusterHealthStatus as EsHealthChecksParams[\"minClusterHealthStatus\"],\n        maxProcessorPercent: argv.esHealthMaxProcessorPercent,\n        maxRamPercent: argv.esHealthMaxRamPercent,\n        maxWaitingTime: argv.esHealthMaxWaitingTime,\n        waitingTimeStep: argv.esHealthWaitingTimeStep\n    });\n\n    await ddbScanWithCallback<CmsEntry>(\n        {\n            entity: ddbEntryEntity,\n            options: {\n                segment: argv.segmentIndex,\n                segments: argv.totalSegments,\n                filters: [\n                    {\n                        attr: \"_et\",\n                        eq: \"CmsEntries\"\n                    }\n                ],\n                startKey: status.lastEvaluatedKey || undefined,\n                limit: 100\n            }\n        },\n        async result => {\n            status.stats.iterationsCount++;\n            status.stats.recordsScanned += result.items.length;\n\n            if (status.stats.iterationsCount % 5 === 0) {\n                // We log every 5th iteration.\n                logger.trace(\n                    `[iteration #${status.stats.iterationsCount}] Reading ${result.items.length} record(s)...`\n                );\n            }\n\n            const ddbItemsToBatchWrite: BatchWriteItem[] = [];\n            const ddbEsItemsToBatchWrite: BatchWriteItem[] = [];\n            const ddbEsItemsToBatchRead: Record<string, BatchReadItem> = {};\n\n            const fallbackDateTime = new Date().toISOString();\n\n            // Update records in primary DynamoDB table. Also do preparations for\n            // subsequent updates on DDB-ES DynamoDB table, and in Elasticsearch.\n            for (const item of result.items) {\n                const isFullyMigrated =\n                    isMigratedEntry(item) &&\n                    hasValidTypeFieldValue(item) &&\n                    hasAllNonNullableValues(item);\n\n                if (isFullyMigrated) {\n                    status.stats.recordsSkipped++;\n                    continue;\n                }\n\n                // 1. Check if the data migration was ever performed. If not, let's perform it.\n                if (!isMigratedEntry(item)) {\n                    // Get the oldest revision's `createdOn` value. We use that to set the entry-level `createdOn` value.\n                    const createdOn = await getOldestRevisionCreatedOn({\n                        entry: item,\n                        entryEntity: ddbEntryEntity\n                    });\n\n                    const firstLastPublishedOnByFields = await getFirstLastPublishedOnBy({\n                        entry: item,\n                        entryEntity: ddbEntryEntity\n                    });\n\n                    assignNewMetaFields(item, {\n                        createdOn,\n                        ...firstLastPublishedOnByFields\n                    });\n                }\n\n                // 2. We've noticed some of the records had an invalid `TYPE` field value\n                //    in the database. This step addresses this issue.\n                if (!hasValidTypeFieldValue(item)) {\n                    // Fixes the value of the `TYPE` field, if it's not valid.\n                    fixTypeFieldValue(item);\n                }\n\n                // 3. Finally, once both of the steps were performed, ensure that all\n                //    new non-nullable meta fields have a value and nothing is missing.\n                if (!hasAllNonNullableValues(item)) {\n                    logger.trace(\n                        getNonNullableFieldsWithMissingValues(item),\n                        `Detected an entry with missing values for non-nullable meta fields (${item.modelId}/${item.id}).`\n                    );\n\n                    try {\n                        const fallbackIdentity = await getFallbackIdentity({\n                            entity: ddbEntryEntity,\n                            tenant: item.tenant\n                        });\n\n                        ensureAllNonNullableValues(item, {\n                            dateTime: fallbackDateTime,\n                            identity: fallbackIdentity\n                        });\n\n                        logger.trace(\n                            `Successfully ensured all non-nullable meta fields have values (${item.modelId}/${item.id}). Will be saving into the database soon.`\n                        );\n                    } catch (e) {\n                        logger.debug(\n                            `Failed to ensure all non-nullable meta fields have values (${item.modelId}/${item.id}): ${e.message}`\n                        );\n                    }\n                }\n\n                ddbItemsToBatchWrite.push(ddbEntryEntity.putBatch(item));\n\n                /**\n                 * Prepare the loading of DynamoDB Elasticsearch part of the records.\n                 */\n\n                const ddbEsLatestRecordKey = `${item.entryId}:L`;\n                if (ddbEsItemsToBatchRead[ddbEsLatestRecordKey]) {\n                    continue;\n                }\n\n                ddbEsItemsToBatchRead[ddbEsLatestRecordKey] = ddbEsEntryEntity.getBatch({\n                    PK: item.PK,\n                    SK: \"L\"\n                });\n\n                const ddbEsPublishedRecordKey = `${item.entryId}:P`;\n                if (item.status === \"published\" || !!item.locked) {\n                    ddbEsItemsToBatchRead[ddbEsPublishedRecordKey] = ddbEsEntryEntity.getBatch({\n                        PK: item.PK,\n                        SK: \"P\"\n                    });\n                }\n            }\n\n            if (Object.keys(ddbEsItemsToBatchRead).length > 0) {\n                /**\n                 * Get all the records from DynamoDB Elasticsearch.\n                 */\n                const ddbEsRecords = await batchReadAll<DynamoDbElasticsearchRecord>({\n                    table: ddbEsEntryEntity.table,\n                    items: Object.values(ddbEsItemsToBatchRead)\n                });\n\n                for (const ddbEsRecord of ddbEsRecords) {\n                    const decompressedData = await getDecompressedData<CmsEntry>(ddbEsRecord.data);\n                    if (!decompressedData) {\n                        logger.trace(\n                            `[DDB-ES Table] Skipping record \"${ddbEsRecord.PK}\" as it is not a valid CMS entry...`\n                        );\n                        continue;\n                    }\n\n                    // 1. Check if the data migration was ever performed. If not, let's perform it.\n                    if (!isMigratedEntry(decompressedData)) {\n                        // Get the oldest revision's `createdOn` value. We use that to set the entry-level `createdOn` value.\n                        const createdOn = await getOldestRevisionCreatedOn({\n                            entry: { ...decompressedData, PK: ddbEsRecord.PK },\n                            entryEntity: ddbEntryEntity\n                        });\n\n                        const firstLastPublishedOnByFields = await getFirstLastPublishedOnBy({\n                            entry: { ...decompressedData, PK: ddbEsRecord.PK },\n                            entryEntity: ddbEntryEntity\n                        });\n\n                        assignNewMetaFields(decompressedData, {\n                            createdOn,\n                            ...firstLastPublishedOnByFields\n                        });\n                    }\n\n                    // 2. Ensure new non-nullable meta fields have a value and nothing is missing.\n                    if (!hasAllNonNullableValues(decompressedData)) {\n                        logger.trace(\n                            getNonNullableFieldsWithMissingValues(decompressedData),\n                            [\n                                `[DDB-ES Table] Detected an entry with missing values for non-nullable meta fields`,\n                                `(${decompressedData.modelId}/${decompressedData.id}).`\n                            ].join(\" \")\n                        );\n\n                        try {\n                            const fallbackIdentity = await getFallbackIdentity({\n                                entity: ddbEntryEntity,\n                                tenant: decompressedData.tenant\n                            });\n\n                            ensureAllNonNullableValues(decompressedData, {\n                                dateTime: fallbackDateTime,\n                                identity: fallbackIdentity\n                            });\n\n                            logger.trace(\n                                [\n                                    `[DDB-ES Table] Successfully ensured all non-nullable meta fields`,\n                                    `have values (${decompressedData.modelId}/${decompressedData.id}).`,\n                                    \"Will be saving the changes soon.\"\n                                ].join(\" \")\n                            );\n                        } catch (e) {\n                            logger.error(\n                                [\n                                    \"[DDB-ES Table] Failed to ensure all non-nullable meta fields have values\",\n                                    `(${decompressedData.modelId}/${decompressedData.id}): ${e.message}`\n                                ].join(\" \")\n                            );\n                        }\n                    }\n\n                    const compressedData = await getCompressedData(decompressedData);\n\n                    ddbEsItemsToBatchWrite.push(\n                        ddbEsEntryEntity.putBatch({\n                            ...ddbEsRecord,\n                            data: compressedData\n                        })\n                    );\n                }\n            }\n\n            if (ddbItemsToBatchWrite.length) {\n                // Store data in primary DynamoDB table.\n                const execute = () => {\n                    return batchWriteAll({\n                        table: ddbEntryEntity.table,\n                        items: ddbItemsToBatchWrite\n                    });\n                };\n\n                logger.trace(\n                    `Storing ${ddbItemsToBatchWrite.length} record(s) in primary DynamoDB table...`\n                );\n                await executeWithRetry(execute, {\n                    onFailedAttempt: error => {\n                        logger.warn(\n                            `Batch write attempt #${error.attemptNumber} failed: ${error.message}`\n                        );\n                    }\n                });\n\n                if (ddbEsItemsToBatchWrite.length) {\n                    logger.trace(\n                        `Storing ${ddbEsItemsToBatchWrite.length} record(s) in DDB-ES DynamoDB table...`\n                    );\n                    const results = await waitUntilHealthy.wait({\n                        async onUnhealthy(params) {\n                            const shouldWaitReason = params.waitingReason.name;\n\n                            logger.warn(\n                                `Cluster is unhealthy (${shouldWaitReason}). Waiting for the cluster to become healthy...`,\n                                params\n                            );\n\n                            if (status.stats.esHealthChecks.unhealthyReasons[shouldWaitReason]) {\n                                status.stats.esHealthChecks.unhealthyReasons[shouldWaitReason]++;\n                            } else {\n                                status.stats.esHealthChecks.unhealthyReasons[shouldWaitReason] = 1;\n                            }\n                        }\n                    });\n\n                    status.stats.esHealthChecks.checksCount++;\n                    status.stats.esHealthChecks.timeSpentWaiting += results.runningTime;\n\n                    // Store data in DDB-ES DynamoDB table.\n                    const executeDdbEs = () => {\n                        return batchWriteAll({\n                            table: ddbEsEntryEntity.table,\n                            items: ddbEsItemsToBatchWrite\n                        });\n                    };\n\n                    await executeWithRetry(executeDdbEs, {\n                        onFailedAttempt: error => {\n                            logger.warn(\n                                `[DDB-ES Table] Batch write attempt #${error.attemptNumber} failed: ${error.message}`\n                            );\n                        }\n                    });\n                }\n\n                status.stats.recordsUpdated += ddbItemsToBatchWrite.length;\n            }\n\n            // Update checkpoint after every batch.\n            let lastEvaluatedKey: LastEvaluatedKey = true;\n            if (result.lastEvaluatedKey) {\n                lastEvaluatedKey = result.lastEvaluatedKey as unknown as LastEvaluatedKeyObject;\n            }\n\n            status.lastEvaluatedKey = lastEvaluatedKey;\n\n            if (lastEvaluatedKey === true) {\n                return false;\n            }\n\n            // Continue further scanning.\n            return true;\n        }\n    );\n\n    // Store status in tmp file.\n    logger.trace({ status }, \"Segment processing completed. Saving status to tmp file...\");\n    const logFilePath = path.join(\n        os.tmpdir(),\n        `webiny-5-39-6-meta-fields-data-migration-log-${argv.runId}-${argv.segmentIndex}.log`\n    );\n\n    // Save segment processing stats to a file.\n    fs.writeFileSync(logFilePath, JSON.stringify(status.stats, null, 2));\n\n    logger.trace(`Segment processing stats saved in ${logFilePath}.`);\n})();\n"],"mappings":";;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AACA,IAAAC,OAAA,GAAAD,OAAA;AACA,IAAAE,cAAA,GAAAF,OAAA;AACA,IAAAG,eAAA,GAAAH,OAAA;AACA,IAAAI,iBAAA,GAAAJ,OAAA;AACA,IAAAK,MAAA,GAAAC,sBAAA,CAAAN,OAAA;AACA,IAAAO,QAAA,GAAAP,OAAA;AACA,IAAAQ,gBAAA,GAAAR,OAAA;AACA,IAAAS,uBAAA,GAAAT,OAAA;AACA,IAAAU,wBAAA,GAAAV,OAAA;AACA,IAAAW,2BAAA,GAAAX,OAAA;AACA,IAAAY,wBAAA,GAAAZ,OAAA;AACA,IAAAa,oBAAA,GAAAb,OAAA;AACA,IAAAc,kBAAA,GAAAd,OAAA;AACA,IAAAe,oBAAA,GAAAf,OAAA;AACA,IAAAgB,2BAAA,GAAAhB,OAAA;AACA,IAAAiB,oBAAA,GAAAjB,OAAA;AACA,IAAAkB,kBAAA,GAAAlB,OAAA;AAEA,IAAAmB,kBAAA,GAAAnB,OAAA;AAIA,IAAAoB,OAAA,GAAApB,OAAA;AAOA,IAAAqB,iBAAA,GAAArB,OAAA;AACA,IAAAsB,WAAA,GAAAhB,sBAAA,CAAAN,OAAA;AAEA,IAAAuB,KAAA,GAAAjB,sBAAA,CAAAN,OAAA;AACA,IAAAwB,GAAA,GAAAlB,sBAAA,CAAAN,OAAA;AACA,IAAAyB,GAAA,GAAAnB,sBAAA,CAAAN,OAAA;AACA,IAAA0B,sCAAA,GAAA1B,OAAA;AAEA,MAAM2B,IAAI,GAAG,IAAAC,cAAK,EAAC,IAAAC,gBAAO,EAACC,OAAO,CAACH,IAAI,CAAC,CAAC,CACpCI,OAAO,CAAC;EACLC,KAAK,EAAE;IAAEC,IAAI,EAAE,QAAQ;IAAEC,YAAY,EAAE;EAAK,CAAC;EAC7CC,QAAQ,EAAE;IAAEF,IAAI,EAAE,QAAQ;IAAEC,YAAY,EAAE;EAAK,CAAC;EAChDE,UAAU,EAAE;IAAEH,IAAI,EAAE,QAAQ;IAAEC,YAAY,EAAE;EAAK,CAAC;EAClDG,UAAU,EAAE;IAAEJ,IAAI,EAAE,QAAQ;IAAEC,YAAY,EAAE;EAAK,CAAC;EAClDI,YAAY,EAAE;IAAEL,IAAI,EAAE,QAAQ;IAAEC,YAAY,EAAE;EAAK,CAAC;EACpDK,aAAa,EAAE;IAAEN,IAAI,EAAE,QAAQ;IAAEC,YAAY,EAAE;EAAK,CAAC;EAErD;EACAM,8BAA8B,EAAE;IAAEP,IAAI,EAAE,QAAQ;IAAEC,YAAY,EAAE;EAAK,CAAC;EACtEO,2BAA2B,EAAE;IAAER,IAAI,EAAE,QAAQ;IAAEC,YAAY,EAAE;EAAK,CAAC;EACnEQ,qBAAqB,EAAE;IAAET,IAAI,EAAE,QAAQ;IAAEC,YAAY,EAAE;EAAK,CAAC;EAC7DS,sBAAsB,EAAE;IAAEV,IAAI,EAAE,QAAQ;IAAEC,YAAY,EAAE;EAAK,CAAC;EAC9DU,uBAAuB,EAAE;IAAEX,IAAI,EAAE,QAAQ;IAAEC,YAAY,EAAE;EAAK;AAClE,CAAC,CAAC,CACDW,SAAS,CAAC,CAAC;AAkChB,MAAMC,mBAAmB,GAAGA,CAAA,KAAuB;EAC/C,OAAO;IACHC,gBAAgB,EAAE,IAAI;IACtBC,KAAK,EAAE;MACHC,eAAe,EAAE,CAAC;MAClBC,cAAc,EAAE,CAAC;MACjBC,cAAc,EAAE,CAAC;MACjBC,cAAc,EAAE,CAAC;MACjBC,cAAc,EAAE;QACZC,gBAAgB,EAAE,CAAC;QACnBC,WAAW,EAAE,CAAC;QACdC,gBAAgB,EAAE,CAAC;MACvB;IACJ;EACJ,CAAC;AACL,CAAC;AAED,CAAC,YAAY;EACT,MAAMC,MAAM,GAAG,IAAAC,wBAAgB,EAC3B;IACIC,KAAK,EAAE,IAAAC,mBAAW,EAAC9B,OAAO,CAAC+B,GAAG,CAACC,oBAAoB,EAAE,OAAO,CAAC;IAC7DC,SAAS,EAAG,aAAYpC,IAAI,CAACW,YAAa;EAC9C,CAAC,EACD,IAAA0B,mBAAU,EAAC;IAAEC,MAAM,EAAE;EAAe,CAAC,CACzC,CAAC;EAED,MAAMC,cAAc,GAAG,IAAAC,iCAAiB,EAAC,CAAC;EAC1C,MAAMC,mBAAmB,GAAG,IAAAC,2CAAyB,EAAC;IAClDC,QAAQ,EAAG,WAAU3C,IAAI,CAACU,UAAW;EACzC,CAAC,CAAC;EAEF,MAAMkC,YAAY,GAAG,IAAAC,0BAAW,EAAC;IAC7BC,IAAI,EAAE9C,IAAI,CAACQ,QAAQ;IACnB+B;EACJ,CAAC,CAAC;EACF,MAAMQ,eAAe,GAAG,IAAAF,0BAAW,EAAC;IAChCC,IAAI,EAAE9C,IAAI,CAACS,UAAU;IACrB8B;EACJ,CAAC,CAAC;EAEF,MAAMS,cAAc,GAAG,IAAAC,uCAAoB,EAACL,YAAY,CAAC;EACzD,MAAMM,gBAAgB,GAAG,IAAAC,yCAAsB,EAACJ,eAAe,CAAC;EAEhE,MAAMK,MAAM,GAAGjC,mBAAmB,CAAC,CAAC;EAEpC,MAAMkC,gBAAgB,GAAG,IAAAC,wCAAsB,EAACb,mBAAmB,EAAE;IACjEc,sBAAsB,EAClBvD,IAAI,CAACa,8BAAgF;IACzF2C,mBAAmB,EAAExD,IAAI,CAACc,2BAA2B;IACrD2C,aAAa,EAAEzD,IAAI,CAACe,qBAAqB;IACzC2C,cAAc,EAAE1D,IAAI,CAACgB,sBAAsB;IAC3C2C,eAAe,EAAE3D,IAAI,CAACiB;EAC1B,CAAC,CAAC;EAEF,MAAM,IAAA2C,2BAAmB,EACrB;IACIC,MAAM,EAAEb,cAAc;IACtB5C,OAAO,EAAE;MACL0D,OAAO,EAAE9D,IAAI,CAACW,YAAY;MAC1BoD,QAAQ,EAAE/D,IAAI,CAACY,aAAa;MAC5BoD,OAAO,EAAE,CACL;QACIC,IAAI,EAAE,KAAK;QACXC,EAAE,EAAE;MACR,CAAC,CACJ;MACDC,QAAQ,EAAEf,MAAM,CAAChC,gBAAgB,IAAIgD,SAAS;MAC9CC,KAAK,EAAE;IACX;EACJ,CAAC,EACD,MAAMC,MAAM,IAAI;IACZlB,MAAM,CAAC/B,KAAK,CAACC,eAAe,EAAE;IAC9B8B,MAAM,CAAC/B,KAAK,CAACE,cAAc,IAAI+C,MAAM,CAACC,KAAK,CAACC,MAAM;IAElD,IAAIpB,MAAM,CAAC/B,KAAK,CAACC,eAAe,GAAG,CAAC,KAAK,CAAC,EAAE;MACxC;MACAQ,MAAM,CAAC2C,KAAK,CACP,eAAcrB,MAAM,CAAC/B,KAAK,CAACC,eAAgB,aAAYgD,MAAM,CAACC,KAAK,CAACC,MAAO,eAChF,CAAC;IACL;IAEA,MAAME,oBAAsC,GAAG,EAAE;IACjD,MAAMC,sBAAwC,GAAG,EAAE;IACnD,MAAMC,qBAAoD,GAAG,CAAC,CAAC;IAE/D,MAAMC,gBAAgB,GAAG,IAAIC,IAAI,CAAC,CAAC,CAACC,WAAW,CAAC,CAAC;;IAEjD;IACA;IACA,KAAK,MAAMC,IAAI,IAAIV,MAAM,CAACC,KAAK,EAAE;MAC7B,MAAMU,eAAe,GACjB,IAAAC,gCAAe,EAACF,IAAI,CAAC,IACrB,IAAAG,8CAAsB,EAACH,IAAI,CAAC,IAC5B,IAAAI,gDAAuB,EAACJ,IAAI,CAAC;MAEjC,IAAIC,eAAe,EAAE;QACjB7B,MAAM,CAAC/B,KAAK,CAACI,cAAc,EAAE;QAC7B;MACJ;;MAEA;MACA,IAAI,CAAC,IAAAyD,gCAAe,EAACF,IAAI,CAAC,EAAE;QACxB;QACA,MAAMK,SAAS,GAAG,MAAM,IAAAC,sDAA0B,EAAC;UAC/CC,KAAK,EAAEP,IAAI;UACXQ,WAAW,EAAExC;QACjB,CAAC,CAAC;QAEF,MAAMyC,4BAA4B,GAAG,MAAM,IAAAC,kDAAyB,EAAC;UACjEH,KAAK,EAAEP,IAAI;UACXQ,WAAW,EAAExC;QACjB,CAAC,CAAC;QAEF,IAAA2C,wCAAmB,EAACX,IAAI,EAAE;UACtBK,SAAS;UACT,GAAGI;QACP,CAAC,CAAC;MACN;;MAEA;MACA;MACA,IAAI,CAAC,IAAAN,8CAAsB,EAACH,IAAI,CAAC,EAAE;QAC/B;QACA,IAAAY,oCAAiB,EAACZ,IAAI,CAAC;MAC3B;;MAEA;MACA;MACA,IAAI,CAAC,IAAAI,gDAAuB,EAACJ,IAAI,CAAC,EAAE;QAChClD,MAAM,CAAC2C,KAAK,CACR,IAAAoB,4EAAqC,EAACb,IAAI,CAAC,EAC1C,uEAAsEA,IAAI,CAACc,OAAQ,IAAGd,IAAI,CAACe,EAAG,IACnG,CAAC;QAED,IAAI;UACA,MAAMC,gBAAgB,GAAG,MAAM,IAAAC,wCAAmB,EAAC;YAC/CpC,MAAM,EAAEb,cAAc;YACtBkD,MAAM,EAAElB,IAAI,CAACkB;UACjB,CAAC,CAAC;UAEF,IAAAC,sDAA0B,EAACnB,IAAI,EAAE;YAC7BoB,QAAQ,EAAEvB,gBAAgB;YAC1BwB,QAAQ,EAAEL;UACd,CAAC,CAAC;UAEFlE,MAAM,CAAC2C,KAAK,CACP,kEAAiEO,IAAI,CAACc,OAAQ,IAAGd,IAAI,CAACe,EAAG,2CAC9F,CAAC;QACL,CAAC,CAAC,OAAOO,CAAC,EAAE;UACRxE,MAAM,CAACyE,KAAK,CACP,8DAA6DvB,IAAI,CAACc,OAAQ,IAAGd,IAAI,CAACe,EAAG,MAAKO,CAAC,CAACE,OAAQ,EACzG,CAAC;QACL;MACJ;MAEA9B,oBAAoB,CAAC+B,IAAI,CAACzD,cAAc,CAAC0D,QAAQ,CAAC1B,IAAI,CAAC,CAAC;;MAExD;AAChB;AACA;;MAEgB,MAAM2B,oBAAoB,GAAI,GAAE3B,IAAI,CAAC4B,OAAQ,IAAG;MAChD,IAAIhC,qBAAqB,CAAC+B,oBAAoB,CAAC,EAAE;QAC7C;MACJ;MAEA/B,qBAAqB,CAAC+B,oBAAoB,CAAC,GAAGzD,gBAAgB,CAAC2D,QAAQ,CAAC;QACpEC,EAAE,EAAE9B,IAAI,CAAC8B,EAAE;QACXC,EAAE,EAAE;MACR,CAAC,CAAC;MAEF,MAAMC,uBAAuB,GAAI,GAAEhC,IAAI,CAAC4B,OAAQ,IAAG;MACnD,IAAI5B,IAAI,CAAC5B,MAAM,KAAK,WAAW,IAAI,CAAC,CAAC4B,IAAI,CAACiC,MAAM,EAAE;QAC9CrC,qBAAqB,CAACoC,uBAAuB,CAAC,GAAG9D,gBAAgB,CAAC2D,QAAQ,CAAC;UACvEC,EAAE,EAAE9B,IAAI,CAAC8B,EAAE;UACXC,EAAE,EAAE;QACR,CAAC,CAAC;MACN;IACJ;IAEA,IAAIG,MAAM,CAACC,IAAI,CAACvC,qBAAqB,CAAC,CAACJ,MAAM,GAAG,CAAC,EAAE;MAC/C;AAChB;AACA;MACgB,MAAM4C,YAAY,GAAG,MAAM,IAAAC,oBAAY,EAA8B;QACjEC,KAAK,EAAEpE,gBAAgB,CAACoE,KAAK;QAC7B/C,KAAK,EAAE2C,MAAM,CAACK,MAAM,CAAC3C,qBAAqB;MAC9C,CAAC,CAAC;MAEF,KAAK,MAAM4C,WAAW,IAAIJ,YAAY,EAAE;QACpC,MAAMK,gBAAgB,GAAG,MAAM,IAAAC,wCAAmB,EAAWF,WAAW,CAACG,IAAI,CAAC;QAC9E,IAAI,CAACF,gBAAgB,EAAE;UACnB3F,MAAM,CAAC2C,KAAK,CACP,mCAAkC+C,WAAW,CAACV,EAAG,qCACtD,CAAC;UACD;QACJ;;QAEA;QACA,IAAI,CAAC,IAAA5B,gCAAe,EAACuC,gBAAgB,CAAC,EAAE;UACpC;UACA,MAAMpC,SAAS,GAAG,MAAM,IAAAC,sDAA0B,EAAC;YAC/CC,KAAK,EAAE;cAAE,GAAGkC,gBAAgB;cAAEX,EAAE,EAAEU,WAAW,CAACV;YAAG,CAAC;YAClDtB,WAAW,EAAExC;UACjB,CAAC,CAAC;UAEF,MAAMyC,4BAA4B,GAAG,MAAM,IAAAC,kDAAyB,EAAC;YACjEH,KAAK,EAAE;cAAE,GAAGkC,gBAAgB;cAAEX,EAAE,EAAEU,WAAW,CAACV;YAAG,CAAC;YAClDtB,WAAW,EAAExC;UACjB,CAAC,CAAC;UAEF,IAAA2C,wCAAmB,EAAC8B,gBAAgB,EAAE;YAClCpC,SAAS;YACT,GAAGI;UACP,CAAC,CAAC;QACN;;QAEA;QACA,IAAI,CAAC,IAAAL,gDAAuB,EAACqC,gBAAgB,CAAC,EAAE;UAC5C3F,MAAM,CAAC2C,KAAK,CACR,IAAAoB,4EAAqC,EAAC4B,gBAAgB,CAAC,EACvD,CACK,mFAAkF,EAClF,IAAGA,gBAAgB,CAAC3B,OAAQ,IAAG2B,gBAAgB,CAAC1B,EAAG,IAAG,CAC1D,CAAC6B,IAAI,CAAC,GAAG,CACd,CAAC;UAED,IAAI;YACA,MAAM5B,gBAAgB,GAAG,MAAM,IAAAC,wCAAmB,EAAC;cAC/CpC,MAAM,EAAEb,cAAc;cACtBkD,MAAM,EAAEuB,gBAAgB,CAACvB;YAC7B,CAAC,CAAC;YAEF,IAAAC,sDAA0B,EAACsB,gBAAgB,EAAE;cACzCrB,QAAQ,EAAEvB,gBAAgB;cAC1BwB,QAAQ,EAAEL;YACd,CAAC,CAAC;YAEFlE,MAAM,CAAC2C,KAAK,CACR,CACK,kEAAiE,EACjE,gBAAegD,gBAAgB,CAAC3B,OAAQ,IAAG2B,gBAAgB,CAAC1B,EAAG,IAAG,EACnE,kCAAkC,CACrC,CAAC6B,IAAI,CAAC,GAAG,CACd,CAAC;UACL,CAAC,CAAC,OAAOtB,CAAC,EAAE;YACRxE,MAAM,CAAC+F,KAAK,CACR,CACI,0EAA0E,EACzE,IAAGJ,gBAAgB,CAAC3B,OAAQ,IAAG2B,gBAAgB,CAAC1B,EAAG,MAAKO,CAAC,CAACE,OAAQ,EAAC,CACvE,CAACoB,IAAI,CAAC,GAAG,CACd,CAAC;UACL;QACJ;QAEA,MAAME,cAAc,GAAG,MAAM,IAAAC,oCAAiB,EAACN,gBAAgB,CAAC;QAEhE9C,sBAAsB,CAAC8B,IAAI,CACvBvD,gBAAgB,CAACwD,QAAQ,CAAC;UACtB,GAAGc,WAAW;UACdG,IAAI,EAAEG;QACV,CAAC,CACL,CAAC;MACL;IACJ;IAEA,IAAIpD,oBAAoB,CAACF,MAAM,EAAE;MAC7B;MACA,MAAMwD,OAAO,GAAGA,CAAA,KAAM;QAClB,OAAO,IAAAC,qBAAa,EAAC;UACjBX,KAAK,EAAEtE,cAAc,CAACsE,KAAK;UAC3B/C,KAAK,EAAEG;QACX,CAAC,CAAC;MACN,CAAC;MAED5C,MAAM,CAAC2C,KAAK,CACP,WAAUC,oBAAoB,CAACF,MAAO,yCAC3C,CAAC;MACD,MAAM,IAAA0D,uBAAgB,EAACF,OAAO,EAAE;QAC5BG,eAAe,EAAEN,KAAK,IAAI;UACtB/F,MAAM,CAACsG,IAAI,CACN,wBAAuBP,KAAK,CAACQ,aAAc,YAAWR,KAAK,CAACrB,OAAQ,EACzE,CAAC;QACL;MACJ,CAAC,CAAC;MAEF,IAAI7B,sBAAsB,CAACH,MAAM,EAAE;QAC/B1C,MAAM,CAAC2C,KAAK,CACP,WAAUE,sBAAsB,CAACH,MAAO,wCAC7C,CAAC;QACD,MAAM8D,OAAO,GAAG,MAAMjF,gBAAgB,CAACkF,IAAI,CAAC;UACxC,MAAMC,WAAWA,CAACC,MAAM,EAAE;YACtB,MAAMC,gBAAgB,GAAGD,MAAM,CAACE,aAAa,CAAC7F,IAAI;YAElDhB,MAAM,CAACsG,IAAI,CACN,yBAAwBM,gBAAiB,iDAAgD,EAC1FD,MACJ,CAAC;YAED,IAAIrF,MAAM,CAAC/B,KAAK,CAACK,cAAc,CAACG,gBAAgB,CAAC6G,gBAAgB,CAAC,EAAE;cAChEtF,MAAM,CAAC/B,KAAK,CAACK,cAAc,CAACG,gBAAgB,CAAC6G,gBAAgB,CAAC,EAAE;YACpE,CAAC,MAAM;cACHtF,MAAM,CAAC/B,KAAK,CAACK,cAAc,CAACG,gBAAgB,CAAC6G,gBAAgB,CAAC,GAAG,CAAC;YACtE;UACJ;QACJ,CAAC,CAAC;QAEFtF,MAAM,CAAC/B,KAAK,CAACK,cAAc,CAACE,WAAW,EAAE;QACzCwB,MAAM,CAAC/B,KAAK,CAACK,cAAc,CAACC,gBAAgB,IAAI2G,OAAO,CAACM,WAAW;;QAEnE;QACA,MAAMC,YAAY,GAAGA,CAAA,KAAM;UACvB,OAAO,IAAAZ,qBAAa,EAAC;YACjBX,KAAK,EAAEpE,gBAAgB,CAACoE,KAAK;YAC7B/C,KAAK,EAAEI;UACX,CAAC,CAAC;QACN,CAAC;QAED,MAAM,IAAAuD,uBAAgB,EAACW,YAAY,EAAE;UACjCV,eAAe,EAAEN,KAAK,IAAI;YACtB/F,MAAM,CAACsG,IAAI,CACN,uCAAsCP,KAAK,CAACQ,aAAc,YAAWR,KAAK,CAACrB,OAAQ,EACxF,CAAC;UACL;QACJ,CAAC,CAAC;MACN;MAEApD,MAAM,CAAC/B,KAAK,CAACG,cAAc,IAAIkD,oBAAoB,CAACF,MAAM;IAC9D;;IAEA;IACA,IAAIpD,gBAAkC,GAAG,IAAI;IAC7C,IAAIkD,MAAM,CAAClD,gBAAgB,EAAE;MACzBA,gBAAgB,GAAGkD,MAAM,CAAClD,gBAAqD;IACnF;IAEAgC,MAAM,CAAChC,gBAAgB,GAAGA,gBAAgB;IAE1C,IAAIA,gBAAgB,KAAK,IAAI,EAAE;MAC3B,OAAO,KAAK;IAChB;;IAEA;IACA,OAAO,IAAI;EACf,CACJ,CAAC;;EAED;EACAU,MAAM,CAAC2C,KAAK,CAAC;IAAErB;EAAO,CAAC,EAAE,4DAA4D,CAAC;EACtF,MAAM0F,WAAW,GAAGC,aAAI,CAACnB,IAAI,CACzBoB,WAAE,CAACC,MAAM,CAAC,CAAC,EACV,gDAA+CjJ,IAAI,CAACK,KAAM,IAAGL,IAAI,CAACW,YAAa,MACpF,CAAC;;EAED;EACAuI,WAAE,CAACC,aAAa,CAACL,WAAW,EAAEM,IAAI,CAACC,SAAS,CAACjG,MAAM,CAAC/B,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;EAEpES,MAAM,CAAC2C,KAAK,CAAE,qCAAoCqE,WAAY,GAAE,CAAC;AACrE,CAAC,EAAE,CAAC"}

package/package.json

@@ -0,0 +1,45 @@
+{
+	"name": "@webiny/migrations",
+	"version": "5.39.7-beta.2",
+	"scripts": {
+		"build": "yarn webiny run build",
+		"watch": "yarn webiny run watch"
+	},
+	"dependencies": {
+		"@elastic/elasticsearch": "7.12.0",
+		"@webiny/api-elasticsearch": "5.39.7-beta.2",
+		"@webiny/aws-sdk": "5.39.7-beta.2",
+		"@webiny/cli-plugin-deploy-pulumi": "5.39.7-beta.2",
+		"@webiny/data-migration": "5.39.7-beta.2",
+		"@webiny/db-dynamodb": "5.39.7-beta.2",
+		"@webiny/error": "5.39.7-beta.2",
+		"@webiny/ioc": "5.39.7-beta.2",
+		"@webiny/logger": "5.39.7-beta.2",
+		"@webiny/utils": "5.39.7-beta.2",
+		"execa": "5.1.1",
+		"fast-glob": "3.2.12",
+		"jsonpack": "1.1.5",
+		"lodash": "4.17.21",
+		"pino-pretty": "9.4.0",
+		"pluralize": "8.0.0",
+		"yargs": "17.6.2"
+	},
+	"publishConfig": {
+		"access": "public",
+		"directory": "dist"
+	},
+	"devDependencies": {
+		"@types/execa": "2.0.0",
+		"@webiny/api-headless-cms": "5.39.7-beta.2",
+		"@webiny/api-headless-cms-ddb-es": "5.39.7-beta.2",
+		"@webiny/cli": "5.39.7-beta.2",
+		"@webiny/handler-aws": "5.39.7-beta.2",
+		"@webiny/plugins": "5.39.7-beta.2",
+		"@webiny/project-utils": "5.39.7-beta.2",
+		"elastic-ts": "0.8.0",
+		"jest-dynalite": "3.6.1",
+		"ttypescript": "1.5.15",
+		"typescript": "4.7.4"
+	},
+	"gitHead": "0abcb9da80cb1ee22506c6f53e4f3bf2183ec9fd"
+}

package/utils/createEntity.d.ts

@@ -0,0 +1,89 @@
+import { AttributeDefinitions, Entity, Table } from "@webiny/db-dynamodb/toolbox";
+export declare const createLegacyEntity: (table: Table<string, string, string>, entityName: string, attributes: AttributeDefinitions) => Entity<string, import("dynamodb-toolbox/dist/cjs/classes/Entity").Overlay, import("dynamodb-toolbox/dist/cjs/classes/Entity").Overlay, Table<string, string, string>, true, true, boolean, string, string, string, boolean, {
+    PK: {
+        partitionKey: true;
+    };
+    SK: {
+        sortKey: true;
+    };
+    GSI1_PK: {
+        type: "string";
+    };
+    GSI1_SK: {
+        type: "string";
+    };
+    TYPE: {
+        type: "string";
+    };
+}, {
+    PK: {
+        partitionKey: true;
+    };
+    SK: {
+        sortKey: true;
+    };
+    GSI1_PK: {
+        type: "string";
+    };
+    GSI1_SK: {
+        type: "string";
+    };
+    TYPE: {
+        type: "string";
+    };
+}, import("dynamodb-toolbox/dist/cjs/classes/Entity").ParsedAttributes<import("ts-toolbelt/out/Any/Key").Key>, any, {
+    [x: string]: any;
+    [x: number]: any;
+    [x: symbol]: any;
+}, {
+    [x: string]: any;
+    [x: number]: any;
+    [x: symbol]: any;
+}>;
+export declare const createStandardEntity: (table: Table<string, string, string>, entityName: string, attributes?: AttributeDefinitions) => Entity<string, import("dynamodb-toolbox/dist/cjs/classes/Entity").Overlay, import("dynamodb-toolbox/dist/cjs/classes/Entity").Overlay, Table<string, string, string>, true, true, boolean, string, string, string, boolean, {
+    PK: {
+        partitionKey: true;
+    };
+    SK: {
+        sortKey: true;
+    };
+    GSI1_PK: {
+        type: "string";
+    };
+    GSI1_SK: {
+        type: "string";
+    };
+    TYPE: {
+        type: "string";
+    };
+    data: {
+        type: "map";
+    };
+}, {
+    PK: {
+        partitionKey: true;
+    };
+    SK: {
+        sortKey: true;
+    };
+    GSI1_PK: {
+        type: "string";
+    };
+    GSI1_SK: {
+        type: "string";
+    };
+    TYPE: {
+        type: "string";
+    };
+    data: {
+        type: "map";
+    };
+}, import("dynamodb-toolbox/dist/cjs/classes/Entity").ParsedAttributes<import("ts-toolbelt/out/Any/Key").Key>, any, {
+    [x: string]: any;
+    [x: number]: any;
+    [x: symbol]: any;
+}, {
+    [x: string]: any;
+    [x: number]: any;
+    [x: symbol]: any;
+}>;

package/utils/createEntity.js

@@ -0,0 +1,67 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.createStandardEntity = exports.createLegacyEntity = void 0;
+var _toolbox = require("@webiny/db-dynamodb/toolbox");
+const createLegacyEntity = (table, entityName, attributes) => {
+  return new _toolbox.Entity({
+    table,
+    name: entityName,
+    attributes: {
+      PK: {
+        partitionKey: true
+      },
+      SK: {
+        sortKey: true
+      },
+      GSI1_PK: {
+        type: "string"
+      },
+      GSI1_SK: {
+        type: "string"
+      },
+      TYPE: {
+        type: "string"
+      },
+      ...(attributes || {})
+    },
+    autoExecute: true,
+    autoParse: true
+  });
+};
+exports.createLegacyEntity = createLegacyEntity;
+const createStandardEntity = (table, entityName, attributes = {}) => {
+  return new _toolbox.Entity({
+    table,
+    name: entityName,
+    attributes: {
+      PK: {
+        partitionKey: true
+      },
+      SK: {
+        sortKey: true
+      },
+      GSI1_PK: {
+        type: "string"
+      },
+      GSI1_SK: {
+        type: "string"
+      },
+      TYPE: {
+        type: "string"
+      },
+      data: {
+        type: "map"
+      },
+      // When moving attributes to `data` envelope, we need to keep the old attributes in place for 1 version.
+      ...attributes
+    },
+    autoExecute: true,
+    autoParse: true
+  });
+};
+exports.createStandardEntity = createStandardEntity;
+
+//# sourceMappingURL=createEntity.js.map

package/utils/createEntity.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_toolbox","require","createLegacyEntity","table","entityName","attributes","Entity","name","PK","partitionKey","SK","sortKey","GSI1_PK","type","GSI1_SK","TYPE","autoExecute","autoParse","exports","createStandardEntity","data"],"sources":["createEntity.ts"],"sourcesContent":["import { AttributeDefinitions, Entity, Table } from \"@webiny/db-dynamodb/toolbox\";\n\nexport const createLegacyEntity = (\n    table: Table<string, string, string>,\n    entityName: string,\n    attributes: AttributeDefinitions\n) => {\n    return new Entity({\n        table,\n        name: entityName,\n        attributes: {\n            PK: {\n                partitionKey: true\n            },\n            SK: {\n                sortKey: true\n            },\n            GSI1_PK: {\n                type: \"string\"\n            },\n            GSI1_SK: {\n                type: \"string\"\n            },\n            TYPE: {\n                type: \"string\"\n            },\n            ...(attributes || {})\n        },\n        autoExecute: true,\n        autoParse: true\n    });\n};\n\nexport const createStandardEntity = (\n    table: Table<string, string, string>,\n    entityName: string,\n    attributes: AttributeDefinitions = {}\n) => {\n    return new Entity({\n        table,\n        name: entityName,\n        attributes: {\n            PK: {\n                partitionKey: true\n            },\n            SK: {\n                sortKey: true\n            },\n            GSI1_PK: {\n                type: \"string\"\n            },\n            GSI1_SK: {\n                type: \"string\"\n            },\n            TYPE: {\n                type: \"string\"\n            },\n            data: {\n                type: \"map\"\n            },\n            // When moving attributes to `data` envelope, we need to keep the old attributes in place for 1 version.\n            ...attributes\n        },\n        autoExecute: true,\n        autoParse: true\n    });\n};\n"],"mappings":";;;;;;AAAA,IAAAA,QAAA,GAAAC,OAAA;AAEO,MAAMC,kBAAkB,GAAGA,CAC9BC,KAAoC,EACpCC,UAAkB,EAClBC,UAAgC,KAC/B;EACD,OAAO,IAAIC,eAAM,CAAC;IACdH,KAAK;IACLI,IAAI,EAAEH,UAAU;IAChBC,UAAU,EAAE;MACRG,EAAE,EAAE;QACAC,YAAY,EAAE;MAClB,CAAC;MACDC,EAAE,EAAE;QACAC,OAAO,EAAE;MACb,CAAC;MACDC,OAAO,EAAE;QACLC,IAAI,EAAE;MACV,CAAC;MACDC,OAAO,EAAE;QACLD,IAAI,EAAE;MACV,CAAC;MACDE,IAAI,EAAE;QACFF,IAAI,EAAE;MACV,CAAC;MACD,IAAIR,UAAU,IAAI,CAAC,CAAC;IACxB,CAAC;IACDW,WAAW,EAAE,IAAI;IACjBC,SAAS,EAAE;EACf,CAAC,CAAC;AACN,CAAC;AAACC,OAAA,CAAAhB,kBAAA,GAAAA,kBAAA;AAEK,MAAMiB,oBAAoB,GAAGA,CAChChB,KAAoC,EACpCC,UAAkB,EAClBC,UAAgC,GAAG,CAAC,CAAC,KACpC;EACD,OAAO,IAAIC,eAAM,CAAC;IACdH,KAAK;IACLI,IAAI,EAAEH,UAAU;IAChBC,UAAU,EAAE;MACRG,EAAE,EAAE;QACAC,YAAY,EAAE;MAClB,CAAC;MACDC,EAAE,EAAE;QACAC,OAAO,EAAE;MACb,CAAC;MACDC,OAAO,EAAE;QACLC,IAAI,EAAE;MACV,CAAC;MACDC,OAAO,EAAE;QACLD,IAAI,EAAE;MACV,CAAC;MACDE,IAAI,EAAE;QACFF,IAAI,EAAE;MACV,CAAC;MACDO,IAAI,EAAE;QACFP,IAAI,EAAE;MACV,CAAC;MACD;MACA,GAAGR;IACP,CAAC;IACDW,WAAW,EAAE,IAAI;IACjBC,SAAS,EAAE;EACf,CAAC,CAAC;AACN,CAAC;AAACC,OAAA,CAAAC,oBAAA,GAAAA,oBAAA"}