@webiny/api-headless-cms-ddb-es 0.0.0-mt-1

package/upgrades/v5.0.0/createOldVersionIndiceName.js

@@ -0,0 +1,12 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.createOldVersionIndiceName = void 0;
+
+const createOldVersionIndiceName = context => {
+  return `${context.tenancy.getCurrentTenant().id}-headless-cms`;
+};
+
+exports.createOldVersionIndiceName = createOldVersionIndiceName;

package/upgrades/v5.0.0/entryValueFixer.d.ts

@@ -0,0 +1,4 @@
+import { ModelFieldFinder } from "./fieldFinder";
+import { CmsIndexEntry } from "../../types";
+import { CmsEntry, CmsModel } from "@webiny/api-headless-cms/types";
+export declare const entryValueFixer: (_: CmsModel, finder: ModelFieldFinder, target: CmsIndexEntry) => CmsEntry;

package/upgrades/v5.0.0/entryValueFixer.js

@@ -0,0 +1,124 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.entryValueFixer = void 0;
+
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+
+var _cleanDatabaseRecord = require("./cleanDatabaseRecord");
+
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
+
+const convertTimeToNumber = time => {
+  if (!time) {
+    return null;
+  }
+
+  const [hours, minutes, seconds = 0] = time.split(":").map(Number);
+  return hours * 60 * 60 + minutes * 60 + seconds;
+};
+
+const fixRawValues = (target, finder) => {
+  const entry = _objectSpread({}, target);
+
+  for (const fieldId in entry.rawValues) {
+    // if field is ref field then continue, otherwise break
+    const field = finder.findById(target.modelId, fieldId);
+
+    if (field.type !== "ref") {
+      continue;
+    }
+
+    const existingValue = entry.values[fieldId];
+    const rawValue = entry.rawValues[fieldId]; // always remove from rawValue
+
+    delete entry.rawValues[fieldId]; // if there is something in values - do not switch it
+
+    if (typeof existingValue !== "undefined" || rawValue === undefined || rawValue === null) {
+      continue;
+    }
+
+    entry.values[fieldId] = rawValue;
+  }
+
+  return entry;
+};
+
+const fixTime = (field, time) => {
+  if (isNaN(time) === false) {
+    return parseInt(time);
+  } // lets not reinvent the wheel
+
+
+  try {
+    return convertTimeToNumber(time);
+  } catch (err) {
+    console.log(`fixTime on field "${field.fieldId}" failed`, err);
+    return null;
+  }
+};
+
+const fixDateTime = (field, dateTime) => {
+  switch (field.settings.type) {
+    case "time":
+      return fixTime(field, dateTime);
+
+    case "date":
+      return new Date(dateTime).toISOString();
+
+    case "dateTimeWithoutTimezone":
+      return new Date(dateTime).toISOString();
+
+    default:
+      return dateTime;
+  }
+};
+
+const fixNumber = (_, value) => {
+  if (value === undefined || value === null) {
+    return null;
+  }
+
+  return parseFloat(value);
+};
+
+const fieldFixMethods = {
+  datetime: fixDateTime,
+  number: fixNumber
+};
+
+const fixFieldValues = (target, finder) => {
+  const entry = _objectSpread({}, target);
+
+  for (const fieldId in target.values) {
+    const field = finder.findById(target.modelId, fieldId);
+
+    if (!field) {
+      continue;
+    }
+
+    const fixMethod = fieldFixMethods[field.type];
+
+    if (!fixMethod) {
+      continue;
+    } else if (target.values[fieldId] === null || target.values[fieldId] === undefined) {
+      continue;
+    }
+
+    target.values[fieldId] = fixMethod(field, target.values[fieldId]);
+  }
+
+  return entry;
+};
+
+const entryValueFixer = (_, finder, target) => {
+  return (0, _cleanDatabaseRecord.cleanDatabaseRecord)(fixFieldValues(fixRawValues(target, finder), finder));
+};
+
+exports.entryValueFixer = entryValueFixer;

package/upgrades/v5.0.0/fieldFinder.d.ts

@@ -0,0 +1,6 @@
+import { CmsModel, CmsModelField } from "@webiny/api-headless-cms/types";
+export interface ModelFieldFinder {
+    findById: (model: CmsModel | string, fieldId: string) => CmsModelField;
+    findByType: (model: CmsModel | string, type: string) => CmsModelField[];
+}
+export declare const createFieldFinder: (models: CmsModel[]) => ModelFieldFinder;

package/upgrades/v5.0.0/fieldFinder.js

@@ -0,0 +1,42 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.createFieldFinder = void 0;
+
+const createFieldFinder = models => {
+  const modelsList = {};
+
+  for (const model of models) {
+    const values = {
+      id: {},
+      type: {}
+    };
+
+    for (const field of model.fields) {
+      values.id[field.fieldId] = field;
+
+      if (!values.type[field.type]) {
+        values.type[field.type] = [];
+      }
+
+      values.type[field.type].push(field);
+    }
+
+    modelsList[model.modelId] = values;
+  }
+
+  return {
+    findById: (model, fieldId) => {
+      const modelId = typeof model === "string" ? model : model.modelId;
+      return modelsList[modelId].id[fieldId];
+    },
+    findByType: (model, type) => {
+      const modelId = typeof model === "string" ? model : model.modelId;
+      return modelsList[modelId].type[type];
+    }
+  };
+};
+
+exports.createFieldFinder = createFieldFinder;

package/upgrades/v5.0.0/helpers.d.ts

@@ -0,0 +1,4 @@
+export * from "./cleanDatabaseRecord";
+export * from "./entryValueFixer";
+export * from "./fieldFinder";
+export * from "./createOldVersionIndiceName";

package/upgrades/v5.0.0/helpers.js

@@ -0,0 +1,57 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+
+var _cleanDatabaseRecord = require("./cleanDatabaseRecord");
+
+Object.keys(_cleanDatabaseRecord).forEach(function (key) {
+  if (key === "default" || key === "__esModule") return;
+  if (key in exports && exports[key] === _cleanDatabaseRecord[key]) return;
+  Object.defineProperty(exports, key, {
+    enumerable: true,
+    get: function () {
+      return _cleanDatabaseRecord[key];
+    }
+  });
+});
+
+var _entryValueFixer = require("./entryValueFixer");
+
+Object.keys(_entryValueFixer).forEach(function (key) {
+  if (key === "default" || key === "__esModule") return;
+  if (key in exports && exports[key] === _entryValueFixer[key]) return;
+  Object.defineProperty(exports, key, {
+    enumerable: true,
+    get: function () {
+      return _entryValueFixer[key];
+    }
+  });
+});
+
+var _fieldFinder = require("./fieldFinder");
+
+Object.keys(_fieldFinder).forEach(function (key) {
+  if (key === "default" || key === "__esModule") return;
+  if (key in exports && exports[key] === _fieldFinder[key]) return;
+  Object.defineProperty(exports, key, {
+    enumerable: true,
+    get: function () {
+      return _fieldFinder[key];
+    }
+  });
+});
+
+var _createOldVersionIndiceName = require("./createOldVersionIndiceName");
+
+Object.keys(_createOldVersionIndiceName).forEach(function (key) {
+  if (key === "default" || key === "__esModule") return;
+  if (key in exports && exports[key] === _createOldVersionIndiceName[key]) return;
+  Object.defineProperty(exports, key, {
+    enumerable: true,
+    get: function () {
+      return _createOldVersionIndiceName[key];
+    }
+  });
+});

package/upgrades/v5.0.0/index.d.ts

@@ -0,0 +1,4 @@
+import { UpgradePlugin } from "@webiny/api-upgrade/types";
+import { CmsContext } from "@webiny/api-headless-cms/types";
+declare const plugin: () => UpgradePlugin<CmsContext>;
+export default plugin;

package/upgrades/v5.0.0/index.js

@@ -0,0 +1,232 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+
+var _error = _interopRequireDefault(require("@webiny/error"));
+
+var _helpers = require("./helpers");
+
+var _utils = require("../utils");
+
+var _configurations = _interopRequireDefault(require("../../configurations"));
+
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
+
+const plugin = () => ({
+  name: "api-upgrade-cms-5.0.0",
+  type: "api-upgrade",
+  app: "headless-cms",
+  version: "5.0.0",
+
+  async apply(context) {
+    const {
+      db,
+      fileManager
+    } = context;
+    const elasticsearch = context.elasticsearch;
+
+    if (!elasticsearch) {
+      throw new _error.default("Missing Elasticsearch client on the context.");
+    } // Check if we still have the old elasticsearch index
+
+
+    const esIndex = (0, _helpers.createOldVersionIndiceName)(context);
+    const {
+      body: exists
+    } = await elasticsearch.indices.exists({
+      index: esIndex
+    });
+
+    if (!exists) {
+      return;
+    }
+
+    try {
+      await elasticsearch.indices.putTemplate({
+        name: "headless-cms-entries-index",
+        body: {
+          index_patterns: ["*headless-cms*"],
+          settings: {
+            analysis: {
+              analyzer: {
+                lowercase_analyzer: {
+                  type: "custom",
+                  filter: ["lowercase", "trim"],
+                  tokenizer: "keyword"
+                }
+              }
+            }
+          },
+          mappings: {
+            properties: {
+              property: {
+                type: "text",
+                fields: {
+                  keyword: {
+                    type: "keyword",
+                    ignore_above: 256
+                  }
+                },
+                analyzer: "lowercase_analyzer"
+              },
+              rawValues: {
+                type: "object",
+                enabled: false
+              }
+            }
+          }
+        }
+      });
+    } catch (err) {
+      console.log(err);
+      throw new _error.default("Put index template failed!");
+    } // Load existing ES items
+
+
+    let hasMoreItems = true;
+    let after = undefined;
+    const limit = 1000;
+    let esItems = []; // go through old index and load data in bulks of 1000
+
+    while (hasMoreItems) {
+      const response = await elasticsearch.search({
+        index: esIndex,
+        body: {
+          sort: {
+            createdOn: {
+              order: "asc",
+              // eslint-disable-next-line
+              unmapped_type: "date"
+            }
+          },
+          size: limit + 1,
+          after
+        }
+      });
+      const {
+        hits
+      } = response.body.hits;
+      hasMoreItems = hits.length > limit;
+      after = hasMoreItems ? hits[limit - 1].sort : undefined;
+      esItems = [...esItems, ...hits.filter(item => !item._id.includes("T#root#"))];
+    }
+
+    console.log(`Fetched ${esItems.length} items from Elasticsearch`);
+
+    if (esItems.length === 0) {
+      return;
+    } // Store a backup of old items
+
+
+    const esJSON = JSON.stringify(esItems);
+    const {
+      file
+    } = await fileManager.storage.storagePlugin.upload({
+      name: "upgrade-headless-cms-es-5.0.0.json",
+      type: "application/json",
+      size: esJSON.length,
+      buffer: Buffer.from(esJSON)
+    });
+    console.log(`Stored backup of Elasticsearch items to ${file.key}`); // Load models for each locale and distribute ES items to new per-model indexes
+
+    const locales = context.i18n.getLocales();
+    const esOperations = [];
+
+    for (const locale of locales) {
+      const [models] = await db.read(_objectSpread(_objectSpread({}, _configurations.default.db()), {}, {
+        query: {
+          PK: `T#root#L#${locale.code}#CMS#CM`,
+          SK: {
+            $gt: " "
+          }
+        }
+      })); // Sleep for 2 seconds
+
+      await new Promise(resolve => setTimeout(resolve, 2000));
+      const modelsById = models.reduce((acc, model) => {
+        acc[model.modelId] = model;
+        return acc;
+      }, {});
+      /**
+       *  Build a list of items for ES DDB table
+       */
+
+      const ddbItems = esItems.filter(hit => {
+        return hit._source.locale === locale.code;
+      }).map(hit => {
+        const entry = hit._source;
+        const model = modelsById[entry.modelId];
+
+        if (!model) {
+          return null;
+        }
+
+        const modelFieldFinder = (0, _helpers.createFieldFinder)(models);
+        const indexName = `root-headless-cms-${locale.code}-${model.modelId}`.toLowerCase();
+        const PK = `T#root#L#${locale.code}#CMS#CME#${entry.id.split("#")[0]}`;
+        const SK = entry.__type === "cms.entry.l" ? "L" : "P";
+
+        const esData = _objectSpread(_objectSpread({}, (0, _helpers.entryValueFixer)(model, modelFieldFinder, (0, _helpers.cleanDatabaseRecord)(entry))), {}, {
+          webinyVersion: "5.0.0"
+        });
+
+        esOperations.push({
+          index: {
+            _index: indexName,
+            _id: `${PK}:${SK}`
+          }
+        }, esData);
+        return {
+          PK,
+          SK,
+          index: indexName,
+          data: esData,
+          savedOn: new Date().toISOString(),
+          version: "5.0.0",
+          ignore: true
+        };
+      }).filter(Boolean);
+      console.log(`[${locale.code}] Prepared ${ddbItems.length} ES DDB items.`);
+      await (0, _utils.paginateBatch)(ddbItems, 25, async items => {
+        const batch = db.batch();
+        await batch.create(...items.map(item => {
+          return _objectSpread(_objectSpread({}, _configurations.default.esDb()), {}, {
+            data: item
+          });
+        })).execute();
+      });
+      console.log(`[${locale.code}] Inserted ES DDB items.`); // update all models to latest version
+
+      await (0, _utils.paginateBatch)(models, 25, async items => {
+        await db.batch().create(...items.map(model => {
+          return _objectSpread(_objectSpread({}, _configurations.default.db()), {}, {
+            data: _objectSpread(_objectSpread({}, model), {}, {
+              locale: locale.code,
+              webinyVersion: "5.0.0"
+            })
+          });
+        })).execute();
+      });
+      console.log(`[${locale.code}] Updated DDB model records with version number and locale code.`);
+    } // ES BULK INSERT
+
+
+    const bulkInsert = await elasticsearch.bulk({
+      body: esOperations
+    });
+    console.log("ES bulk index", bulkInsert);
+  }
+
+});
+
+var _default = plugin;
+exports.default = _default;

package/upgrades/v5.8.0/index.d.ts

@@ -0,0 +1,4 @@
+import { UpgradePlugin } from "@webiny/api-upgrade/types";
+import { CmsContext } from "@webiny/api-headless-cms/types";
+declare const _default: () => UpgradePlugin<CmsContext>;
+export default _default;