@paralect/hive 0.0.1

package/starter/package.json

@@ -0,0 +1,84 @@
+{
+  "name": "koa-api-starter",
+  "version": "2.3.0",
+  "description": "Koa api",
+  "private": false,
+  "main": "src/app.js",
+  "author": "Paralect",
+  "license": "MIT",
+  "keywords": [
+    "koa",
+    "rest api",
+    "paralect"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/paralect/koa-api-starter.git"
+  },
+  "scripts": {
+    "build-assets": "mjml ./src/assets/emails/*.mjml -o ./src/assets/emails/dist/",
+    "dev": "nodemon --watch src src/app.js",
+    "init-project": "./bin/init-project.sh",
+    "start": "node src/app.js",
+    "migrate": "node ./src/migrator.js",
+    "schedule-dev": "nodemon --watch ./src ./src/scheduler ./src/scheduler.js",
+    "schedule": "node ./src/scheduler.js",
+    "precommit": "lint-staged",
+    "prepare": "husky install"
+  },
+  "dependencies": {
+    "@koa/cors": "3.1.0",
+    "@koa/multer": "3.0.0",
+    "@koa/router": "10.1.1",
+    "@paralect/node-mongo": "2.1.1",
+    "@sendgrid/mail": "7.6.1",
+    "@socket.io/redis-adapter": "7.1.0",
+    "@socket.io/redis-emitter": "4.1.1",
+    "app-module-path": "2.2.0",
+    "aws-sdk": "2.1080.0",
+    "bcryptjs": "2.4.3",
+    "dotenv": "16.0.0",
+    "eslint-config-prettier": "8.5.0",
+    "handlebars": "4.7.7",
+    "joi": "17.6.0",
+    "koa": "2.13.4",
+    "koa-bodyparser": "4.3.0",
+    "koa-helmet": "6.1.0",
+    "koa-logger": "3.2.1",
+    "koa-mount": "4.0.0",
+    "koa-qs": "3.0.0",
+    "lodash": "4.17.21",
+    "mjml": "4.12.0",
+    "mkdirp": "1.0.4",
+    "moment": "2.29.1",
+    "moment-duration-format": "2.3.2",
+    "multer": "1.4.4",
+    "node-schedule": "2.1.0",
+    "nodemon": "2.0.15",
+    "prettier": "2.6.2",
+    "psl": "1.8.0",
+    "redis": "3.1.2",
+    "require-dir": "1.2.0",
+    "socket.io": "4.4.1",
+    "socket.io-emitter": "3.2.0",
+    "tail": "2.2.4",
+    "winston": "3.6.0"
+  },
+  "devDependencies": {
+    "eslint": "8.9.0",
+    "eslint-config-airbnb-base": "15.0.0",
+    "eslint-plugin-import": "2.25.4",
+    "husky": "7.0.4",
+    "lint-staged": "12.3.4"
+  },
+  "lint-staged": {
+    "*.js": [
+      "prettier --write",
+      "git add"
+    ],
+    ".js": [
+      "npm run lint:fix",
+      "npm run lint"
+    ]
+  }
+}

package/starter/src/app.js

@@ -0,0 +1,61 @@
+require("app-module-path").addPath(__dirname);
+
+if (process.env.HIVE_SRC) {
+  require("app-module-path").addPath(process.env.HIVE_SRC);
+}
+
+const fs = require("fs");
+
+const Koa = require("koa");
+const http = require("http");
+const config = require("config");
+const logger = require("logger");
+const cors = require("@koa/cors");
+const helmet = require("koa-helmet");
+const qs = require("koa-qs");
+const bodyParser = require("koa-bodyparser");
+const requestLogger = require("koa-logger");
+const db = require("db");
+
+const socketIo = require("socketIo");
+const mount = require("koa-mount");
+
+const main = async () => {
+  await db.init();
+
+  const routes = require("routes");
+
+  process.on("unhandledRejection", (reason, p) => {
+    console.trace(reason.stack);
+
+    logger.error(reason.stack);
+  });
+
+  const app = new Koa();
+
+  app.use(cors({ credentials: true }));
+  app.use(helmet());
+  qs(app);
+  app.use(bodyParser({ enableTypes: ["json", "form", "text"] }));
+
+  app.use(mount("/health", require("resources/health/endpoints/get").handler));
+
+  app.use(requestLogger());
+
+  routes(app);
+
+  const server = http.createServer(app.callback());
+  server.listen(config.port, () => {
+    console.log(
+      `Api server listening on ${config.port}, in ${process.env.NODE_ENV} mode and ${process.env.APP_ENV} environment`
+    );
+  });
+
+  await socketIo(server);
+
+  require("scheduler");
+};
+
+main();
+
+module.exports = main;

package/starter/src/assets/emails/components/header.mjml

@@ -0,0 +1,13 @@
+<mj-section background-color="#ffffff" padding-bottom="0px" padding-top="0">
+  <mj-column vertical-align="top" width="100%">
+    <mj-text
+      align="center"
+      font-size="30px"
+      font-weight="bold"
+      font-family="Trebuchet MS, Arial, sans-serif"
+      color="#55299a"
+    >
+      Ship
+    </mj-text>
+  </mj-column>
+</mj-section>

package/starter/src/assets/emails/signup-welcome.mjml

@@ -0,0 +1,34 @@
+<mjml>
+  <mj-head>
+    <mj-include path="./styles/index.mjml" />
+  </mj-head>
+
+  <mj-body>
+    <mj-include path="./components/header.mjml" />
+
+    <mj-section mj-class="main-section">
+      <mj-column vertical-align="middle" width="100%">
+        <mj-text mj-class="big-text"> Hi, there. </mj-text>
+
+        <mj-text mj-class="text">
+          Thanks for checking out Ship, we hope our products can make your
+          routine life a little more enjoyable.
+        </mj-text>
+
+        <mj-button mj-class="btn" href="{{verifyEmailUrl}}">
+          Verify Your Email
+        </mj-button>
+      </mj-column>
+    </mj-section>
+
+    <mj-section mj-class="bottom-section">
+      <mj-column vertical-align="middle" width="70%">
+        <mj-text mj-class="small-text"> INTERESTING FACT </mj-text>
+
+        <mj-text mj-class="tiny-text">
+          It took more than three years to build the Titanic.
+        </mj-text>
+      </mj-column>
+    </mj-section>
+  </mj-body>
+</mjml>

package/starter/src/assets/emails/styles/index.mjml

@@ -0,0 +1,77 @@
+<mj-attributes>
+  <mj-class
+    name="big-text"
+    align="center"
+    font-weight="bold"
+    font-family="Trebuchet MS, Arial, sans-serif"
+    color="#3b1d6c"
+    font-size="30px"
+    padding-left="25px"
+    padding-right="25px"
+  />
+
+  <mj-class
+    name="text"
+    align="center"
+    color="#000"
+    font-size="18px"
+    font-family="Century Gothic, Arial, sans-serif"
+    padding-left="25px"
+    padding-right="25px"
+    line-height="26px"
+  />
+
+  <mj-class
+    name="small-text"
+    align="center"
+    font-weight="bold"
+    font-family="Trebuchet MS, Arial, sans-serif"
+    color="#3b1d6c"
+    font-size="15px"
+    padding-left="25px"
+    padding-right="25px"
+  />
+
+  <mj-class
+    name="tiny-text"
+    align="center"
+    color="#000"
+    font-size="15px"
+    font-family="Verdana, Arial, sans-serif"
+    line-height="20px"
+  />
+
+  <mj-class
+    name="btn"
+    align="center"
+    font-size="22px"
+    background-color="#4c258b"
+    border-radius="4px"
+    color="#fff"
+    font-family="open Sans Helvetica, Arial, sans-serif"
+  />
+
+  <mj-class
+    name="main-section"
+    background-color="#cccccc"
+    vertical-align="top"
+    padding-bottom="20px"
+    padding-top="0"
+  />
+
+  <mj-class
+    name="main-section-image"
+    background-color="#cccccc"
+    vertical-align="top"
+    padding-bottom="0"
+    padding-top="0"
+  />
+
+  <mj-class
+    name="bottom-section"
+    background-color="#ffffff"
+    vertical-align="top"
+    padding-bottom="20px"
+    padding-top="20px"
+  />
+</mj-attributes>

package/starter/src/autoMap/addHandlers.js

@@ -0,0 +1,167 @@
+const _ = require("lodash");
+const db = require("db");
+
+const ifUpdated = require("helpers/db/ifUpdated");
+
+const schemaMappings = require("./schemaMappings");
+
+const getDependentFields = (schema, dependentFieldName) => {
+  let targetSchema = schema[dependentFieldName];
+
+  if (targetSchema.type === "array") {
+    [targetSchema] = targetSchema.items;
+  }
+
+  return Object.keys(targetSchema.keys).filter(
+    (key) => !_.includes(["_id", "createdOn", "updatedOn"], key)
+  );
+};
+
+const updatedSchemaMappings = (() => {
+  const result = {};
+
+  const schemaNames = Object.keys(schemaMappings);
+
+  schemaNames.forEach((schemaName) => {
+    const dependentFieldNames = Object.keys(schemaMappings[schemaName]);
+    const schema = db.schemas[schemaName].describe().keys;
+
+    dependentFieldNames.forEach((dependentFieldName) => {
+      const dependentFields = getDependentFields(schema, dependentFieldName);
+
+      if (!_.isEmpty(dependentFields)) {
+        result[schemaName] = result[schemaName] || {};
+
+        result[schemaName][dependentFieldName] = {
+          schema: schemaMappings[schemaName][dependentFieldName].schema,
+          dependentFields,
+        };
+      }
+    });
+  });
+
+  return result;
+})();
+
+const getToUpdate = async ({ doc, schemaName }) => {
+  const dependentFieldNames = Object.keys(updatedSchemaMappings[schemaName]);
+
+  const toUpdate = {};
+
+  await Promise.all(
+    dependentFieldNames.map(async (dependentFieldName) => {
+      const { schema: dependentSchemaName } =
+        updatedSchemaMappings[schemaName][dependentFieldName];
+
+      const { dependentFields } =
+        updatedSchemaMappings[schemaName][dependentFieldName];
+
+      if (!_.isEmpty(doc[dependentFieldName])) {
+        if (_.isArray(doc[dependentFieldName])) {
+          toUpdate[dependentFieldName] = (
+            await db.services[dependentSchemaName].find(
+              {
+                _id: { $in: _.uniq(_.map(doc[dependentFieldName], "_id")) },
+              },
+              {
+                fields: dependentFields,
+              }
+            )
+          ).results;
+        } else {
+          toUpdate[dependentFieldName] = await db.services[
+            dependentSchemaName
+          ].findOne(
+            {
+              _id: doc[dependentFieldName]._id,
+            },
+            {
+              fields: dependentFields,
+            }
+          );
+        }
+      }
+    })
+  );
+
+  return toUpdate;
+};
+
+const populateOnCreate = ({ schemaName }) => {
+  db.services[schemaName]._options.onBeforeCreated = async ({ docs }) => {
+    const res = await Promise.all(
+      docs.map(async (doc) => {
+        const toUpdate = await getToUpdate({ schemaName, doc });
+
+        return {
+          ...doc,
+          ...toUpdate,
+        };
+      })
+    );
+
+    return res;
+  };
+};
+
+// const addOnEntityCreatedHandler = ({ schemaName }) => {
+//   db.services[schemaName].on('created', async ({ doc }) => {
+//     const toUpdate = await getToUpdate({ schemaName, doc });
+
+//     if (!_.isEmpty(toUpdate)) {
+//       await db.services[schemaName].atomic.update(
+//         { _id: doc._id },
+//         { $set: toUpdate }
+//       );
+//     }
+//   });
+// };
+
+const addOnDependentEntitiesUpdatedHandlers = ({ schemaName }) => {
+  const dependentFieldNames = Object.keys(schemaMappings[schemaName]);
+
+  dependentFieldNames.forEach((dependentFieldName) => {
+    const dependentFieldSchemaName =
+      schemaMappings[schemaName][dependentFieldName].schema;
+    const schema = db.schemas[schemaName].describe().keys;
+
+    const dependentFields = getDependentFields(schema, dependentFieldName);
+
+    db.services[dependentFieldSchemaName].on(
+      "updated",
+      ifUpdated(dependentFields, async ({ doc }) => {
+        const toUpdate = _.pick(doc, ["_id", ...dependentFields]);
+
+        if (schema[dependentFieldName].type === "array") {
+          db.services[schemaName].atomic.update(
+            { [`${dependentFieldName}._id`]: doc._id },
+            {
+              $set: {
+                [`${dependentFieldName}.$`]: toUpdate,
+              },
+            }
+          );
+        } else {
+          db.services[schemaName].atomic.update(
+            { [`${dependentFieldName}._id`]: doc._id },
+            {
+              $set: {
+                [`${dependentFieldName}`]: toUpdate,
+              },
+            }
+          );
+        }
+      })
+    );
+  });
+};
+
+module.exports = async () => {
+  const schemaNames = Object.keys(updatedSchemaMappings);
+
+  schemaNames.forEach((schemaName) => {
+    // addOnEntityCreatedHandler({ schemaName });
+    populateOnCreate({ schemaName });
+    addOnDependentEntitiesUpdatedHandlers({ schemaName });
+  });
+};

package/starter/src/autoMap/mapSchema.js

@@ -0,0 +1,112 @@
+const _ = require("lodash");
+
+const db = require("db");
+const schemaMappings = require("./schemaMappings");
+
+const schemaMappingService = db.services.schemaMappings;
+
+const getDependentFields = (schema, dependentFieldName) => {
+  let targetSchema = schema[dependentFieldName];
+  if (targetSchema.type === "array") {
+    targetSchema = targetSchema.items[0];
+  }
+
+  return Object.keys(targetSchema.keys).filter(
+    (key) => !_.includes(["_id", "createdOn", "updatedOn"], key)
+  );
+};
+
+const joiSchemaToSchemaMappings = () => {
+  const newSchemaMappings = {};
+
+  Object.keys(schemaMappings).forEach((schemaName) => {
+    const schema = db.schemas[schemaName].describe().keys;
+
+    newSchemaMappings[schemaName] = {};
+
+    Object.keys(schemaMappings[schemaName]).forEach((fieldName) => {
+      newSchemaMappings[schemaName][fieldName] = {
+        schema: schemaMappings[schemaName][fieldName].schema,
+        fields: getDependentFields(schema, fieldName),
+      };
+    });
+  });
+
+  return newSchemaMappings;
+};
+
+module.exports = async () => {
+  const prevSchema = await schemaMappingService.findOne({});
+  const prevSchemaMappings = prevSchema?.mappings;
+
+  if (!prevSchemaMappings) {
+    const initialSchemaMappings = joiSchemaToSchemaMappings();
+
+    schemaMappingService.create({ mappings: initialSchemaMappings });
+  } else {
+    const schemaNames = Object.keys(schemaMappings);
+
+    await Promise.all(
+      schemaNames.map(async (schemaName) => {
+        const schema = db.schemas[schemaName].describe().keys;
+
+        const fieldNames = Object.keys(schemaMappings[schemaName]);
+
+        await Promise.all(
+          fieldNames.map(async (fieldName) => {
+            const dependentFields = getDependentFields(schema, fieldName);
+
+            const prevSchema = (prevSchemaMappings[schemaName] &&
+              prevSchemaMappings[schemaName][fieldName]) || { fields: [] };
+
+            const { fields: prevDependentFields } = prevSchema;
+
+            if (
+              _.difference(dependentFields, prevDependentFields).length !== 0
+            ) {
+              console.log(`Mapping schema changes: ${schemaName}.${fieldName}`);
+
+              const uniqueDependentEntityIds = await db.services[
+                schemaName
+              ].distinct(`${fieldName}._id`);
+
+              const { results: uniqueDependentEntities } = await db.services[
+                schemaMappings[schemaName][fieldName].schema
+              ].find(
+                {
+                  _id: { $in: uniqueDependentEntityIds },
+                },
+                { fields: ["_id", ...dependentFields] }
+              );
+
+              await Promise.all(
+                uniqueDependentEntities.map(async (entity) => {
+                  if (schema[fieldName].type === "array") {
+                    await db.services[schemaName].atomic.update(
+                      { [`${fieldName}._id`]: entity._id },
+                      {
+                        $set: { [`${fieldName}.$`]: entity },
+                      }
+                    );
+                  } else {
+                    await db.services[schemaName].atomic.update(
+                      { [`${fieldName}._id`]: entity._id },
+                      {
+                        $set: { [`${fieldName}`]: entity },
+                      }
+                    );
+                  }
+                })
+              );
+            }
+          })
+        );
+      })
+    );
+
+    schemaMappingService.atomic.update(
+      { _id: prevSchema._id },
+      { $set: { mappings: joiSchemaToSchemaMappings() } }
+    );
+  }
+};

package/starter/src/autoMap/schemaMappings.js

@@ -0,0 +1,7 @@
+module.exports = {
+  // contacts: {
+  //   master: {
+  //     schema: 'users',
+  //   },
+  // },
+};

package/starter/src/config/app.js

@@ -0,0 +1,3 @@
+module.exports = {
+  // myVar: process.env.MY_VAR
+};

package/starter/src/config/index.js

@@ -0,0 +1,24 @@
+require("dotenv").config({ path: `${__dirname}/.env` });
+require("dotenv").config({ path: `${__dirname}/.env.app` });
+
+const appConfig = require("./app");
+
+const env = process.env.APP_ENV || "development";
+
+const config = {
+  env,
+  port: process.env.PORT || 3001,
+  isDev: env === "development",
+
+  projectId: process.env.PROJECT_ID,
+
+  mongoUri: process.env.MONGODB_URI,
+
+  redis: {
+    url: process.env.REDIS_URI,
+  },
+
+  ...appConfig,
+};
+
+module.exports = config;

package/starter/src/db.js

@@ -0,0 +1,48 @@
+const fs = require("fs");
+const _ = require("lodash");
+const requireDir = require("require-dir");
+
+const getSchemas = require("helpers/getSchemas");
+const getResources = require("helpers/getResources");
+
+const config = require("config");
+const db = require("lib/node-mongo").connect(config.mongoUri);
+
+db.services = {};
+db.schemas = {};
+
+db.init = async () => {
+  const schemaPaths = await getSchemas();
+
+  _.each(
+    schemaPaths,
+    ({ file: schemaFile, resourceName, name: schemaName }) => {
+      const schema = require(schemaFile);
+      db.schemas[schemaName] = schema;
+
+      console.log("Registering service", resourceName);
+
+      db.services[schemaName] = db.createService(`${resourceName}`, {
+        validate: (obj) => schema.validate(obj, { allowUnknown: true }),
+      });
+    }
+  );
+
+  const resourcePaths = await getResources();
+
+  console.log('resourcePaths', resourcePaths);
+
+  _.each(resourcePaths, ({ dir }) => {
+    if (fs.existsSync(`${dir}/handlers`)) {
+      requireDir(`${dir}/handlers`);
+    }
+  });
+
+  const mapSchema = require("autoMap/mapSchema");
+  await mapSchema();
+
+  const addHandlers = require("autoMap/addHandlers");
+  await addHandlers();
+};
+
+module.exports = db;

package/starter/src/helpers/db/ifUpdated.js

@@ -0,0 +1,22 @@
+const _ = require("lodash");
+
+module.exports = (fieldNames, callback) => {
+  return async ({ doc, prevDoc }) => {
+    let isFieldChanged = false;
+
+    _.forEach(fieldNames, (fieldName) => {
+      if (!_.isEqual(doc[fieldName], prevDoc[fieldName])) {
+        isFieldChanged = true;
+        return false; // break loop
+      }
+
+      return true;
+    });
+
+    if (isFieldChanged) {
+      return callback({ doc, prevDoc });
+    }
+
+    return () => {};
+  };
+};

package/starter/src/helpers/getResourceEndpoints.js

@@ -0,0 +1,26 @@
+const fs = require("fs");
+const _ = require("lodash");
+
+const {
+  promises: { readdir },
+} = fs;
+
+module.exports = async (resourceName) => {
+  console.log('get resource endpoint', resourceName);
+  let isHiveEndpoint;
+  
+  if (fs.existsSync(`${__dirname}/../resources/${resourceName}/endpoints`) || (process.env.HIVE_SRC && (isHiveEndpoint = true) && fs.existsSync(`${process.env.HIVE_SRC}/resources/${resourceName}/endpoints`)) ) {
+    const endpointFiles = await readdir(
+      isHiveEndpoint ? 
+      `${process.env.HIVE_SRC}/resources/${resourceName}/endpoints` :
+      `${__dirname}/../resources/${resourceName}/endpoints`
+    );
+
+    return endpointFiles.map((f) => ({
+      file: isHiveEndpoint ?  `${process.env.HIVE_SRC}/resources/${resourceName}/endpoints/${f}`: `${__dirname}/../resources/${resourceName}/endpoints/${f}`,
+      name: _.last(f.split("/")).replace(".js", ""),
+    }));
+  }
+
+  return [];
+};