@paralect/hive 0.1.49 → 0.1.50-alpha.0

package/.hive/package.json

@@ -0,0 +1,98 @@
+{
+  "name": "koa-api-starter",
+  "version": "2.3.0",
+  "description": "Koa api",
+  "private": false,
+  "main": "src/app.js",
+  "author": "Paralect",
+  "license": "MIT",
+  "keywords": [
+    "koa",
+    "rest api",
+    "paralect"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/paralect/koa-api-starter.git"
+  },
+  "scripts": {
+    "build-assets": "mjml ./src/assets/emails/*.mjml -o ./src/assets/emails/dist/",
+    "build-emails": "babel src/emails --out-dir src/emails/compiled",
+    "dev": "tsx --watch src/app.js",
+    "init-project": "./bin/init-project.sh",
+    "start": "tsx src/app.js",
+    "migrate": "node ./src/migrator.js",
+    "schedule-dev": "nodemon --watch ./src ./src/scheduler ./src/scheduler.js",
+    "schedule": "node ./src/scheduler.js",
+    "precommit": "lint-staged",
+    "prepare": "husky install"
+  },
+  "dependencies": {
+    "@koa/cors": "5.0.0",
+    "@koa/multer": "3.0.2",
+    "@koa/router": "12.0.1",
+    "@paralect/node-mongo": "3.2.0",
+    "@react-email/components": "^0.0.21",
+    "@react-email/render": "^0.0.16",
+    "@sendgrid/mail": "8.1.3",
+    "@socket.io/redis-adapter": "8.3.0",
+    "@socket.io/redis-emitter": "5.1.0",
+    "app-module-path": "2.2.0",
+    "aws-sdk": "2.1661.0",
+    "bcryptjs": "2.4.3",
+    "bullmq": "^5.10.3",
+    "dotenv": "16.4.5",
+    "eslint-config-prettier": "9.1.0",
+    "handlebars": "4.7.8",
+    "joi": "^17.13.3",
+    "koa": "2.15.3",
+    "koa-bodyparser": "4.4.1",
+    "koa-helmet": "7.0.2",
+    "koa-logger": "3.2.1",
+    "koa-mount": "4.0.0",
+    "koa-qs": "3.0.0",
+    "lodash": "4.17.21",
+    "mjml": "4.15.3",
+    "mkdirp": "3.0.1",
+    "moment": "2.30.1",
+    "moment-duration-format": "2.3.2",
+    "monk": "^7.3.4",
+    "multer": "1.4.4",
+    "node-schedule": "2.1.1",
+    "nodemailer": "^6.9.14",
+    "prettier": "3.3.3",
+    "psl": "1.9.0",
+    "redis": "^3.1.2",
+    "require-dir": "1.2.0",
+    "slug": "^9.1.0",
+    "socket.io": "4.7.5",
+    "socket.io-emitter": "3.2.0",
+    "tail": "2.2.6",
+    "winston": "3.13.1",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.24.8",
+    "@babel/core": "^7.24.9",
+    "@babel/preset-env": "^7.24.8",
+    "@babel/preset-react": "^7.24.7",
+    "@types/node": "^20.14.11",
+    "eslint": "^8.2.0",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "eslint-plugin-import": "^2.25.2",
+    "husky": "9.1.1",
+    "lint-staged": "15.2.7",
+    "tsx": "^4.16.2",
+    "typescript": "^5.5.4"
+  },
+  "lint-staged": {
+    "*.js": [
+      "prettier --write",
+      "git add"
+    ],
+    ".js": [
+      "npm run lint:fix",
+      "npm run lint"
+    ]
+  }
+}

package/.hive/src/app-config/app.js

@@ -0,0 +1,3 @@
+export default {
+  // myVar: process.env.MY_VAR
+};

package/.hive/src/app-config/assertEnv.js

@@ -0,0 +1,15 @@
+import _ from 'lodash';
+
+export default (envs = []) => {
+  let missingEnvs = [];
+  
+  _.each(envs, envName => {
+    if (!process.env[envName]) {
+      missingEnvs.push(envName);
+    }
+  });
+
+  if (!_.isEmpty(missingEnvs)) {
+    throw Error(`Missing env variables: ${missingEnvs.join(', ')}`)
+  }
+};

package/.hive/src/app-config/index.js

@@ -0,0 +1,62 @@
+import _ from 'lodash';
+import fs from "fs";
+import dotenv from "dotenv";
+import appConfig from "./app.js";
+
+dotenv.config({ path: `${__dirname}/.env` });
+dotenv.config({ path: `${__dirname}/.env.app` });
+
+console.log('process.env.HIVE_SRC', process.env.HIVE_SRC)
+
+if (process.env.HIVE_SRC) {
+  dotenv.config({
+    path: `${process.env.HIVE_SRC}/app-config/.env`,
+  });
+
+  dotenv.config({
+    path: `${process.env.HIVE_SRC}/app-config/.env.app`,
+  });
+}
+
+const env = process.env.APP_ENV || "development";
+
+let config = {
+  async init() {
+    if (process.env.HIVE_SRC) {
+      let pluginConfigPath = `${process.env.HIVE_SRC}/app-config/app.js`;
+
+      if (fs.existsSync(pluginConfigPath)) {
+        const { default: pluginConfig } = await (import(pluginConfigPath))
+        _.extend(config, pluginConfig);
+      }
+    }
+  },
+
+  _hive: {},
+  env,
+  port: process.env.PORT || 3001,
+  domain:
+    env === "production" ? "https://api.yourapp.com" : process.env.DOMAIN || "",
+  isDev: env === "development",
+  mongoUri: process.env.MONGODB_URI,
+  redis: {
+    url: process.env.REDIS_URI,
+  },
+  smtp: {
+    host: process.env.SMTP_SERVER,
+    port: process.env.SMTP_PORT,
+    secure: true,
+    auth: {
+      user: process.env.SMTP_USER,
+      pass: process.env.SMTP_KEY,
+    },
+  },
+  ...appConfig,
+  assert(configKey) {
+    if (!config[configKey]) {
+      throw Error(`Config [${configKey}] is missing`);
+    }
+  },
+};
+
+export default config;

package/.hive/src/app.js

@@ -0,0 +1,69 @@
+
+import appModulePath from "app-module-path";
+appModulePath.addPath(__dirname);
+
+if (process.env.HIVE_SRC) {
+  appModulePath.addPath(process.env.HIVE_SRC);
+}
+
+import Koa from "koa";
+import http from "http";
+import config from "app-config";
+import logger from "logger";
+import cors from "@koa/cors";
+import helmet from "koa-helmet";
+import qs from "koa-qs";
+import bodyParser from "koa-bodyparser";
+
+import requestLogger from "koa-logger";
+import db from "db";
+import socketIo from "socketIo";
+import mount from "koa-mount";
+import get from "resources/health/endpoints/get";
+
+const main = async () => {
+  await config.init();
+  await db.init();
+
+  const routes = (await import("routes")).default;
+  const scheduler = (await import("./scheduler")).default;
+
+  process.on("unhandledRejection", (reason, p) => {
+    console.trace(reason.stack);
+    logger.error(reason.stack);
+  });
+
+  const app = new Koa();
+
+  app.use(cors({ credentials: true }));
+  app.use(helmet());
+
+  qs(app);
+
+  app.use(bodyParser({
+    enableTypes: ["json", "form", "text"],
+
+    formLimit: config._hive.bodyParser?.formLimit || "10mb",
+    textLimit: config._hive.bodyParser?.textLimit || "10mb",
+    jsonLimit: config._hive.bodyParser?.jsonLimit || "10mb",
+  }));
+
+  app.use(mount("/health", get.handler));
+  app.use(requestLogger());
+
+  await routes(app);
+
+  const server = http.createServer(app.callback());
+
+  server.listen(config.port, () => {
+    console.log(`Api server listening on ${config.port}, in ${process.env.NODE_ENV} mode and ${process.env.APP_ENV} environment`);
+  });
+
+  scheduler();
+
+  await socketIo(server);
+};
+
+main();
+
+export default main;

package/.hive/src/assets/emails/components/header.mjml

@@ -0,0 +1,13 @@
+<mj-section background-color="#ffffff" padding-bottom="0px" padding-top="0">
+  <mj-column vertical-align="top" width="100%">
+    <mj-text
+      align="center"
+      font-size="30px"
+      font-weight="bold"
+      font-family="Trebuchet MS, Arial, sans-serif"
+      color="#55299a"
+    >
+      Ship
+    </mj-text>
+  </mj-column>
+</mj-section>

package/.hive/src/assets/emails/signup-welcome.mjml

@@ -0,0 +1,34 @@
+<mjml>
+  <mj-head>
+    <mj-include path="./styles/index.mjml" />
+  </mj-head>
+
+  <mj-body>
+    <mj-include path="./components/header.mjml" />
+
+    <mj-section mj-class="main-section">
+      <mj-column vertical-align="middle" width="100%">
+        <mj-text mj-class="big-text"> Hi, there. </mj-text>
+
+        <mj-text mj-class="text">
+          Thanks for checking out Ship, we hope our products can make your
+          routine life a little more enjoyable.
+        </mj-text>
+
+        <mj-button mj-class="btn" href="{{verifyEmailUrl}}">
+          Verify Your Email
+        </mj-button>
+      </mj-column>
+    </mj-section>
+
+    <mj-section mj-class="bottom-section">
+      <mj-column vertical-align="middle" width="70%">
+        <mj-text mj-class="small-text"> INTERESTING FACT </mj-text>
+
+        <mj-text mj-class="tiny-text">
+          It took more than three years to build the Titanic.
+        </mj-text>
+      </mj-column>
+    </mj-section>
+  </mj-body>
+</mjml>

package/.hive/src/assets/emails/styles/index.mjml

@@ -0,0 +1,77 @@
+<mj-attributes>
+  <mj-class
+    name="big-text"
+    align="center"
+    font-weight="bold"
+    font-family="Trebuchet MS, Arial, sans-serif"
+    color="#3b1d6c"
+    font-size="30px"
+    padding-left="25px"
+    padding-right="25px"
+  />
+
+  <mj-class
+    name="text"
+    align="center"
+    color="#000"
+    font-size="18px"
+    font-family="Century Gothic, Arial, sans-serif"
+    padding-left="25px"
+    padding-right="25px"
+    line-height="26px"
+  />
+
+  <mj-class
+    name="small-text"
+    align="center"
+    font-weight="bold"
+    font-family="Trebuchet MS, Arial, sans-serif"
+    color="#3b1d6c"
+    font-size="15px"
+    padding-left="25px"
+    padding-right="25px"
+  />
+
+  <mj-class
+    name="tiny-text"
+    align="center"
+    color="#000"
+    font-size="15px"
+    font-family="Verdana, Arial, sans-serif"
+    line-height="20px"
+  />
+
+  <mj-class
+    name="btn"
+    align="center"
+    font-size="22px"
+    background-color="#4c258b"
+    border-radius="4px"
+    color="#fff"
+    font-family="open Sans Helvetica, Arial, sans-serif"
+  />
+
+  <mj-class
+    name="main-section"
+    background-color="#cccccc"
+    vertical-align="top"
+    padding-bottom="20px"
+    padding-top="0"
+  />
+
+  <mj-class
+    name="main-section-image"
+    background-color="#cccccc"
+    vertical-align="top"
+    padding-bottom="0"
+    padding-top="0"
+  />
+
+  <mj-class
+    name="bottom-section"
+    background-color="#ffffff"
+    vertical-align="top"
+    padding-bottom="20px"
+    padding-top="20px"
+  />
+</mj-attributes>

package/.hive/src/autoMap/addHandlers.js

@@ -0,0 +1,142 @@
+import _ from "lodash";
+import db from "db";
+import ifUpdated from "helpers/db/ifUpdated";
+import schemaMappings from "./schemaMappings";
+import getDependentFields from './getDependentFields';
+import isZodArray from "helpers/isZodArray";
+
+const updatedSchemaMappings = (() => {
+  const result = {};
+  const schemaNames = Object.keys(schemaMappings);
+  schemaNames.forEach((schemaName) => {
+    const dependentFieldNames = Object.keys(schemaMappings[schemaName]);
+    const schema = db.schemas[schemaName];
+    dependentFieldNames.forEach((dependentFieldName) => {
+      const dependentFields = getDependentFields(schema, dependentFieldName);
+      if (!_.isEmpty(dependentFields)) {
+        result[schemaName] = result[schemaName] || {};
+        result[schemaName][dependentFieldName] = {
+          schema: schemaMappings[schemaName][dependentFieldName].schema,
+          dependentFields,
+        };
+      }
+    });
+
+  });
+  return result;
+})();
+
+const getToUpdate = async ({ doc, schemaName }) => {
+  const dependentFieldNames = Object.keys(updatedSchemaMappings[schemaName]);
+  const toUpdate = {};
+  await Promise.all(
+    dependentFieldNames.map(async (dependentFieldName) => {
+      const { schema: dependentSchemaName } =
+        updatedSchemaMappings[schemaName][dependentFieldName];
+      const { dependentFields } =
+        updatedSchemaMappings[schemaName][dependentFieldName];
+      if (!_.isEmpty(doc[dependentFieldName])) {
+        if (_.isArray(doc[dependentFieldName])) {
+          toUpdate[dependentFieldName] = (
+            await db.services[dependentSchemaName].find(
+              {
+                _id: { $in: _.uniq(_.map(doc[dependentFieldName], "_id")) },
+              },
+              {
+                fields: dependentFields,
+              }
+            )
+          ).results;
+        } else {
+          toUpdate[dependentFieldName] = await db.services[
+            dependentSchemaName
+          ].findOne(
+            {
+              _id: doc[dependentFieldName]._id,
+            },
+            {
+              fields: dependentFields,
+            }
+          );
+        }
+      }
+    })
+  );
+  return toUpdate;
+};
+
+const populateOnCreate = ({ schemaName }) => {
+  db.services[schemaName]._options.onBeforeCreated = async ({ docs }) => {
+    const res = await Promise.all(
+      docs.map(async (doc) => {
+        const toUpdate = await getToUpdate({ schemaName, doc });
+        return {
+          ...doc,
+          ...toUpdate,
+        };
+      })
+    );
+    return res;
+  };
+};
+
+// const addOnEntityCreatedHandler = ({ schemaName }) => {
+//   db.services[schemaName].on('created', async ({ doc }) => {
+//     const toUpdate = await getToUpdate({ schemaName, doc });
+//     if (!_.isEmpty(toUpdate)) {
+//       await db.services[schemaName].atomic.update(
+//         { _id: doc._id },
+//         { $set: toUpdate }
+//       );
+//     }
+//   });
+// };
+
+const addOnDependentEntitiesUpdatedHandlers = ({ schemaName }) => {
+  const dependentFieldNames = Object.keys(schemaMappings[schemaName]);
+  dependentFieldNames.forEach((dependentFieldName) => {
+    const dependentFieldSchemaName =
+      schemaMappings[schemaName][dependentFieldName].schema;
+    const schema = db.schemas[schemaName];
+    const dependentFields = getDependentFields(schema, dependentFieldName);
+    db.services[dependentFieldSchemaName].on(
+      "updated",
+      ifUpdated(dependentFields, async ({ doc }) => {
+        const toUpdate = _.pick(doc, ["_id", ...dependentFields]);
+        if (isZodArray(schema.shape[dependentFieldName])) {
+          db.services[schemaName].atomic.update(
+            { [`${dependentFieldName}._id`]: doc._id },
+            {
+              $set: {
+                [`${dependentFieldName}.$`]: toUpdate,
+              },
+            },
+            {
+              multi: true,
+            }
+          );
+        } else {
+          db.services[schemaName].atomic.update(
+            { [`${dependentFieldName}._id`]: doc._id },
+            {
+              $set: {
+                [`${dependentFieldName}`]: toUpdate,
+              },
+            },
+            {
+              multi: true,
+            }
+          );
+        }
+      })
+    );
+  });
+};
+export default async () => {
+  const schemaNames = Object.keys(updatedSchemaMappings);
+  schemaNames.forEach((schemaName) => {
+    // addOnEntityCreatedHandler({ schemaName });
+    populateOnCreate({ schemaName });
+    addOnDependentEntitiesUpdatedHandlers({ schemaName });
+  });
+};

package/.hive/src/autoMap/getDependentFields.js

@@ -0,0 +1,37 @@
+import _ from 'lodash';
+import { z } from 'zod';
+import isZodArray from "helpers/isZodArray";
+
+const getZodKeys = schema => {
+  // make sure schema is not null or undefined
+  if (schema === null || schema === undefined) return [];
+  // check if schema is nullable or optional
+  if (schema instanceof z.ZodNullable || schema instanceof z.ZodOptional) return getZodKeys(schema.unwrap());
+  // check if schema is an array
+  if (isZodArray(schema)) return getZodKeys(schema.element);
+  // check if schema is an object
+  if (schema instanceof z.ZodObject) {
+    // get key/value pairs from schema
+    const entries = Object.entries(schema.shape);
+    // loop through key/value pairs
+    return entries.flatMap(([key, value]) => {
+      // get nested keys
+      const nested = value instanceof z.ZodType ? getZodKeys(value).map(subKey => `${key}.${subKey}`) : [];
+      // return nested keys
+      return nested.length ? nested : key;
+    });
+  }
+  // return empty array
+  return [];
+};
+
+export default (schema, dependentFieldName) => {
+  let targetSchema = schema.shape[dependentFieldName];
+  let zodKeys = getZodKeys(targetSchema);
+
+  zodKeys = _.uniq(zodKeys.map(key => key.split('.')[0]));
+
+  return zodKeys.filter(
+    (key) => !_.includes(['_id', 'createdOn', 'updatedOn'], key)
+  );
+};

package/.hive/src/autoMap/mapSchema.js

@@ -0,0 +1,99 @@
+import _ from "lodash";
+import db from "db";
+
+import schemaMappings from "./schemaMappings";
+import getDependentFields from './getDependentFields';
+import isZodArray from "helpers/isZodArray";
+
+const schemaMappingService = db.services.schemaMappings;
+
+const zodSchemaToSchemaMappings = () => {
+  const newSchemaMappings = {};
+
+  Object.keys(schemaMappings).forEach((schemaName) => {
+    const schema = db.schemas[schemaName];
+
+    newSchemaMappings[schemaName] = {};
+
+    Object.keys(schemaMappings[schemaName]).forEach((fieldName) => {
+      newSchemaMappings[schemaName][fieldName] = {
+        schema: schemaMappings[schemaName][fieldName].schema,
+        fields: getDependentFields(schema, fieldName),
+      };
+    });
+  });
+  console.log('newSchemaMappings', newSchemaMappings)
+  return newSchemaMappings;
+};
+
+
+export default async () => {
+  const prevSchema = await schemaMappingService.findOne({});
+  const prevSchemaMappings = prevSchema?.mappings;
+  if (!prevSchemaMappings) {
+    const initialSchemaMappings = zodSchemaToSchemaMappings();
+    schemaMappingService.create({ mappings: initialSchemaMappings });
+  } else {
+    const schemaNames = Object.keys(schemaMappings);
+    await Promise.all(
+      schemaNames.map(async (schemaName) => {
+        const schema = db.schemas[schemaName];
+
+        const fieldNames = Object.keys(schemaMappings[schemaName]);
+        await Promise.all(
+          fieldNames.map(async (fieldName) => {
+            const dependentFields = getDependentFields(schema, fieldName);
+            const prevSchema = (prevSchemaMappings[schemaName] &&
+              prevSchemaMappings[schemaName][fieldName]) || { fields: [] };
+            const { fields: prevDependentFields } = prevSchema;
+            if (
+              _.difference(dependentFields, prevDependentFields).length !== 0
+            ) {
+              console.log(`Mapping schema changes: ${schemaName}.${fieldName}`);
+              const uniqueDependentEntityIds = await db.services[
+                schemaName
+              ].distinct(`${fieldName}._id`);
+              const { results: uniqueDependentEntities } = await db.services[
+                schemaMappings[schemaName][fieldName].schema
+              ].find(
+                {
+                  _id: { $in: uniqueDependentEntityIds },
+                },
+                { fields: ["_id", ...dependentFields] }
+              );
+              await Promise.all(
+                uniqueDependentEntities.map(async (entity) => {
+                  if (isZodArray(schema.shape[fieldName])) {
+                    await db.services[schemaName].atomic.update(
+                      { [`${fieldName}._id`]: entity._id },
+                      {
+                        $set: { [`${fieldName}.$`]: entity },
+                      },
+                      {
+                        multi: true,
+                      }
+                    );
+                  } else {
+                    await db.services[schemaName].atomic.update(
+                      { [`${fieldName}._id`]: entity._id },
+                      {
+                        $set: { [`${fieldName}`]: entity },
+                      },
+                      {
+                        multi: true,
+                      }
+                    );
+                  }
+                })
+              );
+            }
+          })
+        );
+      })
+    );
+    schemaMappingService.atomic.update(
+      { _id: prevSchema._id },
+      { $set: { mappings: zodSchemaToSchemaMappings() } }
+    );
+  }
+};

package/.hive/src/autoMap/schemaMappings.js

@@ -0,0 +1,13 @@
+import fs from 'fs';
+
+let schemaMappings = {};
+
+if (fs.existsSync('./schemaMappings.json')) {
+  schemaMappings = JSON.parse(fs.readFileSync('./schemaMappings.json', 'utf8') || '{}');
+}
+
+if (process.env.HIVE_SRC && fs.existsSync(`${process.env.HIVE_SRC}/autoMap/schemaMappings.json`)) {
+  schemaMappings = JSON.parse(fs.readFileSync(`${process.env.HIVE_SRC}/autoMap/schemaMappings.json`, 'utf8') || '{}');
+}
+
+export default schemaMappings;

package/.hive/src/autoMap/schemaMappings.json

@@ -0,0 +1,3 @@
+{
+
+}