@budibase/server 1.0.79-alpha.8 → 1.0.80-alpha.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.79-alpha.8",
+  "version": "1.0.80-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -21,9 +21,6 @@
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
     "dev:builder": "yarn run dev:stack:up && nodemon",
-    "generate:proxy:compose": "node scripts/proxy/generateProxyConfig compose",
-    "generate:proxy:preprod": "node scripts/proxy/generateProxyConfig preprod",
-    "generate:proxy:prod": "node scripts/proxy/generateProxyConfig prod",
     "format": "prettier --config ../../.prettierrc.json 'src/**/*.ts' --write",
     "specs": "node specs/generate.js && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
     "lint": "eslint --fix src/",
@@ -74,9 +71,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.79-alpha.8",
-    "@budibase/client": "^1.0.79-alpha.8",
-    "@budibase/string-templates": "^1.0.79-alpha.8",
+    "@budibase/backend-core": "^1.0.80-alpha.0",
+    "@budibase/client": "^1.0.80-alpha.0",
+    "@budibase/string-templates": "^1.0.80-alpha.0",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -111,6 +108,7 @@
     "koa-send": "5.0.0",
     "koa-session": "5.12.0",
     "koa-static": "5.0.0",
+    "koa2-ratelimit": "^1.1.0",
     "lodash": "4.17.21",
     "memorystream": "^0.3.1",
     "mongodb": "3.6.3",
@@ -126,6 +124,7 @@
     "pouchdb-all-dbs": "1.0.2",
     "pouchdb-find": "^7.2.2",
     "pouchdb-replication-stream": "1.2.9",
+    "redis": "4",
     "server-destroy": "1.0.1",
     "svelte": "^3.38.2",
     "swagger-parser": "^10.0.3",
@@ -151,6 +150,7 @@
     "@types/koa-router": "^7.4.2",
     "@types/node": "^15.12.4",
     "@types/oracledb": "^5.2.1",
+    "@types/redis": "^4.0.11",
     "@typescript-eslint/parser": "4.28.0",
     "apidoc": "^0.50.2",
     "babel-jest": "^27.0.2",
@@ -175,5 +175,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "71629374930c0cd4c471696054dadcea093f8eb5"
+  "gitHead": "d328728978edf01385904c8cfa3a53294e112647"
 }

package/src/api/controllers/row/ExternalRequest.ts

@@ -58,7 +58,7 @@ module External {
   ) {
     const primary = table.primary
     // if passed in array need to copy for shifting etc
-    let idCopy = cloneDeep(id)
+    let idCopy: undefined | string | any[] = cloneDeep(id)
     if (filters) {
       // need to map over the filters and make sure the _id field isn't present
       for (let filter of Object.values(filters)) {

package/src/api/routes/public/index.ts

@@ -8,18 +8,54 @@ import authorized from "../../../middleware/authorized"
 import { paramResource, paramSubResource } from "../../../middleware/resourceId"
 import { CtxFn } from "./utils/Endpoint"
 import mapperMiddleware from "./middleware/mapper"
+import env from "../../../environment"
+// below imports don't have declaration files
 const Router = require("@koa/router")
+const { RateLimit, Stores } = require("koa2-ratelimit")
 const {
   PermissionLevels,
   PermissionTypes,
 } = require("@budibase/backend-core/permissions")
+const { getRedisOptions } = require("@budibase/backend-core/redis").utils
 
 const PREFIX = "/api/public/v1"
+// allow a lot more requests when in test
+const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10
+
+function getApiLimitPerSecond(): number {
+  if (!env.API_REQ_LIMIT_PER_SEC) {
+    return DEFAULT_API_REQ_LIMIT_PER_SEC
+  }
+  return parseInt(env.API_REQ_LIMIT_PER_SEC)
+}
+
+if (!env.isTest()) {
+  const REDIS_OPTS = getRedisOptions()
+  RateLimit.defaultOptions({
+    store: new Stores.Redis({
+      // @ts-ignore
+      socket: {
+        host: REDIS_OPTS.host,
+        port: REDIS_OPTS.port,
+      },
+      password: REDIS_OPTS.opts.password,
+      database: 1,
+    }),
+  })
+}
+// rate limiting, allows for 2 requests per second
+const limiter = RateLimit.middleware({
+  interval: { sec: 1 },
+  // per ip, per interval
+  max: getApiLimitPerSecond(),
+})
 
 const publicRouter = new Router({
   prefix: PREFIX,
 })
 
+publicRouter.use(limiter)
+
 function addMiddleware(
   endpoints: any,
   middleware: CtxFn,

package/src/environment.js

@@ -45,6 +45,7 @@ module.exports = {
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
+  API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC,
   // environment
   NODE_ENV: process.env.NODE_ENV,
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,

package/src/integrations/mysql.ts

@@ -16,7 +16,7 @@ import {
 import { DatasourcePlus } from "./base/datasourcePlus"
 
 module MySQLModule {
-  const mysql = require("mysql2")
+  const mysql = require("mysql2/promise")
   const Sql = require("./base/sql")
 
   interface MySQLConfig {
@@ -29,7 +29,7 @@ module MySQLModule {
   }
 
   const SCHEMA: Integration = {
-    docs: "https://github.com/mysqljs/mysql",
+    docs: "https://github.com/sidorares/node-mysql2",
     plus: true,
     friendlyName: "MySQL",
     description:
@@ -82,7 +82,7 @@ module MySQLModule {
 
   class MySQLIntegration extends Sql implements DatasourcePlus {
     private config: MySQLConfig
-    private readonly client: any
+    private client: any
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
 
@@ -92,92 +92,99 @@ module MySQLModule {
       if (config.ssl && Object.keys(config.ssl).length === 0) {
         delete config.ssl
       }
-      this.client = mysql.createConnection(config)
+      this.config = config
     }
 
     getBindingIdentifier(): string {
       return "?"
     }
 
-    internalQuery(
+    async connect() {
+      this.client = await mysql.createConnection(this.config)
+    }
+
+    async disconnect() {
+      await this.client.end()
+    }
+
+    async internalQuery(
       query: SqlQuery,
       connect: boolean = true
     ): Promise<any[] | any> {
-      const client = this.client
-      // Node MySQL is callback based, so we must wrap our call in a promise
-      return new Promise((resolve, reject) => {
+      try {
         if (connect) {
-          client.connect()
+          await this.connect()
         }
-        return client.query(
+        // Node MySQL is callback based, so we must wrap our call in a promise
+        const response = await this.client.query(
           query.sql,
-          query.bindings || {},
-          (error: any, results: object[]) => {
-            if (error) {
-              reject(error)
-            } else {
-              resolve(results)
-            }
-            if (connect) {
-              client.end()
-            }
-          }
+          query.bindings || []
         )
-      })
+        return response[0]
+      } finally {
+        if (connect) {
+          await this.disconnect()
+        }
+      }
     }
 
     async buildSchema(datasourceId: string, entities: Record<string, Table>) {
       const tables: { [key: string]: Table } = {}
       const database = this.config.database
-      this.client.connect()
-
-      // get the tables first
-      const tablesResp = await this.internalQuery(
-        { sql: "SHOW TABLES;" },
-        false
-      )
-      const tableNames = tablesResp.map(
-        (obj: any) =>
-          obj[`Tables_in_${database}`] ||
-          obj[`Tables_in_${database.toLowerCase()}`]
-      )
-      for (let tableName of tableNames) {
-        const primaryKeys = []
-        const schema: TableSchema = {}
-        const descResp = await this.internalQuery(
-          { sql: `DESCRIBE \`${tableName}\`;` },
+      await this.connect()
+
+      try {
+        // get the tables first
+        const tablesResp = await this.internalQuery(
+          { sql: "SHOW TABLES;" },
           false
         )
-        for (let column of descResp) {
-          const columnName = column.Field
-          if (column.Key === "PRI" && primaryKeys.indexOf(column.Key) === -1) {
-            primaryKeys.push(columnName)
-          }
-          const constraints = {
-            presence: column.Null !== "YES",
-          }
-          const isAuto: boolean =
-            typeof column.Extra === "string" &&
-            (column.Extra === "auto_increment" ||
-              column.Extra.toLowerCase().includes("generated"))
-          schema[columnName] = {
-            name: columnName,
-            autocolumn: isAuto,
-            type: convertSqlType(column.Type),
-            constraints,
+        const tableNames = tablesResp.map(
+          (obj: any) =>
+            obj[`Tables_in_${database}`] ||
+            obj[`Tables_in_${database.toLowerCase()}`]
+        )
+        for (let tableName of tableNames) {
+          const primaryKeys = []
+          const schema: TableSchema = {}
+          const descResp = await this.internalQuery(
+            { sql: `DESCRIBE \`${tableName}\`;` },
+            false
+          )
+          for (let column of descResp) {
+            const columnName = column.Field
+            if (
+              column.Key === "PRI" &&
+              primaryKeys.indexOf(column.Key) === -1
+            ) {
+              primaryKeys.push(columnName)
+            }
+            const constraints = {
+              presence: column.Null !== "YES",
+            }
+            const isAuto: boolean =
+              typeof column.Extra === "string" &&
+              (column.Extra === "auto_increment" ||
+                column.Extra.toLowerCase().includes("generated"))
+            schema[columnName] = {
+              name: columnName,
+              autocolumn: isAuto,
+              type: convertSqlType(column.Type),
+              constraints,
+            }
           }
-        }
-        if (!tables[tableName]) {
-          tables[tableName] = {
-            _id: buildExternalTableId(datasourceId, tableName),
-            primary: primaryKeys,
-            name: tableName,
-            schema,
+          if (!tables[tableName]) {
+            tables[tableName] = {
+              _id: buildExternalTableId(datasourceId, tableName),
+              primary: primaryKeys,
+              name: tableName,
+              schema,
+            }
           }
         }
+      } finally {
+        await this.disconnect()
       }
-
-      this.client.end()
       const final = finaliseExternalTables(tables, entities)
       this.tables = final.tables
       this.schemaErrors = final.errors
@@ -203,11 +210,13 @@ module MySQLModule {
     }
 
     async query(json: QueryJson) {
-      this.client.connect()
-      const queryFn = (query: any) => this.internalQuery(query, false)
-      const output = await this.queryWithReturning(json, queryFn)
-      this.client.end()
-      return output
+      await this.connect()
+      try {
+        const queryFn = (query: any) => this.internalQuery(query, false)
+        return await this.queryWithReturning(json, queryFn)
+      } finally {
+        await this.disconnect()
+      }
     }
   }
 

package/src/integrations/tests/mysql.spec.js

@@ -19,7 +19,7 @@ describe("MySQL Integration", () => {
     await config.integration.create({
       sql
     })
-    expect(config.integration.client.query).toHaveBeenCalledWith(sql, {}, expect.any(Function))
+    expect(config.integration.client.query).toHaveBeenCalledWith(sql, [])
   })
 
   it("calls the read method with the correct params", async () => {
@@ -27,7 +27,7 @@ describe("MySQL Integration", () => {
     await config.integration.read({
       sql
     })
-    expect(config.integration.client.query).toHaveBeenCalledWith(sql, {}, expect.any(Function))
+    expect(config.integration.client.query).toHaveBeenCalledWith(sql, [])
   })
 
   it("calls the update method with the correct params", async () => {
@@ -35,7 +35,7 @@ describe("MySQL Integration", () => {
     await config.integration.update({
       sql
     })
-    expect(config.integration.client.query).toHaveBeenCalledWith(sql, {}, expect.any(Function))
+    expect(config.integration.client.query).toHaveBeenCalledWith(sql, [])
   })
 
   it("calls the delete method with the correct params", async () => {
@@ -43,7 +43,7 @@ describe("MySQL Integration", () => {
     await config.integration.delete({
       sql
     })
-    expect(config.integration.client.query).toHaveBeenCalledWith(sql, {}, expect.any(Function))
+    expect(config.integration.client.query).toHaveBeenCalledWith(sql, [])
   })
 
   describe("no rows returned", () => {

package/src/integrations/utils.ts

@@ -40,7 +40,7 @@ const SQL_TYPE_MAP = {
 export enum SqlClients {
   MS_SQL = "mssql",
   POSTGRES = "pg",
-  MY_SQL = "mysql",
+  MY_SQL = "mysql2",
   ORACLE = "oracledb",
 }
 

package/scripts/proxy/generateProxyConfig.js

@@ -1,70 +0,0 @@
-#!/usr/bin/env node
-const path = require("path")
-const fs = require("fs")
-const { processStringSync } = require("@budibase/string-templates")
-
-const Configs = {
-  prod: {
-    k8s: true,
-    apps: "app-service.budibase.svc.cluster.local",
-    worker: "worker-service.budibase.svc.cluster.local",
-    minio: "minio-service.budibase.svc.cluster.local",
-    couchdb: "budibase-prod-svc-couchdb",
-  },
-  preprod: {
-    k8s: true,
-    apps: "app-service.budibase.svc.cluster.local",
-    worker: "worker-service.budibase.svc.cluster.local",
-    minio: "minio-service.budibase.svc.cluster.local",
-    couchdb: "budibase-preprod-svc-couchdb",
-  },
-  compose: {
-    compose: true,
-    apps: "app-service",
-    worker: "worker-service",
-    minio: "minio-service",
-    couchdb: "couchdb-service",
-    watchtower: "watchtower-service",
-  },
-}
-
-const Commands = {
-  Prod: "prod",
-  Preprod: "preprod",
-  Compose: "compose",
-}
-
-async function init(managementCommand) {
-  const config = Configs[managementCommand]
-  const hostingPath = path.join(process.cwd(), "..", "..", "hosting")
-  const nginxHbsPath = path.join(hostingPath, "nginx.prod.conf.hbs")
-  const nginxOutputPath = path.join(
-    hostingPath,
-    "proxy",
-    ".generated-nginx.prod.conf"
-  )
-  const contents = fs.readFileSync(nginxHbsPath, "utf8")
-  fs.writeFileSync(nginxOutputPath, processStringSync(contents, config))
-}
-
-const managementCommand = process.argv.slice(2)[0]
-
-if (
-  !managementCommand ||
-  !Object.values(Commands).some(command => managementCommand === command)
-) {
-  throw new Error(
-    "You must supply either a 'compose', 'preprod' or 'prod' commmand to generate an NGINX config."
-  )
-}
-
-init(managementCommand)
-  .then(() => {
-    console.log("Done! 🎉")
-  })
-  .catch(err => {
-    console.error(
-      "Something went wrong while creating the nginx configuration",
-      err.message
-    )
-  })