@budibase/server 1.0.190 → 1.0.192-alpha.0

package/dist/utilities/budibaseDir.js

@@ -1,9 +1,3 @@
 "use strict";
-const { join } = require("./centralPath");
-const { homedir } = require("os");
-const env = require("../environment");
 const { budibaseTempDir } = require("@budibase/backend-core/objectStore");
-module.exports.budibaseAppsDir = function () {
-    return env.BUDIBASE_DIR || join(homedir(), ".budibase");
-};
 module.exports.budibaseTempDir = budibaseTempDir;

package/dist/utilities/queue/inMemoryQueue.js

@@ -80,6 +80,14 @@ class InMemoryQueue {
         this._messages.push(newJob(this._name, msg));
         this._emitter.emit("message");
     }
+    /**
+     * replicating the close function from bull, which waits for jobs to finish.
+     */
+    close() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return [];
+        });
+    }
     /**
      * This removes a cron which has been implemented, this is part of Bull API.
      * @param {string} cronJobId The cron which is to be removed.

package/dist/utilities/redis.js

@@ -51,7 +51,7 @@ exports.getAllLocks = () => __awaiter(void 0, void 0, void 0, function* () {
 exports.updateLock = (devAppId, user) => __awaiter(void 0, void 0, void 0, function* () {
     // make sure always global user ID
     const globalId = getGlobalIDFromUserMetadataID(user._id);
-    const inputUser = Object.assign(Object.assign({}, user), { userId: globalId, _id: globalId });
+    const inputUser = Object.assign(Object.assign({}, user), { userId: globalId, _id: globalId, lockedAt: new Date().getTime() });
     yield devAppClient.store(devAppId, inputUser, APP_DEV_LOCK_SECONDS);
 });
 exports.clearLock = (devAppId, user) => __awaiter(void 0, void 0, void 0, function* () {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.190",
+  "version": "1.0.192-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,10 +70,10 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.190",
-    "@budibase/client": "^1.0.190",
-    "@budibase/pro": "1.0.189",
-    "@budibase/string-templates": "^1.0.190",
+    "@budibase/backend-core": "^1.0.192-alpha.0",
+    "@budibase/client": "^1.0.192-alpha.0",
+    "@budibase/pro": "1.0.191",
+    "@budibase/string-templates": "^1.0.192-alpha.0",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -184,5 +184,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "1a378dfe50e9a4f53900e6431eb280524390ad69"
+  "gitHead": "c4a7598c9a81e36f24c557e89d310c8145ac3a22"
 }

package/scripts/likeCypress.ts

@@ -9,7 +9,6 @@ const SERVER_PORT = "4100"
 const WORKER_PORT = "4200"
 
 // @ts-ignore
-process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
 process.env.NODE_ENV = "cypress"
 process.env.ENABLE_ANALYTICS = "false"
 process.env.JWT_SECRET = "budibase"

package/src/api/controllers/templates.js

@@ -1,12 +1,13 @@
 const fetch = require("node-fetch")
 const { downloadTemplate } = require("../../utilities/fileSystem")
+const env = require("../../environment")
 
 // development flag, can be used to test against templates exported locally
 const DEFAULT_TEMPLATES_BUCKET =
   "prod-budi-templates.s3-eu-west-1.amazonaws.com"
 
 exports.fetch = async function (ctx) {
-  const { type = "app" } = ctx.query
+  let type = env.TEMPLATE_REPOSITORY
   let response,
     error = false
   try {

package/src/api/controllers/view/exporters.js

@@ -6,7 +6,7 @@ exports.csv = function (headers, rows) {
       .map(header => {
         let val = row[header]
         val =
-          typeof val === "object"
+          typeof val === "object" && !(val instanceof Date)
             ? `"${JSON.stringify(val).replace(/"/g, "'")}"`
             : `"${val}"`
         return val.trim()

package/src/api/index.js

@@ -12,6 +12,7 @@ const { mainRoutes, staticRoutes, publicRoutes } = require("./routes")
 const pkg = require("../../package.json")
 const env = require("../environment")
 const { middleware: pro } = require("@budibase/pro")
+const { shutdown } = require("./routes/public")
 
 const router = new Router()
 
@@ -90,4 +91,5 @@ router.use(publicRoutes.allowedMethods())
 router.use(staticRoutes.routes())
 router.use(staticRoutes.allowedMethods())
 
-module.exports = router
+module.exports.router = router
+module.exports.shutdown = shutdown

package/src/api/routes/public/index.ts

@@ -29,6 +29,7 @@ function getApiLimitPerSecond(): number {
   return parseInt(env.API_REQ_LIMIT_PER_SEC)
 }
 
+let rateLimitStore: any = null
 if (!env.isTest()) {
   const REDIS_OPTS = getRedisOptions()
   let options
@@ -47,8 +48,9 @@ if (!env.isTest()) {
       database: 1,
     }
   }
+  rateLimitStore = new Stores.Redis(options)
   RateLimit.defaultOptions({
-    store: new Stores.Redis(options),
+    store: rateLimitStore,
   })
 }
 // rate limiting, allows for 2 requests per second
@@ -128,3 +130,10 @@ applyRoutes(queryEndpoints, PermissionTypes.QUERY, "queryId")
 applyRoutes(rowEndpoints, PermissionTypes.TABLE, "tableId", "rowId")
 
 export default publicRouter
+
+export const shutdown = () => {
+  if (rateLimitStore) {
+    rateLimitStore.client.disconnect()
+    rateLimitStore = null
+  }
+}

package/src/app.ts

@@ -14,6 +14,8 @@ const automations = require("./automations/index")
 const Sentry = require("@sentry/node")
 const fileSystem = require("./utilities/fileSystem")
 const bullboard = require("./automations/bullboard")
+const { logAlert } = require("@budibase/backend-core/logging")
+const { Thread } = require("./threads")
 import redis from "./utilities/redis"
 import * as migrations from "./migrations"
 
@@ -49,7 +51,7 @@ app.context.eventEmitter = eventEmitter
 app.context.auth = {}
 
 // api routes
-app.use(api.routes())
+app.use(api.router.routes())
 
 if (env.isProd()) {
   env._set("NODE_ENV", "production")
@@ -68,11 +70,24 @@ if (env.isProd()) {
 const server = http.createServer(app.callback())
 destroyable(server)
 
+let shuttingDown = false,
+  errCode = 0
 server.on("close", async () => {
-  if (env.NODE_ENV !== "jest") {
+  // already in process
+  if (shuttingDown) {
+    return
+  }
+  shuttingDown = true
+  if (!env.isTest()) {
     console.log("Server Closed")
   }
+  await automations.shutdown()
   await redis.shutdown()
+  await Thread.shutdown()
+  api.shutdown()
+  if (!env.isTest()) {
+    process.exit(errCode)
+  }
 })
 
 module.exports = server.listen(env.PORT || 0, async () => {
@@ -90,7 +105,13 @@ const shutdown = () => {
 }
 
 process.on("uncaughtException", err => {
-  console.error(err)
+  // @ts-ignore
+  // don't worry about this error, comes from zlib isn't important
+  if (err && err["code"] === "ERR_INVALID_CHAR") {
+    return
+  }
+  errCode = -1
+  logAlert("Uncaught exception.", err)
   shutdown()
 })
 
@@ -102,7 +123,7 @@ process.on("SIGTERM", () => {
 // not recommended in a clustered environment
 if (!env.HTTP_MIGRATIONS) {
   migrations.migrate().catch(err => {
-    console.error("Error performing migrations. Exiting.\n", err)
+    logAlert("Error performing migrations. Exiting.", err)
     shutdown()
   })
 }

package/src/automations/bullboard.js

@@ -45,4 +45,12 @@ exports.init = () => {
   return serverAdapter.registerPlugin()
 }
 
+exports.shutdown = async () => {
+  if (automationQueue) {
+    clearInterval(cleanupInternal)
+    await automationQueue.close()
+    automationQueue = null
+  }
+}
+
 exports.queue = automationQueue

package/src/automations/index.js

@@ -1,5 +1,5 @@
 const { processEvent } = require("./utils")
-const { queue } = require("./bullboard")
+const { queue, shutdown } = require("./bullboard")
 
 /**
  * This module is built purely to kick off the worker farm and manage the inputs/outputs
@@ -14,4 +14,9 @@ exports.init = function () {
 exports.getQueues = () => {
   return [queue]
 }
+
+exports.shutdown = () => {
+  return shutdown()
+}
+
 exports.queue = queue

package/src/automations/steps/queryRows.js

@@ -50,6 +50,7 @@ exports.definition = {
         limit: {
           type: "number",
           title: "Limit",
+          customType: "queryLimit",
         },
       },
       required: ["tableId"],

package/src/automations/steps/sendSmtpEmail.js

@@ -53,6 +53,7 @@ exports.run = async function ({ inputs }) {
   if (!contents) {
     contents = "<h1>No content</h1>"
   }
+  to = to || undefined
   try {
     let response = await sendSmtpEmail(to, from, subject, contents, true)
     return {

package/src/environment.js

@@ -43,15 +43,12 @@ module.exports = {
   COUCH_DB_URL: process.env.COUCH_DB_URL,
   MINIO_URL: process.env.MINIO_URL,
   WORKER_URL: process.env.WORKER_URL,
-  SELF_HOSTED: process.env.SELF_HOSTED,
   AWS_REGION: process.env.AWS_REGION,
-  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
-  MULTI_TENANCY: process.env.MULTI_TENANCY,
   HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
   API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
@@ -61,28 +58,27 @@ module.exports = {
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
+  TEMPLATE_REPOSITORY: process.env.TEMPLATE_REPOSITORY || "app",
   DISABLE_AUTO_PROD_APP_SYNC: process.env.DISABLE_AUTO_PROD_APP_SYNC,
   // minor
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,
   LOG_LEVEL: process.env.LOG_LEVEL,
-  AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY,
-  AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
   AUTOMATION_MAX_ITERATIONS: process.env.AUTOMATION_MAX_ITERATIONS,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
   QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT),
-  // old - to remove
-  CLIENT_ID: process.env.CLIENT_ID,
-  BUDIBASE_DIR: process.env.BUDIBASE_DIR,
-  DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL,
-  BUDIBASE_API_KEY: process.env.BUDIBASE_API_KEY,
-  USERID_API_KEY: process.env.USERID_API_KEY,
-  DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
+  SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
+  // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,
-  SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
+  DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
+  MULTI_TENANCY: process.env.MULTI_TENANCY,
+  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
+  SELF_HOSTED: process.env.SELF_HOSTED,
+  // old
+  CLIENT_ID: process.env.CLIENT_ID,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value

package/src/middleware/builder.js

@@ -6,7 +6,7 @@ const {
   setDebounce,
 } = require("../utilities/redis")
 const { doWithDB } = require("@budibase/backend-core/db")
-const { DocumentTypes } = require("../db/utils")
+const { DocumentTypes, getGlobalIDFromUserMetadataID } = require("../db/utils")
 const { PermissionTypes } = require("@budibase/backend-core/permissions")
 const { app: appCache } = require("@budibase/backend-core/cache")
 
@@ -51,6 +51,9 @@ async function updateAppUpdatedAt(ctx) {
   await doWithDB(appId, async db => {
     const metadata = await db.get(DocumentTypes.APP_METADATA)
     metadata.updatedAt = new Date().toISOString()
+
+    metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user.userId)
+
     const response = await db.put(metadata)
     metadata._rev = response.rev
     await appCache.invalidateAppMetadata(appId, metadata)
@@ -67,7 +70,15 @@ module.exports = async (ctx, permType) => {
   }
   const isBuilderApi = permType === PermissionTypes.BUILDER
   const referer = ctx.headers["referer"]
-  const editingApp = referer ? referer.includes(appId) : false
+
+  const overviewPath = "/builder/portal/overview/"
+  const overviewContext = !referer ? false : referer.includes(overviewPath)
+  if (overviewContext) {
+    return
+  }
+
+  const hasAppId = !referer ? false : referer.includes(appId)
+  const editingApp = referer ? hasAppId : false
   // check this is a builder call and editing
   if (!isBuilderApi || !editingApp) {
     return

package/src/threads/index.ts

@@ -28,6 +28,8 @@ export class Thread {
   workers: any
   timeoutMs: any
 
+  static workerRefs: any[] = []
+
   constructor(type: any, opts: any = { timeoutMs: null, count: 1 }) {
     this.type = type
     this.count = opts.count ? opts.count : 1
@@ -46,6 +48,7 @@ export class Thread {
         workerOpts.maxCallTime = opts.timeoutMs
       }
       this.workers = workerFarm(workerOpts, typeToFile(type))
+      Thread.workerRefs.push(this.workers)
     }
   }
 
@@ -73,4 +76,23 @@ export class Thread {
       })
     })
   }
+
+  static shutdown() {
+    return new Promise<void>(resolve => {
+      if (Thread.workerRefs.length === 0) {
+        resolve()
+      }
+      let count = 0
+      function complete() {
+        count++
+        if (count >= Thread.workerRefs.length) {
+          resolve()
+        }
+      }
+      for (let worker of Thread.workerRefs) {
+        workerFarm.end(worker, complete)
+      }
+      Thread.workerRefs = []
+    })
+  }
 }

package/src/utilities/budibaseDir.js

@@ -1,10 +1,3 @@
-const { join } = require("./centralPath")
-const { homedir } = require("os")
-const env = require("../environment")
 const { budibaseTempDir } = require("@budibase/backend-core/objectStore")
 
-module.exports.budibaseAppsDir = function () {
-  return env.BUDIBASE_DIR || join(homedir(), ".budibase")
-}
-
 module.exports.budibaseTempDir = budibaseTempDir

package/src/utilities/queue/inMemoryQueue.js

@@ -75,6 +75,13 @@ class InMemoryQueue {
     this._emitter.emit("message")
   }
 
+  /**
+   * replicating the close function from bull, which waits for jobs to finish.
+   */
+  async close() {
+    return []
+  }
+
   /**
    * This removes a cron which has been implemented, this is part of Bull API.
    * @param {string} cronJobId The cron which is to be removed.

package/src/utilities/redis.js

@@ -48,7 +48,9 @@ exports.updateLock = async (devAppId, user) => {
     ...user,
     userId: globalId,
     _id: globalId,
+    lockedAt: new Date().getTime(),
   }
+
   await devAppClient.store(devAppId, inputUser, APP_DEV_LOCK_SECONDS)
 }