@budibase/server 2.6.19-alpha.3 → 2.6.19-alpha.31

package/src/threads/automation.ts

@@ -19,6 +19,7 @@ import {
   AutomationStatus,
   AutomationMetadata,
   AutomationJob,
+  AutomationData,
 } from "@budibase/types"
 import {
   LoopStep,
@@ -37,8 +38,8 @@ const LOOP_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 
-function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
-  const binding = automationUtils.typecastForLooping(loopStep, input)
+function getLoopIterations(loopStep: LoopStep) {
+  let binding = loopStep.inputs.binding
   if (!binding) {
     return 0
   }
@@ -240,7 +241,9 @@ class Orchestrator {
     let loopStepNumber: any = undefined
     let loopSteps: LoopStep[] | undefined = []
     let metadata
+    let timeoutFlag = false
     let wasLoopStep = false
+    let timeout = this._job.data.event.timeout
     // check if this is a recurring automation,
     if (isProdAppID(this._appId) && isRecurring(automation)) {
       metadata = await this.getMetadata()
@@ -249,8 +252,18 @@ class Orchestrator {
         return
       }
     }
-
+    const start = performance.now()
     for (let step of automation.definition.steps) {
+      if (timeoutFlag) {
+        break
+      }
+
+      if (timeout) {
+        setTimeout(() => {
+          timeoutFlag = true
+        }, timeout || 12000)
+      }
+
       stepCount++
       let input: any,
         iterations = 1,
@@ -264,22 +277,17 @@ class Orchestrator {
 
       if (loopStep) {
         input = await processObject(loopStep.inputs, this._context)
-        iterations = getLoopIterations(loopStep as LoopStep, input)
+        iterations = getLoopIterations(loopStep as LoopStep)
       }
       for (let index = 0; index < iterations; index++) {
         let originalStepInput = cloneDeep(step.inputs)
         // Handle if the user has set a max iteration count or if it reaches the max limit set by us
         if (loopStep && input.binding) {
-          let newInput: any = await processObject(
-            loopStep.inputs,
-            cloneDeep(this._context)
-          )
-
           let tempOutput = { items: loopSteps, iterations: iterationCount }
           try {
-            newInput.binding = automationUtils.typecastForLooping(
+            loopStep.inputs.binding = automationUtils.typecastForLooping(
               loopStep as LoopStep,
-              newInput
+              loopStep.inputs as LoopInput
             )
           } catch (err) {
             this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
@@ -290,13 +298,12 @@ class Orchestrator {
             loopStep = undefined
             break
           }
-
           let item = []
           if (
             typeof loopStep.inputs.binding === "string" &&
             loopStep.inputs.option === "String"
           ) {
-            item = automationUtils.stringSplit(newInput.binding)
+            item = automationUtils.stringSplit(loopStep.inputs.binding)
           } else if (Array.isArray(loopStep.inputs.binding)) {
             item = loopStep.inputs.binding
           }
@@ -338,6 +345,7 @@ class Orchestrator {
               }
             }
           }
+
           if (
             index === env.AUTOMATION_MAX_ITERATIONS ||
             index === parseInt(loopStep.inputs.iterations)
@@ -466,8 +474,22 @@ class Orchestrator {
       }
     }
 
+    const end = performance.now()
+    const executionTime = end - start
+
+    console.log(`Execution time: ${executionTime} milliseconds`)
+
     // store the logs for the automation run
-    await storeLog(this._automation, this.executionOutput)
+    try {
+      await storeLog(this._automation, this.executionOutput)
+    } catch (e: any) {
+      if (e.status === 413 && e.request?.data) {
+        // if content is too large we shouldn't log it
+        delete e.request.data
+        e.request.data = { message: "removed due to large size" }
+      }
+      logging.logAlert("Error writing automation log", e)
+    }
     if (isProdAppID(this._appId) && isRecurring(automation) && metadata) {
       await this.updateMetadata(metadata)
     }
@@ -475,22 +497,56 @@ class Orchestrator {
   }
 }
 
-export function execute(job: Job, callback: WorkerCallback) {
+export function execute(job: Job<AutomationData>, callback: WorkerCallback) {
+  const appId = job.data.event.appId
+  const automationId = job.data.automation._id
+  if (!appId) {
+    throw new Error("Unable to execute, event doesn't contain app ID.")
+  }
+  if (!automationId) {
+    throw new Error("Unable to execute, event doesn't contain automation ID.")
+  }
+  return context.doInAutomationContext({
+    appId,
+    automationId,
+    task: async () => {
+      const envVars = await sdkUtils.getEnvironmentVariables()
+      // put into automation thread for whole context
+      await context.doInEnvironmentContext(envVars, async () => {
+        const automationOrchestrator = new Orchestrator(job)
+        try {
+          const response = await automationOrchestrator.execute()
+          callback(null, response)
+        } catch (err) {
+          callback(err)
+        }
+      })
+    },
+  })
+}
+
+export function executeSynchronously(job: Job) {
   const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
+
+  const timeoutPromise = new Promise((resolve, reject) => {
+    setTimeout(() => {
+      reject(new Error("Timeout exceeded"))
+    }, job.data.event.timeout || 12000)
+  })
+
   return context.doInAppContext(appId, async () => {
     const envVars = await sdkUtils.getEnvironmentVariables()
     // put into automation thread for whole context
-    await context.doInEnvironmentContext(envVars, async () => {
+    return context.doInEnvironmentContext(envVars, async () => {
       const automationOrchestrator = new Orchestrator(job)
-      try {
-        const response = await automationOrchestrator.execute()
-        callback(null, response)
-      } catch (err) {
-        callback(err)
-      }
+      const response = await Promise.race([
+        automationOrchestrator.execute(),
+        timeoutPromise,
+      ])
+      return response
     })
   })
 }

package/src/threads/index.ts

@@ -38,6 +38,9 @@ export class Thread {
     this.count = opts.count ? opts.count : 1
     this.disableThreading = this.shouldDisableThreading()
     if (!this.disableThreading) {
+      console.debug(
+        `[${env.FORKED_PROCESS_NAME}] initialising worker farm type=${type}`
+      )
       const workerOpts: any = {
         autoStart: true,
         maxConcurrentWorkers: this.count,
@@ -45,6 +48,7 @@ export class Thread {
           env: {
             ...process.env,
             FORKED_PROCESS: "1",
+            FORKED_PROCESS_NAME: type,
           },
         },
       }
@@ -54,6 +58,10 @@ export class Thread {
       }
       this.workers = workerFarm(workerOpts, typeToFile(type), ["execute"])
       Thread.workerRefs.push(this.workers)
+    } else {
+      console.debug(
+        `[${env.FORKED_PROCESS_NAME}] skipping worker farm type=${type}`
+      )
     }
   }
 
@@ -72,9 +80,7 @@ export class Thread {
       function fire(worker: any) {
         worker.execute(job, (err: any, response: any) => {
           if (err && err.type === "TimeoutError") {
-            reject(
-              new Error(`Query response time exceeded ${timeout}ms timeout.`)
-            )
+            reject(new Error(`Thread timeout exceeded ${timeout}ms timeout.`))
           } else if (err) {
             reject(err)
           } else {

package/src/threads/utils.ts

@@ -26,8 +26,10 @@ function makeVariableKey(queryId: string, variable: string) {
 export function threadSetup() {
   // don't run this if not threading
   if (env.isTest() || env.DISABLE_THREADING || !env.isInThread()) {
+    console.debug(`[${env.FORKED_PROCESS_NAME}] thread setup skipped`)
     return
   }
+  console.debug(`[${env.FORKED_PROCESS_NAME}] thread setup running`)
   db.init()
 }
 

package/src/utilities/fileSystem/app.ts

@@ -35,7 +35,7 @@ export const getComponentLibraryManifest = async (library: string) => {
   const filename = "manifest.json"
 
   if (env.isDev() || env.isTest()) {
-    const path = join(TOP_LEVEL_PATH, "../client", filename)
+    const path = join(TOP_LEVEL_PATH, "packages/client", filename)
     // always load from new so that updates are refreshed
     delete require.cache[require.resolve(path)]
     return require(path)

package/src/utilities/fileSystem/clientLibrary.ts

@@ -1,4 +1,4 @@
-import { join } from "path"
+import path, { join } from "path"
 import { ObjectStoreBuckets } from "../../constants"
 import fs from "fs"
 import { objectStore } from "@budibase/backend-core"
@@ -6,6 +6,10 @@ import { resolve } from "../centralPath"
 import env from "../../environment"
 import { TOP_LEVEL_PATH } from "./filesystem"
 
+export function devClientLibPath() {
+  return require.resolve("@budibase/client")
+}
+
 /**
  * Client library paths in the object store:
  * Previously, the entire client library package was downloaded from NPM
@@ -89,9 +93,10 @@ export async function updateClientLibrary(appId: string) {
   let manifest, client
 
   if (env.isDev()) {
+    const clientPath = devClientLibPath()
     // Load the symlinked version in dev which is always the newest
-    manifest = require.resolve("@budibase/client/manifest.json")
-    client = require.resolve("@budibase/client")
+    manifest = join(path.dirname(path.dirname(clientPath)), "manifest.json")
+    client = clientPath
   } else {
     // Load the bundled version in prod
     manifest = resolve(TOP_LEVEL_PATH, "client", "manifest.json")

package/src/utilities/fileSystem/filesystem.ts

@@ -4,9 +4,11 @@ import { budibaseTempDir } from "../budibaseDir"
 import { join } from "path"
 import env from "../../environment"
 import tar from "tar"
+import environment from "../../environment"
 const uuid = require("uuid/v4")
 
-export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+export const TOP_LEVEL_PATH =
+  environment.TOP_LEVEL_PATH || join(__dirname, "..", "..", "..")
 
 /**
  * Upon first startup of instance there may not be everything we need in tmp directory, set it up.

package/src/utilities/redis.ts

@@ -4,23 +4,33 @@ import { ContextUser } from "@budibase/types"
 
 const APP_DEV_LOCK_SECONDS = 600
 const AUTOMATION_TEST_FLAG_SECONDS = 60
-let devAppClient: any, debounceClient: any, flagClient: any
+let devAppClient: any, debounceClient: any, flagClient: any, socketClient: any
 
-// we init this as we want to keep the connection open all the time
+// We need to maintain a duplicate client for socket.io pub/sub
+let socketSubClient: any
+
+// We init this as we want to keep the connection open all the time
 // reduces the performance hit
 export async function init() {
   devAppClient = new redis.Client(redis.utils.Databases.DEV_LOCKS)
   debounceClient = new redis.Client(redis.utils.Databases.DEBOUNCE)
   flagClient = new redis.Client(redis.utils.Databases.FLAGS)
+  socketClient = new redis.Client(redis.utils.Databases.SOCKET_IO)
   await devAppClient.init()
   await debounceClient.init()
   await flagClient.init()
+  await socketClient.init()
+
+  // Duplicate the socket client for pub/sub
+  socketSubClient = socketClient.getClient().duplicate()
 }
 
 export async function shutdown() {
   if (devAppClient) await devAppClient.finish()
   if (debounceClient) await debounceClient.finish()
   if (flagClient) await flagClient.finish()
+  if (socketClient) await socketClient.finish()
+  if (socketSubClient) socketSubClient.disconnect()
   // shutdown core clients
   await redis.clients.shutdown()
   console.log("Redis shutdown")
@@ -86,3 +96,10 @@ export async function checkTestFlag(id: string) {
 export async function clearTestFlag(id: string) {
   await devAppClient.delete(id)
 }
+
+export function getSocketPubSubClients() {
+  return {
+    pub: socketClient.getClient(),
+    sub: socketSubClient,
+  }
+}

package/src/websockets/builder.ts

@@ -0,0 +1,69 @@
+import authorized from "../middleware/authorized"
+import Socket from "./websocket"
+import { permissions } from "@budibase/backend-core"
+import http from "http"
+import Koa from "koa"
+import { Datasource, Table } from "@budibase/types"
+import { gridSocket } from "./index"
+import { clearLock } from "../utilities/redis"
+
+export default class BuilderSocket extends Socket {
+  constructor(app: Koa, server: http.Server) {
+    super(app, server, "/socket/builder", [authorized(permissions.BUILDER)])
+
+    this.io.on("connection", socket => {
+      // Join a room for this app
+      const user = socket.data.user
+      const appId = socket.data.appId
+      socket.join(appId)
+      socket.to(appId).emit("user-update", user)
+
+      // Initial identification of connected spreadsheet
+      socket.on("get-users", async (payload, callback) => {
+        const sockets = await this.io.in(appId).fetchSockets()
+        callback({
+          users: sockets.map(socket => socket.data.user),
+        })
+      })
+
+      // Disconnection cleanup
+      socket.on("disconnect", async () => {
+        socket.to(appId).emit("user-disconnect", user)
+
+        // Remove app lock from this user if they have no other connections
+        try {
+          const sockets = await this.io.in(appId).fetchSockets()
+          const hasOtherConnection = sockets.some(socket => {
+            const { _id, sessionId } = socket.data.user
+            return _id === user._id && sessionId !== user.sessionId
+          })
+          if (!hasOtherConnection) {
+            await clearLock(appId, user)
+          }
+        } catch (e) {
+          // This is fine, just means this user didn't hold the lock
+        }
+      })
+    })
+  }
+
+  emitTableUpdate(ctx: any, table: Table) {
+    this.io.in(ctx.appId).emit("table-change", { id: table._id, table })
+    gridSocket.emitTableUpdate(table)
+  }
+
+  emitTableDeletion(ctx: any, id: string) {
+    this.io.in(ctx.appId).emit("table-change", { id, table: null })
+    gridSocket.emitTableDeletion(id)
+  }
+
+  emitDatasourceUpdate(ctx: any, datasource: Datasource) {
+    this.io
+      .in(ctx.appId)
+      .emit("datasource-change", { id: datasource._id, datasource })
+  }
+
+  emitDatasourceDeletion(ctx: any, id: string) {
+    this.io.in(ctx.appId).emit("datasource-change", { id, datasource: null })
+  }
+}

package/src/websockets/grid.ts

@@ -3,6 +3,8 @@ import Socket from "./websocket"
 import { permissions } from "@budibase/backend-core"
 import http from "http"
 import Koa from "koa"
+import { getTableId } from "../api/controllers/row/utils"
+import { Row, Table } from "@budibase/types"
 
 export default class GridSocket extends Socket {
   constructor(app: Koa, server: http.Server) {
@@ -10,7 +12,6 @@ export default class GridSocket extends Socket {
 
     this.io.on("connection", socket => {
       const user = socket.data.user
-      console.log(`Spreadsheet user connected: ${user?.id}`)
 
       // Socket state
       let currentRoom: string
@@ -19,37 +20,54 @@ export default class GridSocket extends Socket {
       socket.on("select-table", async (tableId, callback) => {
         // Leave current room
         if (currentRoom) {
-          socket.to(currentRoom).emit("user-disconnect", socket.data.user)
+          socket.to(currentRoom).emit("user-disconnect", user)
           socket.leave(currentRoom)
         }
 
         // Join new room
         currentRoom = tableId
         socket.join(currentRoom)
-        socket.to(currentRoom).emit("user-update", socket.data.user)
+        socket.to(currentRoom).emit("user-update", user)
 
         // Reply with all users in current room
         const sockets = await this.io.in(currentRoom).fetchSockets()
         callback({
           users: sockets.map(socket => socket.data.user),
-          id: user.id,
         })
       })
 
       // Handle users selecting a new cell
       socket.on("select-cell", cellId => {
-        socket.data.user.selectedCellId = cellId
+        socket.data.user.focusedCellId = cellId
         if (currentRoom) {
-          socket.to(currentRoom).emit("user-update", socket.data.user)
+          socket.to(currentRoom).emit("user-update", user)
         }
       })
 
       // Disconnection cleanup
       socket.on("disconnect", () => {
         if (currentRoom) {
-          socket.to(currentRoom).emit("user-disconnect", socket.data.user)
+          socket.to(currentRoom).emit("user-disconnect", user)
         }
       })
     })
   }
+
+  emitRowUpdate(ctx: any, row: Row) {
+    const tableId = getTableId(ctx)
+    this.io.in(tableId).emit("row-change", { id: row._id, row })
+  }
+
+  emitRowDeletion(ctx: any, id: string) {
+    const tableId = getTableId(ctx)
+    this.io.in(tableId).emit("row-change", { id, row: null })
+  }
+
+  emitTableUpdate(table: Table) {
+    this.io.in(table._id!).emit("table-change", { id: table._id, table })
+  }
+
+  emitTableDeletion(id: string) {
+    this.io.in(id).emit("table-change", { id, table: null })
+  }
 }

package/src/websockets/index.ts

@@ -1,14 +1,17 @@
 import http from "http"
 import Koa from "koa"
-import GridSocket from "./grid"
 import ClientAppSocket from "./client"
+import GridSocket from "./grid"
+import BuilderSocket from "./builder"
 
 let clientAppSocket: ClientAppSocket
 let gridSocket: GridSocket
+let builderSocket: BuilderSocket
 
 export const initialise = (app: Koa, server: http.Server) => {
   clientAppSocket = new ClientAppSocket(app, server)
   gridSocket = new GridSocket(app, server)
+  builderSocket = new BuilderSocket(app, server)
 }
 
-export { clientAppSocket, gridSocket }
+export { clientAppSocket, gridSocket, builderSocket }

package/src/websockets/websocket.ts

@@ -5,6 +5,9 @@ import Cookies from "cookies"
 import { userAgent } from "koa-useragent"
 import { auth } from "@budibase/backend-core"
 import currentApp from "../middleware/currentapp"
+import { createAdapter } from "@socket.io/redis-adapter"
+import { getSocketPubSubClients } from "../utilities/redis"
+import uuid from "uuid"
 
 export default class Socket {
   io: Server
@@ -12,7 +15,7 @@ export default class Socket {
   constructor(
     app: Koa,
     server: http.Server,
-    path: string,
+    path: string = "/",
     additionalMiddlewares?: any[]
   ) {
     this.io = new Server(server, {
@@ -59,13 +62,21 @@ export default class Socket {
         for (let [idx, middleware] of middlewares.entries()) {
           await middleware(ctx, () => {
             if (idx === middlewares.length - 1) {
-              // Middlewares are finished.
+              // Middlewares are finished
               // Extract some data from our enriched koa context to persist
               // as metadata for the socket
+              // Add user info, including a deterministic color and label
+              const { _id, email, firstName, lastName } = ctx.user
               socket.data.user = {
-                id: ctx.user._id,
-                email: ctx.user.email,
+                _id,
+                email,
+                firstName,
+                lastName,
+                sessionId: uuid.v4(),
               }
+
+              // Add app ID to help split sockets into rooms
+              socket.data.appId = ctx.appId
               next()
             }
           })
@@ -74,6 +85,11 @@ export default class Socket {
         next(error)
       }
     })
+
+    // Instantiate redis adapter
+    const { pub, sub } = getSocketPubSubClients()
+    const opts = { key: `socket.io-${path}` }
+    this.io.adapter(createAdapter(pub, sub, opts))
   }
 
   // Emit an event to all sockets

package/tsconfig.build.json

@@ -10,7 +10,15 @@
     "incremental": true,
     "types": ["node", "jest"],
     "outDir": "dist/src",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "baseUrl": ".",
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/backend-core": ["../backend-core/src"],
+      "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/shared-core": ["../shared-core/src"],
+      "@budibase/pro": ["../pro/packages/pro/src"]
+    }
   },
   "include": ["src/**/*"],
   "exclude": [

package/tsconfig.json

@@ -5,25 +5,12 @@
     "declaration": true,
     "sourceMap": true,
     "baseUrl": ".",
-    "outDir": "dist",
-    "paths": {
-      "@budibase/types": ["../types/src"],
-      "@budibase/backend-core": ["../backend-core/src"],
-      "@budibase/backend-core/*": ["../backend-core/*"],
-      "@budibase/shared-core": ["../shared-core/src"],
-      "@budibase/pro": ["../pro/packages/pro/src"]
-    }
+    "outDir": "dist"
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"],
     "swc": true
   },
-  "references": [
-    { "path": "../types" },
-    { "path": "../backend-core" },
-    { "path": "../shared-core" },
-    { "path": "../../../budibase-pro/packages/pro" }
-  ],
   "include": ["src/**/*", "specs"],
   "exclude": ["node_modules", "dist"]
 }

package/dist/api/controllers/analytics.js

@@ -1,46 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ping = exports.isEnabled = void 0;
-const backend_core_1 = require("@budibase/backend-core");
-const types_1 = require("@budibase/types");
-const utils_1 = require("../../db/utils");
-const backend_core_2 = require("@budibase/backend-core");
-const isEnabled = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
-    const enabled = yield backend_core_1.events.analytics.enabled();
-    ctx.body = {
-        enabled,
-    };
-});
-exports.isEnabled = isEnabled;
-const ping = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
-    const body = ctx.request.body;
-    switch (body.source) {
-        case types_1.PingSource.APP: {
-            const db = backend_core_2.context.getAppDB({ skip_setup: true });
-            const appInfo = yield db.get(utils_1.DocumentType.APP_METADATA);
-            let appId = backend_core_2.context.getAppId();
-            if ((0, utils_1.isDevAppID)(appId)) {
-                yield backend_core_1.events.serve.servedAppPreview(appInfo, body.timezone);
-            }
-            else {
-                yield backend_core_1.events.serve.servedApp(appInfo, body.timezone);
-            }
-            break;
-        }
-        case types_1.PingSource.BUILDER: {
-            yield backend_core_1.events.serve.servedBuilder(body.timezone);
-            break;
-        }
-    }
-    ctx.status = 200;
-});
-exports.ping = ping;