@budibase/server 2.6.23 → 2.6.24-alpha.1

package/jest.config.ts

@@ -1,6 +1,7 @@
 import { Config } from "@jest/types"
 
 import * as fs from "fs"
+import { join } from "path"
 const preset = require("ts-jest/jest-preset")
 
 const baseConfig: Config.InitialProjectOptions = {
@@ -20,9 +21,9 @@ const baseConfig: Config.InitialProjectOptions = {
 }
 
 // add pro sources if they exist
-if (fs.existsSync("../../../budibase-pro")) {
-  baseConfig.moduleNameMapper["@budibase/pro"] =
-    "<rootDir>/../../../budibase-pro/packages/pro/src"
+if (fs.existsSync("../pro/packages")) {
+  baseConfig.moduleNameMapper!["@budibase/pro"] =
+    "<rootDir>/../pro/packages/pro/src"
 }
 
 const config: Config.InitialOptions = {
@@ -49,4 +50,6 @@ const config: Config.InitialOptions = {
   coverageReporters: ["lcov", "json", "clover"],
 }
 
+process.env.TOP_LEVEL_PATH = join(__dirname, "..", "..")
+
 export default config

package/nodemon.json

@@ -1,6 +1,10 @@
 {
-  "watch": ["src", "../backend-core", "../../../budibase-pro/packages/pro"],
+  "watch": ["src", "../backend-core", "../pro/packages/pro"],
   "ext": "js,ts,json",
-  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../backend-core/dist/**/*"],
-  "exec": "ts-node src/index.ts"
-}
+  "ignore": [
+    "src/**/*.spec.ts",
+    "src/**/*.spec.js",
+    "../backend-core/dist/**/*"
+  ],
+  "exec": "node ./scripts/build.js && node ./dist/index.js"
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.6.23",
+  "version": "2.6.24-alpha.1",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -10,22 +10,23 @@
   },
   "scripts": {
     "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/",
+    "build": "node ./scripts/build.js",
+    "check:types": "tsc -p tsconfig.build.json --noEmit",
+    "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
-    "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
-    "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build --configuration=production",
+    "build:docker": "yarn predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
     "build:docs": "node ./scripts/docs/generate.js open",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
-    "dev:builder": "yarn run dev:stack:up && nodemon",
+    "dev:builder": "yarn run dev:stack:up && rimraf dist/ && nodemon",
     "dev:built": "yarn run dev:stack:up && yarn run run:docker",
     "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
     "initialise": "node scripts/initialise.js",
@@ -45,12 +46,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^2.6.23",
-    "@budibase/client": "^2.6.23",
-    "@budibase/pro": "2.6.22",
-    "@budibase/shared-core": "^2.6.23",
-    "@budibase/string-templates": "^2.6.23",
-    "@budibase/types": "^2.6.23",
+    "@budibase/backend-core": "2.6.24-alpha.1",
+    "@budibase/client": "2.6.24-alpha.1",
+    "@budibase/pro": "2.6.24-alpha.1",
+    "@budibase/shared-core": "2.6.24-alpha.1",
+    "@budibase/string-templates": "2.6.24-alpha.1",
+    "@budibase/types": "2.6.24-alpha.1",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -58,6 +59,7 @@
     "@koa/router": "8.0.8",
     "@sendgrid/mail": "7.1.1",
     "@sentry/node": "6.17.7",
+    "@socket.io/redis-adapter": "^8.2.1",
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
     "aws-sdk": "2.1030.0",
@@ -78,6 +80,7 @@
     "global-agent": "3.0.0",
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
+    "ioredis": "5.3.2",
     "jimp": "0.16.1",
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
@@ -98,8 +101,10 @@
     "mssql": "6.2.3",
     "mysql2": "2.3.3",
     "node-fetch": "2.6.7",
+    "object-sizeof": "2.6.1",
     "open": "8.4.0",
-    "pg": "8.5.1",
+    "openai": "^3.2.1",
+    "pg": "8.10.0",
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
     "pouchdb-adapter-memory": "7.2.2",
@@ -133,7 +138,6 @@
     "@types/bson": "4.2.0",
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
-    "@types/ioredis": "4.28.10",
     "@types/jest": "29.5.0",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
@@ -141,6 +145,7 @@
     "@types/node": "14.18.20",
     "@types/node-fetch": "2.6.1",
     "@types/oracledb": "5.2.2",
+    "@types/pg": "8.6.6",
     "@types/pouchdb": "6.4.0",
     "@types/redis": "4.0.11",
     "@types/server-destroy": "1.0.1",
@@ -152,7 +157,6 @@
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
     "eslint": "6.8.0",
-    "ioredis-mock": "7.2.0",
     "is-wsl": "2.2.0",
     "jest": "29.5.0",
     "jest-openapi": "0.14.2",
@@ -176,5 +180,20 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "565bea58fabc8b214d646ea72b24e4c11662af98"
+  "nx": {
+    "targets": {
+      "test": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
+  "gitHead": "61505132628fc70a287426cb68c551364c8cb014"
 }

package/pm2.config.js

@@ -1,7 +1,7 @@
 module.exports = {
   apps: [
     {
-      script: "dist/index.js",
+      script: "./dist/index.js",
       instances: "max",
       exec_mode: "cluster",
     },

package/scripts/build.js

@@ -0,0 +1,48 @@
+#!/usr/bin/node
+const { join } = require("path")
+const fs = require("fs")
+const coreBuild = require("../../../scripts/build")
+
+const dir = join(__dirname, "..")
+const entryPath = join(dir, "src")
+const outfilePath = join(dir, "dist")
+
+/**
+ * The reasoning for this is that now our built version is simple
+ * dist/index.js - any kind of threaded approach in Node.js requires
+ * a runner file to work from - I played around with a lot of
+ * different methods, but we really want to be able to use forks.
+ *
+ * Rather than trying to rewrite so that forks run the whole system,
+ * I instead went down a path of building the individual threads so
+ * that we have runner files for each of them e.g. dist/automations.js
+ * and dist/query.js - these can be ran totally independently and then
+ * the parent process can pass down data for processing to them.
+ *
+ * The ignoring is simply to remove the files which really don't need
+ * to be built - they could be built and it wouldn't cause any issues,
+ * but this just means if any further threads are added in future
+ * they will naturally work (rather than including, which would mean
+ * adjustments to the build files).
+ */
+const ignoredFiles = ["definitions", "index", "utils"]
+const threadNames = fs
+  .readdirSync(join(dir, "src", "threads"))
+  .filter(path => !ignoredFiles.find(file => path.includes(file)))
+  .map(path => path.replace(".ts", ""))
+const files = [
+  {
+    entry: join(entryPath, "index.ts"),
+    out: join(outfilePath, "index.js"),
+  },
+]
+for (let name of threadNames) {
+  files.push({
+    entry: join(entryPath, "threads", `${name}.ts`),
+    out: join(outfilePath, `${name}.js`),
+  })
+}
+
+for (let file of files) {
+  coreBuild(file.entry, file.out)
+}

package/src/api/controllers/application.ts

@@ -26,10 +26,13 @@ import {
   env as envCore,
 } from "@budibase/backend-core"
 import { USERS_TABLE_SCHEMA } from "../../constants"
-import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
+import {
+  DEFAULT_BB_DATASOURCE_ID,
+  buildDefaultDocs,
+} from "../../db/defaultData/datasource_bb_default"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
 import { stringToReadStream, isQsTrue } from "../../utilities"
-import { getLocksById } from "../../utilities/redis"
+import { getLocksById, doesUserHaveLock } from "../../utilities/redis"
 import {
   updateClientLibrary,
   backupClientLibrary,
@@ -111,11 +114,7 @@ function checkAppName(
   }
 }
 
-async function createInstance(
-  appId: string,
-  template: any,
-  includeSampleData: boolean
-) {
+async function createInstance(appId: string, template: any) {
   const db = context.getAppDB()
   await db.put({
     _id: "_design/database",
@@ -142,21 +141,25 @@ async function createInstance(
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)
-
-    if (includeSampleData) {
-      // create ootb stock db
-      await addDefaultTables(db)
-    }
   }
 
   return { _id: appId }
 }
 
-async function addDefaultTables(db: Database) {
-  const defaultDbDocs = buildDefaultDocs()
+export const addSampleData = async (ctx: UserCtx) => {
+  const db = context.getAppDB()
 
-  // add in the default db data docs - tables, datasource, rows and links
-  await db.bulkDocs([...defaultDbDocs])
+  try {
+    // Check if default datasource exists before creating it
+    await sdk.datasources.get(DEFAULT_BB_DATASOURCE_ID)
+  } catch (err: any) {
+    const defaultDbDocs = buildDefaultDocs()
+
+    // add in the default db data docs - tables, datasource, rows and links
+    await db.bulkDocs([...defaultDbDocs])
+  }
+
+  ctx.status = 200
 }
 
 export async function fetch(ctx: UserCtx) {
@@ -227,6 +230,7 @@ export async function fetchAppPackage(ctx: UserCtx) {
     screens,
     layouts,
     clientLibPath,
+    hasLock: await doesUserHaveLock(application.appId, ctx.user),
   }
 }
 
@@ -247,16 +251,11 @@ async function performAppCreate(ctx: UserCtx) {
   if (ctx.request.files && ctx.request.files.templateFile) {
     instanceConfig.file = ctx.request.files.templateFile
   }
-  const includeSampleData = isQsTrue(ctx.request.body.sampleData)
   const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
   const appId = generateDevAppID(generateAppID(tenantId))
 
   return await context.doInAppContext(appId, async () => {
-    const instance = await createInstance(
-      appId,
-      instanceConfig,
-      includeSampleData
-    )
+    const instance = await createInstance(appId, instanceConfig)
     const db = context.getAppDB()
 
     let newApplication: App = {

package/src/api/controllers/automation.ts

@@ -14,9 +14,16 @@ import { deleteEntityMetadata } from "../../utilities"
 import { MetadataTypes } from "../../constants"
 import { setTestFlag, clearTestFlag } from "../../utilities/redis"
 import { context, cache, events } from "@budibase/backend-core"
-import { automations } from "@budibase/pro"
-import { Automation, BBContext } from "@budibase/types"
+import { automations, features } from "@budibase/pro"
+import {
+  Automation,
+  AutomationActionStepId,
+  AutomationResults,
+  BBContext,
+} from "@budibase/types"
 import { getActionDefinitions as actionDefs } from "../../automations/actions"
+import sdk from "../../sdk"
+import { db as dbCore } from "@budibase/backend-core"
 
 async function getActionDefinitions() {
   return removeDeprecated(await actionDefs())
@@ -257,13 +264,34 @@ export async function getDefinitionList(ctx: BBContext) {
 export async function trigger(ctx: BBContext) {
   const db = context.getAppDB()
   let automation = await db.get(ctx.params.id)
-  await triggers.externalTrigger(automation, {
-    ...ctx.request.body,
-    appId: ctx.appId,
-  })
-  ctx.body = {
-    message: `Automation ${automation._id} has been triggered.`,
-    automation,
+
+  let hasCollectStep = sdk.automations.utils.checkForCollectStep(automation)
+  if (hasCollectStep && (await features.isSyncAutomationsEnabled())) {
+    const response: AutomationResults = await triggers.externalTrigger(
+      automation,
+      {
+        fields: ctx.request.body.fields,
+        timeout: ctx.request.body.timeout * 1000 || 120000,
+      },
+      { getResponses: true }
+    )
+
+    let collectedValue = response.steps.find(
+      step => step.stepId === AutomationActionStepId.COLLECT
+    )
+    ctx.body = collectedValue?.outputs
+  } else {
+    if (ctx.appId && !dbCore.isProdAppID(ctx.appId)) {
+      ctx.throw(400, "Only apps in production support this endpoint")
+    }
+    await triggers.externalTrigger(automation, {
+      ...ctx.request.body,
+      appId: ctx.appId,
+    })
+    ctx.body = {
+      message: `Automation ${automation._id} has been triggered.`,
+      automation,
+    }
   }
 }
 

package/src/api/controllers/datasource.ts

@@ -18,10 +18,88 @@ import {
   Row,
   CreateDatasourceResponse,
   UpdateDatasourceResponse,
-  UpdateDatasourceRequest,
   CreateDatasourceRequest,
+  VerifyDatasourceRequest,
+  VerifyDatasourceResponse,
+  FetchDatasourceInfoRequest,
+  FetchDatasourceInfoResponse,
+  IntegrationBase,
+  DatasourcePlus,
 } from "@budibase/types"
 import sdk from "../../sdk"
+import { builderSocket } from "../../websockets"
+
+function getErrorTables(errors: any, errorType: string) {
+  return Object.entries(errors)
+    .filter(entry => entry[1] === errorType)
+    .map(([name]) => name)
+}
+
+function updateError(error: any, newError: any, tables: string[]) {
+  if (!error) {
+    error = ""
+  }
+  if (error.length > 0) {
+    error += "\n"
+  }
+  error += `${newError} ${tables.join(", ")}`
+  return error
+}
+
+async function getConnector(
+  datasource: Datasource
+): Promise<IntegrationBase | DatasourcePlus> {
+  const Connector = await getIntegration(datasource.source)
+  // can't enrich if it doesn't have an ID yet
+  if (datasource._id) {
+    datasource = await sdk.datasources.enrich(datasource)
+  }
+  // Connect to the DB and build the schema
+  return new Connector(datasource.config)
+}
+
+async function getAndMergeDatasource(datasource: Datasource) {
+  let existingDatasource: undefined | Datasource
+  if (datasource._id) {
+    existingDatasource = await sdk.datasources.get(datasource._id)
+  }
+  let enrichedDatasource = datasource
+  if (existingDatasource) {
+    enrichedDatasource = sdk.datasources.mergeConfigs(
+      datasource,
+      existingDatasource
+    )
+  }
+  return await sdk.datasources.enrich(enrichedDatasource)
+}
+
+async function buildSchemaHelper(datasource: Datasource) {
+  const connector = (await getConnector(datasource)) as DatasourcePlus
+  await connector.buildSchema(datasource._id!, datasource.entities!)
+
+  const errors = connector.schemaErrors
+  let error = null
+  if (errors && Object.keys(errors).length > 0) {
+    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
+    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
+    if (noKey.length) {
+      error = updateError(
+        error,
+        "No primary key constraint found for the following:",
+        noKey
+      )
+    }
+    if (invalidCol.length) {
+      const invalidCols = Object.values(InvalidColumns).join(", ")
+      error = updateError(
+        error,
+        `Cannot use columns ${invalidCols} found in following:`,
+        invalidCol
+      )
+    }
+  }
+  return { tables: connector.tables, error }
+}
 
 export async function fetch(ctx: UserCtx) {
   // Get internal tables
@@ -66,6 +144,38 @@ export async function fetch(ctx: UserCtx) {
   ctx.body = [bbInternalDb, ...datasources]
 }
 
+export async function verify(
+  ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
+) {
+  const { datasource } = ctx.request.body
+  const enrichedDatasource = await getAndMergeDatasource(datasource)
+  const connector = await getConnector(enrichedDatasource)
+  if (!connector.testConnection) {
+    ctx.throw(400, "Connection information verification not supported")
+  }
+  const response = await connector.testConnection()
+
+  ctx.body = {
+    connected: response.connected,
+    error: response.error,
+  }
+}
+
+export async function information(
+  ctx: UserCtx<FetchDatasourceInfoRequest, FetchDatasourceInfoResponse>
+) {
+  const { datasource } = ctx.request.body
+  const enrichedDatasource = await getAndMergeDatasource(datasource)
+  const connector = (await getConnector(enrichedDatasource)) as DatasourcePlus
+  if (!connector.getTableNames) {
+    ctx.throw(400, "Table name fetching not supported by datasource")
+  }
+  const tableNames = await connector.getTableNames()
+  ctx.body = {
+    tableNames,
+  }
+}
+
 export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
@@ -193,6 +303,7 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   ctx.body = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),
   }
+  builderSocket?.emitDatasourceUpdate(ctx, datasource)
 }
 
 export async function save(
@@ -235,6 +346,7 @@ export async function save(
     response.error = schemaError
   }
   ctx.body = response
+  builderSocket?.emitDatasourceUpdate(ctx, datasource)
 }
 
 async function destroyInternalTablesBySourceId(datasourceId: string) {
@@ -294,6 +406,7 @@ export async function destroy(ctx: UserCtx) {
 
   ctx.message = `Datasource deleted.`
   ctx.status = 200
+  builderSocket?.emitDatasourceDeletion(ctx, datasourceId)
 }
 
 export async function find(ctx: UserCtx) {
@@ -311,51 +424,3 @@ export async function query(ctx: UserCtx) {
     ctx.throw(400, err)
   }
 }
-
-function getErrorTables(errors: any, errorType: string) {
-  return Object.entries(errors)
-    .filter(entry => entry[1] === errorType)
-    .map(([name]) => name)
-}
-
-function updateError(error: any, newError: any, tables: string[]) {
-  if (!error) {
-    error = ""
-  }
-  if (error.length > 0) {
-    error += "\n"
-  }
-  error += `${newError} ${tables.join(", ")}`
-  return error
-}
-
-async function buildSchemaHelper(datasource: Datasource) {
-  const Connector = await getIntegration(datasource.source)
-  datasource = await sdk.datasources.enrich(datasource)
-  // Connect to the DB and build the schema
-  const connector = new Connector(datasource.config)
-  await connector.buildSchema(datasource._id, datasource.entities)
-
-  const errors = connector.schemaErrors
-  let error = null
-  if (errors && Object.keys(errors).length > 0) {
-    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
-    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
-    if (noKey.length) {
-      error = updateError(
-        error,
-        "No primary key constraint found for the following:",
-        noKey
-      )
-    }
-    if (invalidCol.length) {
-      const invalidCols = Object.values(InvalidColumns).join(", ")
-      error = updateError(
-        error,
-        `Cannot use columns ${invalidCols} found in following:`,
-        invalidCol
-      )
-    }
-  }
-  return { tables: connector.tables, error }
-}

package/src/api/controllers/deploy/index.ts

@@ -104,6 +104,7 @@ export async function fetchDeployments(ctx: any) {
     }
     ctx.body = Object.values(deployments.history).reverse()
   } catch (err) {
+    console.error(err)
     ctx.body = []
   }
 }

package/src/api/controllers/integration.ts

@@ -1,4 +1,4 @@
-import { getDefinitions } from "../../integrations"
+import { getDefinition, getDefinitions } from "../../integrations"
 import { BBContext } from "@budibase/types"
 
 export async function fetch(ctx: BBContext) {
@@ -7,7 +7,7 @@ export async function fetch(ctx: BBContext) {
 }
 
 export async function find(ctx: BBContext) {
-  const defs = await getDefinitions()
+  const def = await getDefinition(ctx.params.type)
+  ctx.body = def
   ctx.status = 200
-  ctx.body = defs[ctx.params.type]
 }

package/src/api/controllers/plugin/index.ts

@@ -71,7 +71,7 @@ export async function create(ctx: any) {
 
     const doc = await pro.plugins.storePlugin(metadata, directory, source)
 
-    clientAppSocket.emit("plugins-update", { name, hash: doc.hash })
+    clientAppSocket?.emit("plugins-update", { name, hash: doc.hash })
     ctx.body = {
       message: "Plugin uploaded successfully",
       plugins: [doc],