@budibase/server 2.6.19-alpha.5 → 2.6.19-alpha.51

package/jest.config.ts

@@ -1,6 +1,7 @@
 import { Config } from "@jest/types"
 
 import * as fs from "fs"
+import { join } from "path"
 const preset = require("ts-jest/jest-preset")
 
 const baseConfig: Config.InitialProjectOptions = {
@@ -49,4 +50,6 @@ const config: Config.InitialOptions = {
   coverageReporters: ["lcov", "json", "clover"],
 }
 
+process.env.TOP_LEVEL_PATH = join(__dirname, "..", "..")
+
 export default config

package/nodemon.json

@@ -6,5 +6,5 @@
     "src/**/*.spec.js",
     "../backend-core/dist/**/*"
   ],
-  "exec": "ts-node src/index.ts"
+  "exec": "node ./scripts/build.js && node ./dist/index.js"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.6.19-alpha.5",
+  "version": "2.6.19-alpha.51",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -10,22 +10,23 @@
   },
   "scripts": {
     "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/",
+    "build": "node ./scripts/build.js",
+    "check:types": "tsc -p tsconfig.build.json --noEmit",
+    "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
-    "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
-    "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build --configuration=production",
+    "build:docker": "yarn predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
     "build:docs": "node ./scripts/docs/generate.js open",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
-    "dev:builder": "yarn run dev:stack:up && nodemon",
+    "dev:builder": "yarn run dev:stack:up && rimraf dist/ && nodemon",
     "dev:built": "yarn run dev:stack:up && yarn run run:docker",
     "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
     "initialise": "node scripts/initialise.js",
@@ -45,12 +46,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.6.19-alpha.5",
-    "@budibase/client": "2.6.19-alpha.5",
-    "@budibase/pro": "2.6.19-alpha.5",
-    "@budibase/shared-core": "2.6.19-alpha.5",
-    "@budibase/string-templates": "2.6.19-alpha.5",
-    "@budibase/types": "2.6.19-alpha.5",
+    "@budibase/backend-core": "2.6.19-alpha.51",
+    "@budibase/client": "2.6.19-alpha.51",
+    "@budibase/pro": "2.6.19-alpha.51",
+    "@budibase/shared-core": "2.6.19-alpha.51",
+    "@budibase/string-templates": "2.6.19-alpha.51",
+    "@budibase/types": "2.6.19-alpha.51",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -58,6 +59,7 @@
     "@koa/router": "8.0.8",
     "@sendgrid/mail": "7.1.1",
     "@sentry/node": "6.17.7",
+    "@socket.io/redis-adapter": "^8.2.1",
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
     "aws-sdk": "2.1030.0",
@@ -78,6 +80,7 @@
     "global-agent": "3.0.0",
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
+    "ioredis": "5.3.2",
     "jimp": "0.16.1",
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
@@ -98,7 +101,9 @@
     "mssql": "6.2.3",
     "mysql2": "2.3.3",
     "node-fetch": "2.6.7",
+    "object-sizeof": "2.6.1",
     "open": "8.4.0",
+    "openai": "^3.2.1",
     "pg": "8.10.0",
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
@@ -133,7 +138,6 @@
     "@types/bson": "4.2.0",
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
-    "@types/ioredis": "4.28.10",
     "@types/jest": "29.5.0",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
@@ -153,7 +157,6 @@
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
     "eslint": "6.8.0",
-    "ioredis-mock": "7.2.0",
     "is-wsl": "2.2.0",
     "jest": "29.5.0",
     "jest-openapi": "0.14.2",
@@ -177,5 +180,20 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "b3bfc38916955e81701a0f58c12062cd119cf45c"
+  "nx": {
+    "targets": {
+      "test": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
+  "gitHead": "04fc1505cc0fd9102aeeeed1d3d98e34f470fec2"
 }

package/pm2.config.js

@@ -1,7 +1,7 @@
 module.exports = {
   apps: [
     {
-      script: "dist/index.js",
+      script: "./dist/index.js",
       instances: "max",
       exec_mode: "cluster",
     },

package/scripts/build.js

@@ -0,0 +1,48 @@
+#!/usr/bin/node
+const { join } = require("path")
+const fs = require("fs")
+const coreBuild = require("../../../scripts/build")
+
+const dir = join(__dirname, "..")
+const entryPath = join(dir, "src")
+const outfilePath = join(dir, "dist")
+
+/**
+ * The reasoning for this is that now our built version is simple
+ * dist/index.js - any kind of threaded approach in Node.js requires
+ * a runner file to work from - I played around with a lot of
+ * different methods, but we really want to be able to use forks.
+ *
+ * Rather than trying to rewrite so that forks run the whole system,
+ * I instead went down a path of building the individual threads so
+ * that we have runner files for each of them e.g. dist/automations.js
+ * and dist/query.js - these can be ran totally independently and then
+ * the parent process can pass down data for processing to them.
+ *
+ * The ignoring is simply to remove the files which really don't need
+ * to be built - they could be built and it wouldn't cause any issues,
+ * but this just means if any further threads are added in future
+ * they will naturally work (rather than including, which would mean
+ * adjustments to the build files).
+ */
+const ignoredFiles = ["definitions", "index", "utils"]
+const threadNames = fs
+  .readdirSync(join(dir, "src", "threads"))
+  .filter(path => !ignoredFiles.find(file => path.includes(file)))
+  .map(path => path.replace(".ts", ""))
+const files = [
+  {
+    entry: join(entryPath, "index.ts"),
+    out: join(outfilePath, "index.js"),
+  },
+]
+for (let name of threadNames) {
+  files.push({
+    entry: join(entryPath, "threads", `${name}.ts`),
+    out: join(outfilePath, `${name}.js`),
+  })
+}
+
+for (let file of files) {
+  coreBuild(file.entry, file.out)
+}

package/src/api/controllers/application.ts

@@ -26,10 +26,13 @@ import {
   env as envCore,
 } from "@budibase/backend-core"
 import { USERS_TABLE_SCHEMA } from "../../constants"
-import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
+import {
+  DEFAULT_BB_DATASOURCE_ID,
+  buildDefaultDocs,
+} from "../../db/defaultData/datasource_bb_default"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
 import { stringToReadStream, isQsTrue } from "../../utilities"
-import { getLocksById } from "../../utilities/redis"
+import { getLocksById, doesUserHaveLock } from "../../utilities/redis"
 import {
   updateClientLibrary,
   backupClientLibrary,
@@ -111,11 +114,7 @@ function checkAppName(
   }
 }
 
-async function createInstance(
-  appId: string,
-  template: any,
-  includeSampleData: boolean
-) {
+async function createInstance(appId: string, template: any) {
   const db = context.getAppDB()
   await db.put({
     _id: "_design/database",
@@ -142,21 +141,25 @@ async function createInstance(
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)
-
-    if (includeSampleData) {
-      // create ootb stock db
-      await addDefaultTables(db)
-    }
   }
 
   return { _id: appId }
 }
 
-async function addDefaultTables(db: Database) {
-  const defaultDbDocs = buildDefaultDocs()
+export const addSampleData = async (ctx: UserCtx) => {
+  const db = context.getAppDB()
 
-  // add in the default db data docs - tables, datasource, rows and links
-  await db.bulkDocs([...defaultDbDocs])
+  try {
+    // Check if default datasource exists before creating it
+    await sdk.datasources.get(DEFAULT_BB_DATASOURCE_ID)
+  } catch (err: any) {
+    const defaultDbDocs = buildDefaultDocs()
+
+    // add in the default db data docs - tables, datasource, rows and links
+    await db.bulkDocs([...defaultDbDocs])
+  }
+
+  ctx.status = 200
 }
 
 export async function fetch(ctx: UserCtx) {
@@ -227,6 +230,7 @@ export async function fetchAppPackage(ctx: UserCtx) {
     screens,
     layouts,
     clientLibPath,
+    hasLock: await doesUserHaveLock(application.appId, ctx.user),
   }
 }
 
@@ -247,16 +251,11 @@ async function performAppCreate(ctx: UserCtx) {
   if (ctx.request.files && ctx.request.files.templateFile) {
     instanceConfig.file = ctx.request.files.templateFile
   }
-  const includeSampleData = isQsTrue(ctx.request.body.sampleData)
   const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
   const appId = generateDevAppID(generateAppID(tenantId))
 
   return await context.doInAppContext(appId, async () => {
-    const instance = await createInstance(
-      appId,
-      instanceConfig,
-      includeSampleData
-    )
+    const instance = await createInstance(appId, instanceConfig)
     const db = context.getAppDB()
 
     let newApplication: App = {

package/src/api/controllers/automation.ts

@@ -14,9 +14,16 @@ import { deleteEntityMetadata } from "../../utilities"
 import { MetadataTypes } from "../../constants"
 import { setTestFlag, clearTestFlag } from "../../utilities/redis"
 import { context, cache, events } from "@budibase/backend-core"
-import { automations } from "@budibase/pro"
-import { Automation, BBContext } from "@budibase/types"
+import { automations, features } from "@budibase/pro"
+import {
+  Automation,
+  AutomationActionStepId,
+  AutomationResults,
+  BBContext,
+} from "@budibase/types"
 import { getActionDefinitions as actionDefs } from "../../automations/actions"
+import sdk from "../../sdk"
+import { db as dbCore } from "@budibase/backend-core"
 
 async function getActionDefinitions() {
   return removeDeprecated(await actionDefs())
@@ -257,13 +264,34 @@ export async function getDefinitionList(ctx: BBContext) {
 export async function trigger(ctx: BBContext) {
   const db = context.getAppDB()
   let automation = await db.get(ctx.params.id)
-  await triggers.externalTrigger(automation, {
-    ...ctx.request.body,
-    appId: ctx.appId,
-  })
-  ctx.body = {
-    message: `Automation ${automation._id} has been triggered.`,
-    automation,
+
+  let hasCollectStep = sdk.automations.utils.checkForCollectStep(automation)
+  if (hasCollectStep && (await features.isSyncAutomationsEnabled())) {
+    const response: AutomationResults = await triggers.externalTrigger(
+      automation,
+      {
+        fields: ctx.request.body.fields,
+        timeout: ctx.request.body.timeout * 1000 || 120000,
+      },
+      { getResponses: true }
+    )
+
+    let collectedValue = response.steps.find(
+      step => step.stepId === AutomationActionStepId.COLLECT
+    )
+    ctx.body = collectedValue?.outputs
+  } else {
+    if (ctx.appId && !dbCore.isProdAppID(ctx.appId)) {
+      ctx.throw(400, "Only apps in production support this endpoint")
+    }
+    await triggers.externalTrigger(automation, {
+      ...ctx.request.body,
+      appId: ctx.appId,
+    })
+    ctx.body = {
+      message: `Automation ${automation._id} has been triggered.`,
+      automation,
+    }
   }
 }
 

package/src/api/controllers/datasource.ts

@@ -21,11 +21,13 @@ import {
   CreateDatasourceRequest,
   VerifyDatasourceRequest,
   VerifyDatasourceResponse,
+  FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
   IntegrationBase,
   DatasourcePlus,
 } from "@budibase/types"
 import sdk from "../../sdk"
+import { builderSocket } from "../../websockets"
 
 function getErrorTables(errors: any, errorType: string) {
   return Object.entries(errors)
@@ -56,6 +58,21 @@ async function getConnector(
   return new Connector(datasource.config)
 }
 
+async function getAndMergeDatasource(datasource: Datasource) {
+  let existingDatasource: undefined | Datasource
+  if (datasource._id) {
+    existingDatasource = await sdk.datasources.get(datasource._id)
+  }
+  let enrichedDatasource = datasource
+  if (existingDatasource) {
+    enrichedDatasource = sdk.datasources.mergeConfigs(
+      datasource,
+      existingDatasource
+    )
+  }
+  return await sdk.datasources.enrich(enrichedDatasource)
+}
+
 async function buildSchemaHelper(datasource: Datasource) {
   const connector = (await getConnector(datasource)) as DatasourcePlus
   await connector.buildSchema(datasource._id!, datasource.entities!)
@@ -131,17 +148,7 @@ export async function verify(
   ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
 ) {
   const { datasource } = ctx.request.body
-  let existingDatasource: undefined | Datasource
-  if (datasource._id) {
-    existingDatasource = await sdk.datasources.get(datasource._id)
-  }
-  let enrichedDatasource = datasource
-  if (existingDatasource) {
-    enrichedDatasource = sdk.datasources.mergeConfigs(
-      datasource,
-      existingDatasource
-    )
-  }
+  const enrichedDatasource = await getAndMergeDatasource(datasource)
   const connector = await getConnector(enrichedDatasource)
   if (!connector.testConnection) {
     ctx.throw(400, "Connection information verification not supported")
@@ -155,11 +162,11 @@ export async function verify(
 }
 
 export async function information(
-  ctx: UserCtx<void, FetchDatasourceInfoResponse>
+  ctx: UserCtx<FetchDatasourceInfoRequest, FetchDatasourceInfoResponse>
 ) {
-  const datasourceId = ctx.params.datasourceId
-  const datasource = await sdk.datasources.get(datasourceId, { enriched: true })
-  const connector = (await getConnector(datasource)) as DatasourcePlus
+  const { datasource } = ctx.request.body
+  const enrichedDatasource = await getAndMergeDatasource(datasource)
+  const connector = (await getConnector(enrichedDatasource)) as DatasourcePlus
   if (!connector.getTableNames) {
     ctx.throw(400, "Table name fetching not supported by datasource")
   }
@@ -296,6 +303,7 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   ctx.body = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),
   }
+  builderSocket?.emitDatasourceUpdate(ctx, datasource)
 }
 
 export async function save(
@@ -338,6 +346,7 @@ export async function save(
     response.error = schemaError
   }
   ctx.body = response
+  builderSocket?.emitDatasourceUpdate(ctx, datasource)
 }
 
 async function destroyInternalTablesBySourceId(datasourceId: string) {
@@ -397,6 +406,7 @@ export async function destroy(ctx: UserCtx) {
 
   ctx.message = `Datasource deleted.`
   ctx.status = 200
+  builderSocket?.emitDatasourceDeletion(ctx, datasourceId)
 }
 
 export async function find(ctx: UserCtx) {

package/src/api/controllers/deploy/index.ts

@@ -104,6 +104,7 @@ export async function fetchDeployments(ctx: any) {
     }
     ctx.body = Object.values(deployments.history).reverse()
   } catch (err) {
+    console.error(err)
     ctx.body = []
   }
 }

package/src/api/controllers/plugin/index.ts

@@ -71,7 +71,7 @@ export async function create(ctx: any) {
 
     const doc = await pro.plugins.storePlugin(metadata, directory, source)
 
-    clientAppSocket.emit("plugins-update", { name, hash: doc.hash })
+    clientAppSocket?.emit("plugins-update", { name, hash: doc.hash })
     ctx.body = {
       message: "Plugin uploaded successfully",
       plugins: [doc],

package/src/api/controllers/row/external.ts

@@ -1,7 +1,7 @@
 import {
-  SortDirection,
   FieldTypes,
   NoEmptyFilterStrings,
+  SortDirection,
 } from "../../../constants"
 import {
   breakExternalTableId,
@@ -11,20 +11,34 @@ import { ExternalRequest, RunConfig } from "./ExternalRequest"
 import * as exporters from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
-  Operation,
-  UserCtx,
-  Row,
-  PaginationJson,
-  Table,
   Datasource,
   IncludeRelationship,
+  Operation,
+  PaginationJson,
+  Row,
   SortJson,
+  Table,
+  UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import * as utils from "./utils"
 
 const { cleanExportRows } = require("./utils")
 
+async function getRow(
+  tableId: string,
+  rowId: string,
+  opts?: { relationships?: boolean }
+) {
+  const response = (await handleRequest(Operation.READ, tableId, {
+    id: breakRowIdField(rowId),
+    includeSqlRelationships: opts?.relationships
+      ? IncludeRelationship.INCLUDE
+      : IncludeRelationship.EXCLUDE,
+  })) as Row[]
+  return response ? response[0] : response
+}
+
 export async function handleRequest(
   operation: Operation,
   tableId: string,
@@ -63,11 +77,15 @@ export async function patch(ctx: UserCtx) {
   if (!validateResult.valid) {
     throw { validation: validateResult.errors }
   }
-  return handleRequest(Operation.UPDATE, tableId, {
+  const response = await handleRequest(Operation.UPDATE, tableId, {
     id: breakRowIdField(id),
     row: inputs,
-    includeSqlRelationships: IncludeRelationship.INCLUDE,
   })
+  const row = await getRow(tableId, id, { relationships: true })
+  return {
+    ...response,
+    row,
+  }
 }
 
 export async function save(ctx: UserCtx) {
@@ -80,10 +98,20 @@ export async function save(ctx: UserCtx) {
   if (!validateResult.valid) {
     throw { validation: validateResult.errors }
   }
-  return handleRequest(Operation.CREATE, tableId, {
+  const response = await handleRequest(Operation.CREATE, tableId, {
     row: inputs,
-    includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })
+  const responseRow = response as { row: Row }
+  const rowId = responseRow.row._id
+  if (rowId) {
+    const row = await getRow(tableId, rowId, { relationships: true })
+    return {
+      ...response,
+      row,
+    }
+  } else {
+    return response
+  }
 }
 
 export async function fetchView(ctx: UserCtx) {
@@ -104,11 +132,7 @@ export async function fetch(ctx: UserCtx) {
 export async function find(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
-  const response = (await handleRequest(Operation.READ, tableId, {
-    id: breakRowIdField(id),
-    includeSqlRelationships: IncludeRelationship.EXCLUDE,
-  })) as Row[]
-  return response ? response[0] : response
+  return getRow(tableId, id)
 }
 
 export async function destroy(ctx: UserCtx) {