@budibase/server 2.6.19-alpha.2 → 2.6.19-alpha.21

package/jest.config.ts

@@ -1,6 +1,7 @@
 import { Config } from "@jest/types"
 
 import * as fs from "fs"
+import { join } from "path"
 const preset = require("ts-jest/jest-preset")
 
 const baseConfig: Config.InitialProjectOptions = {
@@ -49,4 +50,6 @@ const config: Config.InitialOptions = {
   coverageReporters: ["lcov", "json", "clover"],
 }
 
+process.env.TOP_LEVEL_PATH = join(__dirname, "..", "..")
+
 export default config

package/nodemon.json

@@ -6,5 +6,5 @@
     "src/**/*.spec.js",
     "../backend-core/dist/**/*"
   ],
-  "exec": "ts-node src/index.ts"
+  "exec": "node ./scripts/build.js && node ./dist/index.js"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.6.19-alpha.2",
+  "version": "2.6.19-alpha.21",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -10,22 +10,23 @@
   },
   "scripts": {
     "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/",
+    "build": "node ./scripts/build.js",
+    "check:types": "tsc -p tsconfig.build.json --noEmit",
+    "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
-    "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
-    "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build --configuration=production",
+    "build:docker": "yarn predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
     "build:docs": "node ./scripts/docs/generate.js open",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
-    "dev:builder": "yarn run dev:stack:up && nodemon",
+    "dev:builder": "yarn run dev:stack:up && rimraf dist/ && nodemon",
     "dev:built": "yarn run dev:stack:up && yarn run run:docker",
     "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
     "initialise": "node scripts/initialise.js",
@@ -45,12 +46,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.6.19-alpha.2",
-    "@budibase/client": "2.6.19-alpha.2",
-    "@budibase/pro": "2.6.19-alpha.2",
-    "@budibase/shared-core": "2.6.19-alpha.2",
-    "@budibase/string-templates": "2.6.19-alpha.2",
-    "@budibase/types": "2.6.19-alpha.2",
+    "@budibase/backend-core": "2.6.19-alpha.21",
+    "@budibase/client": "2.6.19-alpha.21",
+    "@budibase/pro": "2.6.19-alpha.21",
+    "@budibase/shared-core": "2.6.19-alpha.21",
+    "@budibase/string-templates": "2.6.19-alpha.21",
+    "@budibase/types": "2.6.19-alpha.21",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -99,6 +100,7 @@
     "mysql2": "2.3.3",
     "node-fetch": "2.6.7",
     "open": "8.4.0",
+    "openai": "^3.2.1",
     "pg": "8.10.0",
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
@@ -177,5 +179,20 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "62e6959d05c2e64f53bb4090b8f714937907efa5"
+  "nx": {
+    "targets": {
+      "test": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
+  "gitHead": "0f816b1e08b50200bf62da9f1488a6203fd80071"
 }

package/pm2.config.js

@@ -1,7 +1,7 @@
 module.exports = {
   apps: [
     {
-      script: "dist/index.js",
+      script: "./dist/index.js",
       instances: "max",
       exec_mode: "cluster",
     },

package/scripts/build.js

@@ -0,0 +1,48 @@
+#!/usr/bin/node
+const { join } = require("path")
+const fs = require("fs")
+const coreBuild = require("../../../scripts/build")
+
+const dir = join(__dirname, "..")
+const entryPath = join(dir, "src")
+const outfilePath = join(dir, "dist")
+
+/**
+ * The reasoning for this is that now our built version is simple
+ * dist/index.js - any kind of threaded approach in Node.js requires
+ * a runner file to work from - I played around with a lot of
+ * different methods, but we really want to be able to use forks.
+ *
+ * Rather than trying to rewrite so that forks run the whole system,
+ * I instead went down a path of building the individual threads so
+ * that we have runner files for each of them e.g. dist/automations.js
+ * and dist/query.js - these can be ran totally independently and then
+ * the parent process can pass down data for processing to them.
+ *
+ * The ignoring is simply to remove the files which really don't need
+ * to be built - they could be built and it wouldn't cause any issues,
+ * but this just means if any further threads are added in future
+ * they will naturally work (rather than including, which would mean
+ * adjustments to the build files).
+ */
+const ignoredFiles = ["definitions", "index", "utils"]
+const threadNames = fs
+  .readdirSync(join(dir, "src", "threads"))
+  .filter(path => !ignoredFiles.find(file => path.includes(file)))
+  .map(path => path.replace(".ts", ""))
+const files = [
+  {
+    entry: join(entryPath, "index.ts"),
+    out: join(outfilePath, "index.js"),
+  },
+]
+for (let name of threadNames) {
+  files.push({
+    entry: join(entryPath, "threads", `${name}.ts`),
+    out: join(outfilePath, `${name}.js`),
+  })
+}
+
+for (let file of files) {
+  coreBuild(file.entry, file.out)
+}

package/src/api/controllers/automation.ts

@@ -14,9 +14,16 @@ import { deleteEntityMetadata } from "../../utilities"
 import { MetadataTypes } from "../../constants"
 import { setTestFlag, clearTestFlag } from "../../utilities/redis"
 import { context, cache, events } from "@budibase/backend-core"
-import { automations } from "@budibase/pro"
-import { Automation, BBContext } from "@budibase/types"
+import { automations, features } from "@budibase/pro"
+import {
+  Automation,
+  AutomationActionStepId,
+  AutomationResults,
+  BBContext,
+} from "@budibase/types"
 import { getActionDefinitions as actionDefs } from "../../automations/actions"
+import sdk from "../../sdk"
+import { db as dbCore } from "@budibase/backend-core"
 
 async function getActionDefinitions() {
   return removeDeprecated(await actionDefs())
@@ -257,13 +264,34 @@ export async function getDefinitionList(ctx: BBContext) {
 export async function trigger(ctx: BBContext) {
   const db = context.getAppDB()
   let automation = await db.get(ctx.params.id)
-  await triggers.externalTrigger(automation, {
-    ...ctx.request.body,
-    appId: ctx.appId,
-  })
-  ctx.body = {
-    message: `Automation ${automation._id} has been triggered.`,
-    automation,
+
+  let hasCollectStep = sdk.automations.utils.checkForCollectStep(automation)
+  if (hasCollectStep && (await features.isSyncAutomationsEnabled())) {
+    const response: AutomationResults = await triggers.externalTrigger(
+      automation,
+      {
+        fields: ctx.request.body.fields,
+        timeout: ctx.request.body.timeout * 1000 || 120000,
+      },
+      { getResponses: true }
+    )
+
+    let collectedValue = response.steps.find(
+      step => step.stepId === AutomationActionStepId.COLLECT
+    )
+    ctx.body = collectedValue?.outputs
+  } else {
+    if (ctx.appId && !dbCore.isProdAppID(ctx.appId)) {
+      ctx.throw(400, "Only apps in production support this endpoint")
+    }
+    await triggers.externalTrigger(automation, {
+      ...ctx.request.body,
+      appId: ctx.appId,
+    })
+    ctx.body = {
+      message: `Automation ${automation._id} has been triggered.`,
+      automation,
+    }
   }
 }
 

package/src/api/controllers/datasource.ts

@@ -21,6 +21,7 @@ import {
   CreateDatasourceRequest,
   VerifyDatasourceRequest,
   VerifyDatasourceResponse,
+  FetchDatasourceInfoResponse,
   IntegrationBase,
   DatasourcePlus,
 } from "@budibase/types"
@@ -153,6 +154,21 @@ export async function verify(
   }
 }
 
+export async function information(
+  ctx: UserCtx<void, FetchDatasourceInfoResponse>
+) {
+  const datasourceId = ctx.params.datasourceId
+  const datasource = await sdk.datasources.get(datasourceId, { enriched: true })
+  const connector = (await getConnector(datasource)) as DatasourcePlus
+  if (!connector.getTableNames) {
+    ctx.throw(400, "Table name fetching not supported by datasource")
+  }
+  const tableNames = await connector.getTableNames()
+  ctx.body = {
+    tableNames,
+  }
+}
+
 export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)

package/src/api/controllers/row/internal.ts

@@ -415,7 +415,7 @@ export async function exportRows(ctx: UserCtx) {
 
     result = await outputProcessing(table, response)
   } else if (query) {
-    let searchResponse = await exports.search(ctx)
+    let searchResponse = await search(ctx)
     result = searchResponse.rows
   }
 

package/src/api/controllers/static/index.ts

@@ -134,7 +134,7 @@ export const serveApp = async function (ctx: any) {
             ? objectStore.getGlobalFileUrl("settings", "logoUrl")
             : "",
       })
-      const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
+      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
       ctx.body = await processString(appHbs, {
         head,
         body: html,
@@ -161,7 +161,7 @@ export const serveApp = async function (ctx: any) {
             : "",
       })
 
-      const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
+      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
       ctx.body = await processString(appHbs, {
         head,
         body: html,
@@ -177,7 +177,7 @@ export const serveBuilderPreview = async function (ctx: any) {
 
   if (!env.isJest()) {
     let appId = context.getAppId()
-    const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
+    const previewHbs = loadHandlebarsFile(`${__dirname}/preview.hbs`)
     ctx.body = await processString(previewHbs, {
       clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
     })

package/src/api/controllers/table/internal.ts

@@ -186,11 +186,7 @@ export async function destroy(ctx: any) {
 export async function bulkImport(ctx: any) {
   const db = context.getAppDB()
   const table = await sdk.tables.getTable(ctx.params.tableId)
-  const { rows } = ctx.request.body
-  await handleDataImport(ctx.user, table, rows)
-
-  // Ensure auto id and other table updates are persisted
-  await db.put(table)
-
+  const { rows, identifierFields } = ctx.request.body
+  await handleDataImport(ctx.user, table, rows, identifierFields)
   return table
 }

package/src/api/controllers/table/utils.ts

@@ -149,7 +149,12 @@ export function importToRows(
   return finalData
 }
 
-export async function handleDataImport(user: any, table: any, rows: any) {
+export async function handleDataImport(
+  user: any,
+  table: any,
+  rows: any,
+  identifierFields: Array<string> = []
+) {
   const schema: unknown = table.schema
 
   if (!rows || !isRows(rows) || !isSchema(schema)) {
@@ -161,6 +166,32 @@ export async function handleDataImport(user: any, table: any, rows: any) {
 
   let finalData: any = importToRows(data, table, user)
 
+  //Set IDs of finalData to match existing row if an update is expected
+  if (identifierFields.length > 0) {
+    const allDocs = await db.allDocs(
+      getRowParams(table._id, null, {
+        include_docs: true,
+      })
+    )
+    allDocs.rows
+      .map(existingRow => existingRow.doc)
+      .forEach((doc: any) => {
+        finalData.forEach((finalItem: any) => {
+          let match = true
+          for (const field of identifierFields) {
+            if (finalItem[field] !== doc[field]) {
+              match = false
+              break
+            }
+          }
+          if (match) {
+            finalItem._id = doc._id
+            finalItem._rev = doc._rev
+          }
+        })
+      })
+  }
+
   await quotas.addRows(finalData.length, () => db.bulkDocs(finalData), {
     tableId: table._id,
   })

package/src/api/controllers/webhook.ts

@@ -6,8 +6,11 @@ import {
   WebhookActionType,
   BBContext,
   Automation,
+  AutomationActionStepId,
 } from "@budibase/types"
 import sdk from "../../sdk"
+import * as pro from "@budibase/pro"
+
 const toJsonSchema = require("to-json-schema")
 const validate = require("jsonschema").validate
 
@@ -78,15 +81,36 @@ export async function trigger(ctx: BBContext) {
       if (webhook.action.type === WebhookActionType.AUTOMATION) {
         // trigger with both the pure request and then expand it
         // incase the user has produced a schema to bind to
-        await triggers.externalTrigger(target, {
-          body: ctx.request.body,
-          ...ctx.request.body,
-          appId: prodAppId,
-        })
-      }
-      ctx.status = 200
-      ctx.body = {
-        message: "Webhook trigger fired successfully",
+        let hasCollectStep = sdk.automations.utils.checkForCollectStep(target)
+
+        if (hasCollectStep && (await pro.features.isSyncAutomationsEnabled())) {
+          const response = await triggers.externalTrigger(
+            target,
+            {
+              body: ctx.request.body,
+              ...ctx.request.body,
+              appId: prodAppId,
+            },
+            { getResponses: true }
+          )
+
+          let collectedValue = response.steps.find(
+            (step: any) => step.stepId === AutomationActionStepId.COLLECT
+          )
+
+          ctx.status = 200
+          ctx.body = collectedValue.outputs
+        } else {
+          await triggers.externalTrigger(target, {
+            body: ctx.request.body,
+            ...ctx.request.body,
+            appId: prodAppId,
+          })
+          ctx.status = 200
+          ctx.body = {
+            message: "Webhook trigger fired successfully",
+          }
+        }
       }
     } catch (err: any) {
       if (err.status === 404) {

package/src/api/routes/automation.ts

@@ -65,7 +65,6 @@ router
   )
   .post(
     "/api/automations/:id/trigger",
-    appInfoMiddleware({ appType: AppType.PROD }),
     paramResource("id"),
     authorized(
       permissions.PermissionType.AUTOMATION,

package/src/api/routes/datasource.ts

@@ -20,6 +20,11 @@ router
     authorized(permissions.BUILDER),
     datasourceController.verify
   )
+  .get(
+    "/api/datasources/:datasourceId/info",
+    authorized(permissions.BUILDER),
+    datasourceController.information
+  )
   .get(
     "/api/datasources/:datasourceId",
     authorized(

package/src/api/routes/static.ts

@@ -5,6 +5,7 @@ import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
 import env from "../../environment"
 import { paramResource } from "../../middleware/resourceId"
+import { devClientLibPath } from "../../utilities/fileSystem"
 const { BUILDER, PermissionType, PermissionLevel } = permissions
 
 const router: Router = new Router()
@@ -17,7 +18,8 @@ router.param("file", async (file: any, ctx: any, next: any) => {
   }
   // test serves from require
   if (env.isTest()) {
-    ctx.devPath = require.resolve("@budibase/client").split(ctx.file)[0]
+    const path = devClientLibPath()
+    ctx.devPath = path.split(ctx.file)[0]
   } else if (env.isDev()) {
     // Serving the client library from your local dir in dev
     ctx.devPath = budibaseTempDir()