@budibase/server 2.6.19-alpha.1 → 2.6.19-alpha.11

package/jest.config.ts

@@ -1,6 +1,7 @@
 import { Config } from "@jest/types"
 
 import * as fs from "fs"
+import { join } from "path"
 const preset = require("ts-jest/jest-preset")
 
 const baseConfig: Config.InitialProjectOptions = {
@@ -49,4 +50,6 @@ const config: Config.InitialOptions = {
   coverageReporters: ["lcov", "json", "clover"],
 }
 
+process.env.TOP_LEVEL_PATH = join(__dirname, "..", "..")
+
 export default config

package/nodemon.json

@@ -6,5 +6,5 @@
     "src/**/*.spec.js",
     "../backend-core/dist/**/*"
   ],
-  "exec": "ts-node src/index.ts"
+  "exec": "node ./scripts/build.js && node ./dist/index.js"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.6.19-alpha.1",
+  "version": "2.6.19-alpha.11",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -10,22 +10,22 @@
   },
   "scripts": {
     "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/",
+    "build": "node ./scripts/build.js",
+    "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
-    "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
-    "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build --configuration=production",
+    "build:docker": "yarn predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
     "build:docs": "node ./scripts/docs/generate.js open",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
-    "dev:builder": "yarn run dev:stack:up && nodemon",
+    "dev:builder": "yarn run dev:stack:up && rimraf dist/ && nodemon",
     "dev:built": "yarn run dev:stack:up && yarn run run:docker",
     "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
     "initialise": "node scripts/initialise.js",
@@ -45,12 +45,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.6.19-alpha.1",
-    "@budibase/client": "2.6.19-alpha.1",
-    "@budibase/pro": "2.6.19-alpha.1",
-    "@budibase/shared-core": "2.6.19-alpha.1",
-    "@budibase/string-templates": "2.6.19-alpha.1",
-    "@budibase/types": "2.6.19-alpha.1",
+    "@budibase/backend-core": "2.6.19-alpha.11",
+    "@budibase/client": "2.6.19-alpha.11",
+    "@budibase/pro": "2.6.19-alpha.11",
+    "@budibase/shared-core": "2.6.19-alpha.11",
+    "@budibase/string-templates": "2.6.19-alpha.11",
+    "@budibase/types": "2.6.19-alpha.11",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -99,6 +99,7 @@
     "mysql2": "2.3.3",
     "node-fetch": "2.6.7",
     "open": "8.4.0",
+    "openai": "^3.2.1",
     "pg": "8.10.0",
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
@@ -177,5 +178,20 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "0c1a7f3a6d7229024e20646bd75ab450abf2f9bb"
+  "nx": {
+    "targets": {
+      "test": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
+  "gitHead": "01a98bcf8dbedde393d2ad3b4019da79826ba2a1"
 }

package/pm2.config.js

@@ -1,7 +1,7 @@
 module.exports = {
   apps: [
     {
-      script: "dist/index.js",
+      script: "./dist/index.js",
       instances: "max",
       exec_mode: "cluster",
     },

package/scripts/build.js

@@ -0,0 +1,48 @@
+#!/usr/bin/node
+const { join } = require("path")
+const fs = require("fs")
+const coreBuild = require("../../../scripts/build")
+
+const dir = join(__dirname, "..")
+const entryPath = join(dir, "src")
+const outfilePath = join(dir, "dist")
+
+/**
+ * The reasoning for this is that now our built version is simple
+ * dist/index.js - any kind of threaded approach in Node.js requires
+ * a runner file to work from - I played around with a lot of
+ * different methods, but we really want to be able to use forks.
+ *
+ * Rather than trying to rewrite so that forks run the whole system,
+ * I instead went down a path of building the individual threads so
+ * that we have runner files for each of them e.g. dist/automations.js
+ * and dist/query.js - these can be ran totally independently and then
+ * the parent process can pass down data for processing to them.
+ *
+ * The ignoring is simply to remove the files which really don't need
+ * to be built - they could be built and it wouldn't cause any issues,
+ * but this just means if any further threads are added in future
+ * they will naturally work (rather than including, which would mean
+ * adjustments to the build files).
+ */
+const ignoredFiles = ["definitions", "index", "utils"]
+const threadNames = fs
+  .readdirSync(join(dir, "src", "threads"))
+  .filter(path => !ignoredFiles.find(file => path.includes(file)))
+  .map(path => path.replace(".ts", ""))
+const files = [
+  {
+    entry: join(entryPath, "index.ts"),
+    out: join(outfilePath, "index.js"),
+  },
+]
+for (let name of threadNames) {
+  files.push({
+    entry: join(entryPath, "threads", `${name}.ts`),
+    out: join(outfilePath, `${name}.js`),
+  })
+}
+
+for (let file of files) {
+  coreBuild(file.entry, file.out)
+}

package/src/api/controllers/datasource.ts

@@ -21,6 +21,7 @@ import {
   CreateDatasourceRequest,
   VerifyDatasourceRequest,
   VerifyDatasourceResponse,
+  FetchDatasourceInfoResponse,
   IntegrationBase,
   DatasourcePlus,
 } from "@budibase/types"
@@ -153,6 +154,21 @@ export async function verify(
   }
 }
 
+export async function information(
+  ctx: UserCtx<void, FetchDatasourceInfoResponse>
+) {
+  const datasourceId = ctx.params.datasourceId
+  const datasource = await sdk.datasources.get(datasourceId, { enriched: true })
+  const connector = (await getConnector(datasource)) as DatasourcePlus
+  if (!connector.getTableNames) {
+    ctx.throw(400, "Table name fetching not supported by datasource")
+  }
+  const tableNames = await connector.getTableNames()
+  ctx.body = {
+    tableNames,
+  }
+}
+
 export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)

package/src/api/controllers/static/index.ts

@@ -134,7 +134,7 @@ export const serveApp = async function (ctx: any) {
             ? objectStore.getGlobalFileUrl("settings", "logoUrl")
             : "",
       })
-      const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
+      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
       ctx.body = await processString(appHbs, {
         head,
         body: html,
@@ -161,7 +161,7 @@ export const serveApp = async function (ctx: any) {
             : "",
       })
 
-      const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
+      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
       ctx.body = await processString(appHbs, {
         head,
         body: html,
@@ -177,7 +177,7 @@ export const serveBuilderPreview = async function (ctx: any) {
 
   if (!env.isJest()) {
     let appId = context.getAppId()
-    const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
+    const previewHbs = loadHandlebarsFile(`${__dirname}/preview.hbs`)
     ctx.body = await processString(previewHbs, {
       clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
     })

package/src/api/controllers/table/internal.ts

@@ -186,11 +186,7 @@ export async function destroy(ctx: any) {
 export async function bulkImport(ctx: any) {
   const db = context.getAppDB()
   const table = await sdk.tables.getTable(ctx.params.tableId)
-  const { rows } = ctx.request.body
-  await handleDataImport(ctx.user, table, rows)
-
-  // Ensure auto id and other table updates are persisted
-  await db.put(table)
-
+  const { rows, identifierFields } = ctx.request.body
+  await handleDataImport(ctx.user, table, rows, identifierFields)
   return table
 }

package/src/api/controllers/table/utils.ts

@@ -149,7 +149,12 @@ export function importToRows(
   return finalData
 }
 
-export async function handleDataImport(user: any, table: any, rows: any) {
+export async function handleDataImport(
+  user: any,
+  table: any,
+  rows: any,
+  identifierFields: Array<string> = []
+) {
   const schema: unknown = table.schema
 
   if (!rows || !isRows(rows) || !isSchema(schema)) {
@@ -161,6 +166,32 @@ export async function handleDataImport(user: any, table: any, rows: any) {
 
   let finalData: any = importToRows(data, table, user)
 
+  //Set IDs of finalData to match existing row if an update is expected
+  if (identifierFields.length > 0) {
+    const allDocs = await db.allDocs(
+      getRowParams(table._id, null, {
+        include_docs: true,
+      })
+    )
+    allDocs.rows
+      .map(existingRow => existingRow.doc)
+      .forEach((doc: any) => {
+        finalData.forEach((finalItem: any) => {
+          let match = true
+          for (const field of identifierFields) {
+            if (finalItem[field] !== doc[field]) {
+              match = false
+              break
+            }
+          }
+          if (match) {
+            finalItem._id = doc._id
+            finalItem._rev = doc._rev
+          }
+        })
+      })
+  }
+
   await quotas.addRows(finalData.length, () => db.bulkDocs(finalData), {
     tableId: table._id,
   })

package/src/api/routes/datasource.ts

@@ -20,6 +20,11 @@ router
     authorized(permissions.BUILDER),
     datasourceController.verify
   )
+  .get(
+    "/api/datasources/:datasourceId/info",
+    authorized(permissions.BUILDER),
+    datasourceController.information
+  )
   .get(
     "/api/datasources/:datasourceId",
     authorized(

package/src/api/routes/static.ts

@@ -5,6 +5,7 @@ import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
 import env from "../../environment"
 import { paramResource } from "../../middleware/resourceId"
+import { devClientLibPath } from "../../utilities/fileSystem"
 const { BUILDER, PermissionType, PermissionLevel } = permissions
 
 const router: Router = new Router()
@@ -17,7 +18,8 @@ router.param("file", async (file: any, ctx: any, next: any) => {
   }
   // test serves from require
   if (env.isTest()) {
-    ctx.devPath = require.resolve("@budibase/client").split(ctx.file)[0]
+    const path = devClientLibPath()
+    ctx.devPath = path.split(ctx.file)[0]
   } else if (env.isDev()) {
     // Serving the client library from your local dir in dev
     ctx.devPath = budibaseTempDir()

package/src/api/routes/tests/datasource.spec.ts

@@ -87,7 +87,7 @@ describe("/datasources", () => {
         expect(contents.rows.length).toEqual(1)
 
         // update the datasource to remove the variables
-        datasource.config.dynamicVariables = []
+        datasource.config!.dynamicVariables = []
         const res = await request
           .put(`/api/datasources/${datasource._id}`)
           .send(datasource)

package/src/automations/actions.ts

@@ -71,10 +71,15 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
 // ran at all
 if (env.SELF_HOSTED) {
   const bash = require("./steps/bash")
+  const openai = require("./steps/openai")
+
   // @ts-ignore
   ACTION_IMPLS["EXECUTE_BASH"] = bash.run
   // @ts-ignore
   BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+
+  ACTION_IMPLS.OPENAI = openai.run
+  BUILTIN_ACTION_DEFINITIONS.OPENAI = openai.definition
 }
 
 export async function getActionDefinitions() {

package/src/automations/steps/openai.ts

@@ -0,0 +1,105 @@
+import { Configuration, OpenAIApi } from "openai"
+import {
+  AutomationActionStepId,
+  AutomationStepSchema,
+  AutomationStepInput,
+  AutomationStepType,
+  AutomationIOType,
+} from "@budibase/types"
+import * as automationUtils from "../automationUtils"
+import environment from "../../environment"
+
+enum Model {
+  GPT_35_TURBO = "gpt-3.5-turbo",
+  // will only work with api keys that have access to the GPT4 API
+  GPT_4 = "gpt-4",
+}
+
+export const definition: AutomationStepSchema = {
+  name: "OpenAI",
+  tagline: "Send prompts to ChatGPT",
+  icon: "Algorithm",
+  description: "Interact with the OpenAI ChatGPT API.",
+  type: AutomationStepType.ACTION,
+  internal: true,
+  stepId: AutomationActionStepId.OPENAI,
+  inputs: {
+    prompt: "",
+  },
+  schema: {
+    inputs: {
+      properties: {
+        prompt: {
+          type: AutomationIOType.STRING,
+          title: "Prompt",
+        },
+        model: {
+          type: AutomationIOType.STRING,
+          title: "Model",
+          enum: Object.values(Model),
+        },
+      },
+      required: ["prompt", "model"],
+    },
+    outputs: {
+      properties: {
+        success: {
+          type: AutomationIOType.BOOLEAN,
+          description: "Whether the action was successful",
+        },
+        response: {
+          type: AutomationIOType.STRING,
+          description: "What was output",
+        },
+      },
+      required: ["success", "response"],
+    },
+  },
+}
+
+export async function run({ inputs, context }: AutomationStepInput) {
+  if (!environment.OPENAI_API_KEY) {
+    return {
+      success: false,
+      response:
+        "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.",
+    }
+  }
+
+  if (inputs.prompt == null) {
+    return {
+      success: false,
+      response: "Budibase OpenAI Automation Failed: No prompt supplied",
+    }
+  }
+
+  try {
+    const configuration = new Configuration({
+      apiKey: environment.OPENAI_API_KEY,
+    })
+
+    const openai = new OpenAIApi(configuration)
+
+    const completion = await openai.createChatCompletion({
+      model: inputs.model,
+      messages: [
+        {
+          role: "user",
+          content: inputs.prompt,
+        },
+      ],
+    })
+
+    const response = completion?.data?.choices[0]?.message?.content
+
+    return {
+      response,
+      success: true,
+    }
+  } catch (err) {
+    return {
+      success: false,
+      response: automationUtils.getError(err),
+    }
+  }
+}

package/src/automations/tests/openai.spec.ts

@@ -0,0 +1,86 @@
+const setup = require("./utilities")
+import environment from "../../environment"
+import openai from "openai"
+
+jest.mock(
+  "openai",
+  jest.fn(() => ({
+    Configuration: jest.fn(),
+    OpenAIApi: jest.fn(() => ({
+      createChatCompletion: jest.fn(() => ({
+        data: {
+          choices: [
+            {
+              message: {
+                content: "This is a test",
+              },
+            },
+          ],
+        },
+      })),
+    })),
+  }))
+)
+
+const OPENAI_PROMPT = "What is the meaning of life?"
+
+describe("test the openai action", () => {
+  let config = setup.getConfig()
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  beforeEach(() => {
+    environment.OPENAI_API_KEY = "abc123"
+  })
+
+  afterAll(setup.afterAll)
+
+  it("should present the correct error message when the OPENAI_API_KEY variable isn't set", async () => {
+    delete environment.OPENAI_API_KEY
+
+    let res = await setup.runStep("OPENAI", {
+      prompt: OPENAI_PROMPT,
+    })
+    expect(res.response).toEqual(
+      "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable."
+    )
+    expect(res.success).toBeFalsy()
+  })
+
+  it("should be able to receive a response from ChatGPT given a prompt", async () => {
+    const res = await setup.runStep("OPENAI", {
+      prompt: OPENAI_PROMPT,
+    })
+    expect(res.response).toEqual("This is a test")
+    expect(res.success).toBeTruthy()
+  })
+
+  it("should present the correct error message when a prompt is not provided", async () => {
+    const res = await setup.runStep("OPENAI", {
+      prompt: null,
+    })
+    expect(res.response).toEqual(
+      "Budibase OpenAI Automation Failed: No prompt supplied"
+    )
+    expect(res.success).toBeFalsy()
+  })
+
+  it("should present the correct error message when an error is thrown from the createChatCompletion call", async () => {
+    openai.OpenAIApi.mockImplementation(() => ({
+      createChatCompletion: jest.fn(() => {
+        throw new Error("An error occurred while calling createChatCompletion")
+      }),
+    }))
+
+    const res = await setup.runStep("OPENAI", {
+      prompt: OPENAI_PROMPT,
+    })
+
+    expect(res.response).toEqual(
+      "Error: An error occurred while calling createChatCompletion"
+    )
+    expect(res.success).toBeFalsy()
+  })
+})

package/src/constants/index.ts

@@ -1,4 +1,5 @@
 import { objectStore, roles, constants } from "@budibase/backend-core"
+import { FieldType as FieldTypes } from "@budibase/types"
 export { FieldType as FieldTypes, RelationshipTypes } from "@budibase/types"
 
 export enum FilterTypes {
@@ -24,14 +25,14 @@ export const NoEmptyFilterStrings = [
 ]
 
 export const CanSwitchTypes = [
-  [exports.FieldTypes.JSON, exports.FieldTypes.ARRAY],
+  [FieldTypes.JSON, FieldTypes.ARRAY],
   [
-    exports.FieldTypes.STRING,
-    exports.FieldTypes.OPTIONS,
-    exports.FieldTypes.LONGFORM,
-    exports.FieldTypes.BARCODEQR,
+    FieldTypes.STRING,
+    FieldTypes.OPTIONS,
+    FieldTypes.LONGFORM,
+    FieldTypes.BARCODEQR,
   ],
-  [exports.FieldTypes.BOOLEAN, exports.FieldTypes.NUMBER],
+  [FieldTypes.BOOLEAN, FieldTypes.NUMBER],
 ]
 
 export const SwitchableTypes = CanSwitchTypes.reduce((prev, current) =>
@@ -77,9 +78,9 @@ export const USERS_TABLE_SCHEMA = {
   // TODO: ADMIN PANEL - when implemented this doesn't need to be carried out
   schema: {
     email: {
-      type: exports.FieldTypes.STRING,
+      type: FieldTypes.STRING,
       constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
         email: true,
         length: {
           maximum: "",
@@ -92,27 +93,27 @@ export const USERS_TABLE_SCHEMA = {
     firstName: {
       name: "firstName",
       fieldName: "firstName",
-      type: exports.FieldTypes.STRING,
+      type: FieldTypes.STRING,
       constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
         presence: false,
       },
     },
     lastName: {
       name: "lastName",
       fieldName: "lastName",
-      type: exports.FieldTypes.STRING,
+      type: FieldTypes.STRING,
       constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
         presence: false,
       },
     },
     roleId: {
       fieldName: "roleId",
       name: "roleId",
-      type: exports.FieldTypes.OPTIONS,
+      type: FieldTypes.OPTIONS,
       constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
         presence: false,
         inclusion: Object.values(roles.BUILTIN_ROLE_IDS),
       },
@@ -120,9 +121,9 @@ export const USERS_TABLE_SCHEMA = {
     status: {
       fieldName: "status",
       name: "status",
-      type: exports.FieldTypes.OPTIONS,
+      type: FieldTypes.OPTIONS,
       constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
         presence: false,
         inclusion: Object.values(constants.UserStatus),
       },

package/src/db/inMemoryView.ts

@@ -25,7 +25,9 @@ export async function runView(
       }))
     )
     let fn = (doc: Document, emit: any) => emit(doc._id)
-    eval("fn = " + view?.map?.replace("function (doc)", "function (doc, emit)"))
+    ;(0, eval)(
+      "fn = " + view?.map?.replace("function (doc)", "function (doc, emit)")
+    )
     const queryFns: any = {
       meta: view.meta,
       map: fn,

package/src/environment.ts

@@ -71,6 +71,7 @@ const environment = {
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
   PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
+  OPENAI_API_KEY: process.env.OPENAI_API_KEY,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,
@@ -96,6 +97,7 @@ const environment = {
   isInThread: () => {
     return process.env.FORKED_PROCESS
   },
+  TOP_LEVEL_PATH: process.env.TOP_LEVEL_PATH,
 }
 
 // threading can cause memory issues with node-ts in development