@budibase/server 2.3.17 → 2.3.18-alpha.1

package/dist/utilities/redis.js

@@ -36,6 +36,8 @@ function shutdown() {
             yield debounceClient.finish();
         if (flagClient)
             yield flagClient.finish();
+        // shutdown core clients
+        yield backend_core_1.redis.clients.shutdown();
         console.log("Redis shutdown");
     });
 }

package/dist/utilities/retry.js

@@ -0,0 +1,30 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.retry = void 0;
+function retry(fn, maxTry = 5, retryCount = 1) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const currRetry = typeof retryCount === "number" ? retryCount : 1;
+        try {
+            const result = yield fn();
+            return result;
+        }
+        catch (e) {
+            console.log(`Retry ${currRetry} failed.`);
+            if (currRetry > maxTry) {
+                console.log(`All ${maxTry} retry attempts exhausted`);
+                throw e;
+            }
+            return retry(fn, maxTry, currRetry + 1);
+        }
+    });
+}
+exports.retry = retry;

package/jest-testcontainers-config.js

@@ -0,0 +1,8 @@
+const { join } = require("path")
+require("dotenv").config({
+  path: join(__dirname, "..", "..", "hosting", ".env"),
+})
+
+const jestTestcontainersConfigGenerator = require("../../jestTestcontainersConfigGenerator")
+
+module.exports = jestTestcontainersConfigGenerator()

package/jest.config.ts

@@ -1,36 +1,48 @@
 import { Config } from "@jest/types"
+
 import * as fs from "fs"
+const preset = require("ts-jest/jest-preset")
 
-const config: Config.InitialOptions = {
-  testEnvironment: "node",
+const baseConfig: Config.InitialProjectOptions = {
+  ...preset,
+  preset: "@trendyol/jest-testcontainers",
   setupFiles: ["./src/tests/jestEnv.ts"],
   setupFilesAfterEnv: ["./src/tests/jestSetup.ts"],
-  collectCoverageFrom: [
-    "src/**/*.{js,ts}",
-    // The use of coverage with couchdb view functions breaks tests
-    "!src/db/views/staticViews.*",
-  ],
-  coverageReporters: ["lcov", "json", "clover"],
   transform: {
     "^.+\\.ts?$": "@swc/jest",
   },
-}
-
-if (!process.env.CI) {
-  // use sources when not in CI
-  config.moduleNameMapper = {
+  moduleNameMapper: {
     "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
     "@budibase/backend-core": "<rootDir>/../backend-core/src",
     "@budibase/types": "<rootDir>/../types/src",
-    "^axios.*$": "<rootDir>/node_modules/axios/lib/axios.js",
-  }
-  // add pro sources if they exist
-  if (fs.existsSync("../../../budibase-pro")) {
-    config.moduleNameMapper["@budibase/pro"] =
-      "<rootDir>/../../../budibase-pro/packages/pro/src"
-  }
-} else {
-  console.log("Running tests with compiled dependency sources")
+  },
+}
+
+// add pro sources if they exist
+if (fs.existsSync("../../../budibase-pro")) {
+  baseConfig.moduleNameMapper["@budibase/pro"] =
+    "<rootDir>/../../../budibase-pro/packages/pro/src"
+}
+
+const config: Config.InitialOptions = {
+  projects: [
+    {
+      ...baseConfig,
+      displayName: "sequential test",
+      testMatch: ["<rootDir>/**/*.seq.spec.[jt]s"],
+      runner: "jest-serial-runner",
+    },
+    {
+      ...baseConfig,
+      testMatch: ["<rootDir>/**/!(*.seq).spec.[jt]s"],
+    },
+  ],
+  collectCoverageFrom: [
+    "src/**/*.{js,ts}",
+    // The use of coverage with couchdb view functions breaks tests
+    "!src/db/views/staticViews.*",
+  ],
+  coverageReporters: ["lcov", "json", "clover"],
 }
 
 export default config

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.3.17",
+  "version": "2.3.18-alpha.1",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -14,7 +14,7 @@
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
     "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
-    "test": "jest --coverage --maxWorkers=2",
+    "test": "jest --coverage --runInBand",
     "test:watch": "jest --watch",
     "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^2.3.17",
-    "@budibase/client": "^2.3.17",
-    "@budibase/pro": "2.3.16",
-    "@budibase/string-templates": "^2.3.17",
-    "@budibase/types": "^2.3.17",
+    "@budibase/backend-core": "2.3.18-alpha.1",
+    "@budibase/client": "2.3.18-alpha.1",
+    "@budibase/pro": "2.3.18-alpha.0",
+    "@budibase/string-templates": "2.3.18-alpha.1",
+    "@budibase/types": "2.3.18-alpha.1",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -64,6 +64,7 @@
     "chokidar": "3.5.3",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
+    "dd-trace": "3.13.2",
     "dotenv": "8.2.0",
     "download": "8.0.0",
     "elastic-apm-node": "3.38.0",
@@ -125,6 +126,7 @@
     "@jest/test-sequencer": "24.9.0",
     "@swc/core": "^1.3.25",
     "@swc/jest": "^0.2.24",
+    "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/apidoc": "0.50.0",
     "@types/bson": "4.2.0",
     "@types/global-agent": "2.1.1",
@@ -140,6 +142,7 @@
     "@types/pouchdb": "6.4.0",
     "@types/redis": "4.0.11",
     "@types/server-destroy": "1.0.1",
+    "@types/supertest": "2.0.12",
     "@types/tar": "6.1.3",
     "@typescript-eslint/parser": "5.45.0",
     "apidoc": "0.50.4",
@@ -151,13 +154,14 @@
     "is-wsl": "2.2.0",
     "jest": "28.1.1",
     "jest-openapi": "0.14.2",
+    "jest-serial-runner": "^1.2.1",
     "nodemon": "2.0.15",
     "openapi-types": "9.3.1",
     "openapi-typescript": "5.2.0",
     "path-to-regexp": "6.2.0",
     "prettier": "2.5.1",
     "rimraf": "3.0.2",
-    "supertest": "4.0.2",
+    "supertest": "6.2.2",
     "swagger-jsdoc": "6.1.0",
     "timekeeper": "2.2.0",
     "ts-jest": "28.0.4",
@@ -169,5 +173,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "a57c922b91c1df8dc276588a5ad583a0c1aea52c"
+  "gitHead": "865cab09b28f2abcdc0b485437465b167adebdf4"
 }

package/scripts/load/create-many-apps.js

@@ -0,0 +1,24 @@
+#!/bin/node
+const { createApp } = require("./utils")
+
+const APP_COUNT = 100
+
+if (!process.argv[2]) {
+  console.error("Please specify an API key as script argument.")
+  process.exit(-1)
+}
+
+async function run() {
+  for (let i = 0; i < APP_COUNT; i++) {
+    const app = await createApp(process.argv[2])
+    console.log(`App created: ${app._id}`)
+  }
+}
+
+run()
+  .then(() => {
+    console.log(`Finished creating ${APP_COUNT} apps.`)
+  })
+  .catch(err => {
+    console.error(err)
+  })

package/scripts/load/create-many-rows.js

@@ -0,0 +1,30 @@
+#!/bin/node
+const { createApp, getTable, createRow } = require("./utils")
+
+const ROW_COUNT = 1000
+
+if (!process.argv[2]) {
+  console.error("Please specify an API key as script argument.")
+  process.exit(-1)
+}
+
+async function run() {
+  const apiKey = process.argv[2]
+  const app = await createApp(apiKey)
+  console.log(`App created: ${app._id}`)
+  const table = await getTable(apiKey, app._id)
+  console.log(`Table found: ${table.name}`)
+  const promises = []
+  for (let i = 0; i < ROW_COUNT; i++) {
+    promises.push(await createRow(apiKey, app._id, table))
+  }
+  await Promise.all(promises)
+}
+
+run()
+  .then(() => {
+    console.log(`Finished creating ${ROW_COUNT} rows.`)
+  })
+  .catch(err => {
+    console.error(err)
+  })

package/scripts/load/utils.js

@@ -0,0 +1,66 @@
+const fetch = require("node-fetch")
+const uuid = require("uuid/v4")
+
+const URL_APP = "http://localhost:10000/api/public/v1/applications"
+const URL_TABLE = "http://localhost:10000/api/public/v1/tables/search"
+
+async function request(apiKey, url, method, body, appId = undefined) {
+  const headers = {
+    "x-budibase-api-key": apiKey,
+    "Content-Type": "application/json",
+  }
+  if (appId) {
+    headers["x-budibase-app-id"] = appId
+  }
+  const res = await fetch(url, {
+    method,
+    headers,
+    body: JSON.stringify(body),
+  })
+  if (res.status !== 200) {
+    throw new Error(await res.text())
+  }
+  return res
+}
+
+exports.createApp = async apiKey => {
+  const name = uuid().replace(/-/g, "")
+  const body = {
+    name,
+    url: `/${name}`,
+    useTemplate: "true",
+    templateKey: "app/school-admin-panel",
+    templateName: "School Admin Panel",
+  }
+  const res = await request(apiKey, URL_APP, "POST", body)
+  const json = await res.json()
+  return json.data
+}
+
+exports.getTable = async (apiKey, appId) => {
+  const res = await request(apiKey, URL_TABLE, "POST", {}, appId)
+  const json = await res.json()
+  return json.data[0]
+}
+
+exports.createRow = async (apiKey, appId, table) => {
+  const body = {}
+  for (let [key, schema] of Object.entries(table.schema)) {
+    let fake
+    switch (schema.type) {
+      default:
+      case "string":
+        fake = schema.constraints.inclusion
+          ? schema.constraints.inclusion[0]
+          : "a"
+        break
+      case "number":
+        fake = 1
+        break
+    }
+    body[key] = fake
+  }
+  const url = `http://localhost:10000/api/public/v1/tables/${table._id}/rows`
+  const res = await request(apiKey, url, "POST", body, appId)
+  return (await res.json()).data
+}

package/specs/openapi.json

@@ -817,7 +817,6 @@
                       "type": "string",
                       "enum": [
                         "string",
-                        "barcodeqr",
                         "longform",
                         "options",
                         "number",
@@ -829,7 +828,8 @@
                         "formula",
                         "auto",
                         "json",
-                        "internal"
+                        "internal",
+                        "barcodeqr"
                       ],
                       "description": "Defines the type of the column, most explain themselves, a link column is a relationship."
                     },
@@ -1021,7 +1021,6 @@
                           "type": "string",
                           "enum": [
                             "string",
-                            "barcodeqr",
                             "longform",
                             "options",
                             "number",
@@ -1033,7 +1032,8 @@
                             "formula",
                             "auto",
                             "json",
-                            "internal"
+                            "internal",
+                            "barcodeqr"
                           ],
                           "description": "Defines the type of the column, most explain themselves, a link column is a relationship."
                         },
@@ -1236,7 +1236,6 @@
                             "type": "string",
                             "enum": [
                               "string",
-                              "barcodeqr",
                               "longform",
                               "options",
                               "number",
@@ -1248,7 +1247,8 @@
                               "formula",
                               "auto",
                               "json",
-                              "internal"
+                              "internal",
+                              "barcodeqr"
                             ],
                             "description": "Defines the type of the column, most explain themselves, a link column is a relationship."
                           },

package/specs/openapi.yaml

@@ -603,7 +603,6 @@ components:
                     type: string
                     enum:
                       - string
-                      - barcodeqr
                       - longform
                       - options
                       - number
@@ -616,6 +615,7 @@ components:
                       - auto
                       - json
                       - internal
+                      - barcodeqr
                     description: Defines the type of the column, most explain themselves, a link
                       column is a relationship.
                   constraints:
@@ -766,7 +766,6 @@ components:
                         type: string
                         enum:
                           - string
-                          - barcodeqr
                           - longform
                           - options
                           - number
@@ -779,6 +778,7 @@ components:
                           - auto
                           - json
                           - internal
+                          - barcodeqr
                         description: Defines the type of the column, most explain themselves, a link
                           column is a relationship.
                       constraints:
@@ -936,7 +936,6 @@ components:
                           type: string
                           enum:
                             - string
-                            - barcodeqr
                             - longform
                             - options
                             - number
@@ -949,6 +948,7 @@ components:
                             - auto
                             - json
                             - internal
+                            - barcodeqr
                           description: Defines the type of the column, most explain themselves, a link
                             column is a relationship.
                         constraints:

package/specs/resources/query.js

@@ -1,6 +1,6 @@
 const Resource = require("./utils/Resource")
 const { object } = require("./utils")
-const { BaseQueryVerbs } = require("../../dist/constants")
+const { BaseQueryVerbs } = require("../../src/constants")
 
 const query = {
   _id: "query_datasource_plus_4d8be0c506b9465daf4bf84d890fdab6_454854487c574d45bc4029b1e153219e",

package/specs/resources/table.js

@@ -2,7 +2,7 @@ const {
   FieldTypes,
   RelationshipTypes,
   FormulaTypes,
-} = require("../../dist/constants")
+} = require("../../src/constants")
 const { object } = require("./utils")
 const Resource = require("./utils/Resource")
 

package/src/api/controllers/query/import/tests/index.spec.js

@@ -41,7 +41,7 @@ const datasets = {
 describe("Rest Importer", () => {
   const config = new TestConfig(false)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
   })
 

package/src/api/controllers/query/index.ts

@@ -186,6 +186,8 @@ export async function preview(ctx: any) {
         schemaFields[key] = fieldType
       }
     }
+    // remove configuration before sending event
+    delete datasource.config
     await events.query.previewed(datasource, query)
     ctx.body = {
       rows,

package/src/api/controllers/row/ExternalRequest.ts

@@ -10,6 +10,7 @@ import {
   FieldSchema,
   Row,
   Table,
+  RelationshipTypes,
 } from "@budibase/types"
 import {
   breakRowIdField,
@@ -18,7 +19,7 @@ import {
   convertRowId,
 } from "../../../integrations/utils"
 import { getDatasourceAndQuery } from "./utils"
-import { FieldTypes, RelationshipTypes } from "../../../constants"
+import { FieldTypes } from "../../../constants"
 import { breakExternalTableId, isSQL } from "../../../integrations/utils"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
@@ -44,6 +45,7 @@ export interface RunConfig {
   row?: Row
   rows?: Row[]
   tables?: Record<string, Table>
+  includeSqlRelationships?: IncludeRelationship
 }
 
 function buildFilters(
@@ -706,7 +708,9 @@ export class ExternalRequest {
       },
       resource: {
         // have to specify the fields to avoid column overlap (for SQL)
-        fields: isSql ? this.buildFields(table) : [],
+        fields: isSql
+          ? this.buildFields(table, config.includeSqlRelationships)
+          : [],
       },
       filters,
       sort,

package/src/api/controllers/row/external.ts

@@ -18,6 +18,7 @@ import {
   PaginationJson,
   Table,
   Datasource,
+  IncludeRelationship,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 
@@ -57,6 +58,7 @@ export async function patch(ctx: BBContext) {
   return handleRequest(Operation.UPDATE, tableId, {
     id: breakRowIdField(id),
     row: inputs,
+    includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })
 }
 
@@ -65,6 +67,7 @@ export async function save(ctx: BBContext) {
   const tableId = ctx.params.tableId
   return handleRequest(Operation.CREATE, tableId, {
     row: inputs,
+    includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })
 }
 
@@ -78,7 +81,9 @@ export async function fetchView(ctx: BBContext) {
 
 export async function fetch(ctx: BBContext) {
   const tableId = ctx.params.tableId
-  return handleRequest(Operation.READ, tableId)
+  return handleRequest(Operation.READ, tableId, {
+    includeSqlRelationships: IncludeRelationship.INCLUDE,
+  })
 }
 
 export async function find(ctx: BBContext) {
@@ -86,6 +91,7 @@ export async function find(ctx: BBContext) {
   const tableId = ctx.params.tableId
   const response = (await handleRequest(Operation.READ, tableId, {
     id: breakRowIdField(id),
+    includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })) as Row[]
   return response ? response[0] : response
 }
@@ -95,6 +101,7 @@ export async function destroy(ctx: BBContext) {
   const id = ctx.request.body._id
   const { row } = (await handleRequest(Operation.DELETE, tableId, {
     id: breakRowIdField(id),
+    includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })) as { row: Row }
   return { response: { ok: true }, row }
 }
@@ -107,6 +114,7 @@ export async function bulkDestroy(ctx: BBContext) {
     promises.push(
       handleRequest(Operation.DELETE, tableId, {
         id: breakRowIdField(row._id),
+        includeSqlRelationships: IncludeRelationship.EXCLUDE,
       })
     )
   }
@@ -149,6 +157,7 @@ export async function search(ctx: BBContext) {
       filters: query,
       sort,
       paginate: paginateObj as PaginationJson,
+      includeSqlRelationships: IncludeRelationship.INCLUDE,
     })) as Row[]
     let hasNextPage = false
     if (paginate && rows.length === limit) {
@@ -159,6 +168,7 @@ export async function search(ctx: BBContext) {
           limit: 1,
           page: bookmark * limit + 1,
         },
+        includeSqlRelationships: IncludeRelationship.INCLUDE,
       })) as Row[]
       hasNextPage = nextRows.length > 0
     }
@@ -249,6 +259,7 @@ export async function fetchEnrichedRow(ctx: BBContext) {
   const response = (await handleRequest(Operation.READ, tableId, {
     id,
     datasource,
+    includeSqlRelationships: IncludeRelationship.INCLUDE,
   })) as Row[]
   const table: Table = tables[tableName]
   const row = response[0]
@@ -276,6 +287,7 @@ export async function fetchEnrichedRow(ctx: BBContext) {
           [primaryLink]: linkedIds,
         },
       },
+      includeSqlRelationships: IncludeRelationship.INCLUDE,
     })
   }
   return row

package/src/api/controllers/row/staticFormula.ts

@@ -38,7 +38,13 @@ export async function updateRelatedFormula(
         if (!relatedRows[relatedTableId]) {
           relatedRows[relatedTableId] = []
         }
-        relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field)
+        // filter down to the rows which are not already included in related
+        const currentIds = relatedRows[relatedTableId].map(row => row._id)
+        const uniqueRelatedRows = field.filter(
+          (row: Row) => !currentIds.includes(row._id)
+        )
+        relatedRows[relatedTableId] =
+          relatedRows[relatedTableId].concat(uniqueRelatedRows)
       }
     }
     for (let tableId of table.relatedFormula) {

package/src/api/controllers/table/external.ts

@@ -8,7 +8,7 @@ import {
   foreignKeyStructure,
   hasTypeChanged,
 } from "./utils"
-import { FieldTypes, RelationshipTypes } from "../../../constants"
+import { FieldTypes } from "../../../constants"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { handleRequest } from "../row/external"
 import { events, context } from "@budibase/backend-core"
@@ -22,6 +22,7 @@ import {
   FieldSchema,
   BBContext,
   TableRequest,
+  RelationshipTypes,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 const { cloneDeep } = require("lodash/fp")
@@ -146,7 +147,7 @@ function generateLinkSchema(
   column: FieldSchema,
   table: Table,
   relatedTable: Table,
-  type: string
+  type: RelationshipTypes
 ) {
   if (!table.primary || !relatedTable.primary) {
     throw new Error("Unable to generate link schema, no primary keys")

package/src/api/index.ts

@@ -1,5 +1,5 @@
 import Router from "@koa/router"
-import { errors, auth } from "@budibase/backend-core"
+import { auth, middleware } from "@budibase/backend-core"
 import currentApp from "../middleware/currentapp"
 import zlib from "zlib"
 import { mainRoutes, staticRoutes, publicRoutes } from "./routes"
@@ -14,6 +14,8 @@ export const router: Router = new Router()
 router.get("/health", ctx => (ctx.status = 200))
 router.get("/version", ctx => (ctx.body = pkg.version))
 
+router.use(middleware.errorHandling)
+
 router
   .use(
     compress({
@@ -54,27 +56,6 @@ router
   .use(currentApp)
   .use(auth.auditLog)
 
-// error handling middleware
-router.use(async (ctx, next) => {
-  try {
-    await next()
-  } catch (err: any) {
-    ctx.status = err.status || err.statusCode || 500
-    const error = errors.getPublicError(err)
-    ctx.body = {
-      message: err.message,
-      status: ctx.status,
-      validationErrors: err.validation,
-      error,
-    }
-    ctx.log.error(err)
-    // unauthorised errors don't provide a useful trace
-    if (!env.isTest()) {
-      console.trace(err)
-    }
-  }
-})
-
 // authenticated routes
 for (let route of mainRoutes) {
   router.use(route.routes())