@budibase/server 2.3.19 → 2.3.21-alpha.0

package/dist/utilities/redis.js

@@ -36,6 +36,8 @@ function shutdown() {
             yield debounceClient.finish();
         if (flagClient)
             yield flagClient.finish();
+        // shutdown core clients
+        yield backend_core_1.redis.clients.shutdown();
         console.log("Redis shutdown");
     });
 }

package/dist/utilities/retry.js

@@ -0,0 +1,30 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.retry = void 0;
+function retry(fn, maxTry = 5, retryCount = 1) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const currRetry = typeof retryCount === "number" ? retryCount : 1;
+        try {
+            const result = yield fn();
+            return result;
+        }
+        catch (e) {
+            console.log(`Retry ${currRetry} failed.`);
+            if (currRetry > maxTry) {
+                console.log(`All ${maxTry} retry attempts exhausted`);
+                throw e;
+            }
+            return retry(fn, maxTry, currRetry + 1);
+        }
+    });
+}
+exports.retry = retry;

package/jest-testcontainers-config.js

@@ -0,0 +1,8 @@
+const { join } = require("path")
+require("dotenv").config({
+  path: join(__dirname, "..", "..", "hosting", ".env"),
+})
+
+const jestTestcontainersConfigGenerator = require("../../jestTestcontainersConfigGenerator")
+
+module.exports = jestTestcontainersConfigGenerator()

package/jest.config.ts

@@ -1,36 +1,49 @@
 import { Config } from "@jest/types"
+
 import * as fs from "fs"
+const preset = require("ts-jest/jest-preset")
 
-const config: Config.InitialOptions = {
-  testEnvironment: "node",
+const baseConfig: Config.InitialProjectOptions = {
+  ...preset,
+  preset: "@trendyol/jest-testcontainers",
   setupFiles: ["./src/tests/jestEnv.ts"],
   setupFilesAfterEnv: ["./src/tests/jestSetup.ts"],
-  collectCoverageFrom: [
-    "src/**/*.{js,ts}",
-    // The use of coverage with couchdb view functions breaks tests
-    "!src/db/views/staticViews.*",
-  ],
-  coverageReporters: ["lcov", "json", "clover"],
   transform: {
     "^.+\\.ts?$": "@swc/jest",
   },
-}
-
-if (!process.env.CI) {
-  // use sources when not in CI
-  config.moduleNameMapper = {
+  moduleNameMapper: {
     "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
     "@budibase/backend-core": "<rootDir>/../backend-core/src",
     "@budibase/types": "<rootDir>/../types/src",
-    "^axios.*$": "<rootDir>/node_modules/axios/lib/axios.js",
-  }
-  // add pro sources if they exist
-  if (fs.existsSync("../../../budibase-pro")) {
-    config.moduleNameMapper["@budibase/pro"] =
-      "<rootDir>/../../../budibase-pro/packages/pro/src"
-  }
-} else {
-  console.log("Running tests with compiled dependency sources")
+  },
+}
+
+// add pro sources if they exist
+if (fs.existsSync("../../../budibase-pro")) {
+  baseConfig.moduleNameMapper["@budibase/pro"] =
+    "<rootDir>/../../../budibase-pro/packages/pro/src"
+}
+
+const config: Config.InitialOptions = {
+  projects: [
+    {
+      ...baseConfig,
+      displayName: "sequential test",
+      testMatch: ["<rootDir>/**/*.seq.spec.[jt]s"],
+      runner: "jest-serial-runner",
+    },
+    {
+      ...baseConfig,
+      testMatch: ["<rootDir>/**/!(*.seq).spec.[jt]s"],
+    },
+  ],
+  collectCoverageFrom: [
+    "src/**/*.{js,ts}",
+    "../backend-core/src/**/*.{js,ts}",
+    // The use of coverage with couchdb view functions breaks tests
+    "!src/db/views/staticViews.*",
+  ],
+  coverageReporters: ["lcov", "json", "clover"],
 }
 
 export default config

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.3.19",
+  "version": "2.3.21-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -14,7 +14,7 @@
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
     "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
-    "test": "jest --coverage --maxWorkers=2",
+    "test": "bash scripts/test.sh",
     "test:watch": "jest --watch",
     "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
@@ -25,7 +25,7 @@
     "dev:stack:down": "node scripts/dev/manage.js down",
     "dev:stack:nuke": "node scripts/dev/manage.js nuke",
     "dev:builder": "yarn run dev:stack:up && nodemon",
-    "specs": "node specs/generate.js && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
+    "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
     "initialise": "node scripts/initialise.js",
     "env:multi:enable": "node scripts/multiTenancy.js enable",
     "env:multi:disable": "node scripts/multiTenancy.js disable",
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^2.3.19",
-    "@budibase/client": "^2.3.19",
-    "@budibase/pro": "2.3.18",
-    "@budibase/string-templates": "^2.3.19",
-    "@budibase/types": "^2.3.19",
+    "@budibase/backend-core": "2.3.21-alpha.0",
+    "@budibase/client": "2.3.21-alpha.0",
+    "@budibase/pro": "2.3.20",
+    "@budibase/string-templates": "2.3.21-alpha.0",
+    "@budibase/types": "2.3.21-alpha.0",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -64,6 +64,7 @@
     "chokidar": "3.5.3",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
+    "dd-trace": "3.13.2",
     "dotenv": "8.2.0",
     "download": "8.0.0",
     "elastic-apm-node": "3.38.0",
@@ -86,6 +87,7 @@
     "koa-send": "5.0.0",
     "koa-session": "5.12.0",
     "koa-static": "5.0.0",
+    "koa-useragent": "^4.1.0",
     "koa2-ratelimit": "1.1.1",
     "lodash": "4.17.21",
     "memorystream": "0.3.1",
@@ -125,6 +127,7 @@
     "@jest/test-sequencer": "24.9.0",
     "@swc/core": "^1.3.25",
     "@swc/jest": "^0.2.24",
+    "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/apidoc": "0.50.0",
     "@types/bson": "4.2.0",
     "@types/global-agent": "2.1.1",
@@ -140,6 +143,7 @@
     "@types/pouchdb": "6.4.0",
     "@types/redis": "4.0.11",
     "@types/server-destroy": "1.0.1",
+    "@types/supertest": "2.0.12",
     "@types/tar": "6.1.3",
     "@typescript-eslint/parser": "5.45.0",
     "apidoc": "0.50.4",
@@ -151,13 +155,14 @@
     "is-wsl": "2.2.0",
     "jest": "28.1.1",
     "jest-openapi": "0.14.2",
+    "jest-serial-runner": "^1.2.1",
     "nodemon": "2.0.15",
     "openapi-types": "9.3.1",
     "openapi-typescript": "5.2.0",
     "path-to-regexp": "6.2.0",
     "prettier": "2.5.1",
     "rimraf": "3.0.2",
-    "supertest": "4.0.2",
+    "supertest": "6.2.2",
     "swagger-jsdoc": "6.1.0",
     "timekeeper": "2.2.0",
     "ts-jest": "28.0.4",
@@ -169,5 +174,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "dfb412df1e5ae11e6d0772a83d4add7c4ee7d1d2"
+  "gitHead": "e07c36dd8ca5d1fcfcf18961457dc539016535ca"
 }

package/scripts/load/create-many-apps.js

@@ -0,0 +1,24 @@
+#!/bin/node
+const { createApp } = require("./utils")
+
+const APP_COUNT = 100
+
+if (!process.argv[2]) {
+  console.error("Please specify an API key as script argument.")
+  process.exit(-1)
+}
+
+async function run() {
+  for (let i = 0; i < APP_COUNT; i++) {
+    const app = await createApp(process.argv[2])
+    console.log(`App created: ${app._id}`)
+  }
+}
+
+run()
+  .then(() => {
+    console.log(`Finished creating ${APP_COUNT} apps.`)
+  })
+  .catch(err => {
+    console.error(err)
+  })

package/scripts/load/create-many-rows.js

@@ -0,0 +1,30 @@
+#!/bin/node
+const { createApp, getTable, createRow } = require("./utils")
+
+const ROW_COUNT = 1000
+
+if (!process.argv[2]) {
+  console.error("Please specify an API key as script argument.")
+  process.exit(-1)
+}
+
+async function run() {
+  const apiKey = process.argv[2]
+  const app = await createApp(apiKey)
+  console.log(`App created: ${app._id}`)
+  const table = await getTable(apiKey, app._id)
+  console.log(`Table found: ${table.name}`)
+  const promises = []
+  for (let i = 0; i < ROW_COUNT; i++) {
+    promises.push(await createRow(apiKey, app._id, table))
+  }
+  await Promise.all(promises)
+}
+
+run()
+  .then(() => {
+    console.log(`Finished creating ${ROW_COUNT} rows.`)
+  })
+  .catch(err => {
+    console.error(err)
+  })

package/scripts/load/utils.js

@@ -0,0 +1,66 @@
+const fetch = require("node-fetch")
+const uuid = require("uuid/v4")
+
+const URL_APP = "http://localhost:10000/api/public/v1/applications"
+const URL_TABLE = "http://localhost:10000/api/public/v1/tables/search"
+
+async function request(apiKey, url, method, body, appId = undefined) {
+  const headers = {
+    "x-budibase-api-key": apiKey,
+    "Content-Type": "application/json",
+  }
+  if (appId) {
+    headers["x-budibase-app-id"] = appId
+  }
+  const res = await fetch(url, {
+    method,
+    headers,
+    body: JSON.stringify(body),
+  })
+  if (res.status !== 200) {
+    throw new Error(await res.text())
+  }
+  return res
+}
+
+exports.createApp = async apiKey => {
+  const name = uuid().replace(/-/g, "")
+  const body = {
+    name,
+    url: `/${name}`,
+    useTemplate: "true",
+    templateKey: "app/school-admin-panel",
+    templateName: "School Admin Panel",
+  }
+  const res = await request(apiKey, URL_APP, "POST", body)
+  const json = await res.json()
+  return json.data
+}
+
+exports.getTable = async (apiKey, appId) => {
+  const res = await request(apiKey, URL_TABLE, "POST", {}, appId)
+  const json = await res.json()
+  return json.data[0]
+}
+
+exports.createRow = async (apiKey, appId, table) => {
+  const body = {}
+  for (let [key, schema] of Object.entries(table.schema)) {
+    let fake
+    switch (schema.type) {
+      default:
+      case "string":
+        fake = schema.constraints.inclusion
+          ? schema.constraints.inclusion[0]
+          : "a"
+        break
+      case "number":
+        fake = 1
+        break
+    }
+    body[key] = fake
+  }
+  const url = `http://localhost:10000/api/public/v1/tables/${table._id}/rows`
+  const res = await request(apiKey, url, "POST", body, appId)
+  return (await res.json()).data
+}

package/scripts/test.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ -n $CI ]]
+then
+  # --runInBand performs better in ci where resources are limited
+  echo "jest --coverage --runInBand"
+  jest --coverage --runInBand
+else
+  # --maxWorkers performs better in development
+  echo "jest --coverage --maxWorkers=2"
+  jest --coverage --maxWorkers=2
+fi

package/specs/{generate.js → generate.ts}

@@ -1,9 +1,9 @@
+import { join } from "path"
+import { writeFileSync } from "fs"
+import { examples, schemas } from "./resources"
+import * as parameters from "./parameters"
+import * as security from "./security"
 const swaggerJsdoc = require("swagger-jsdoc")
-const { join } = require("path")
-const { writeFileSync } = require("fs")
-const { examples, schemas } = require("./resources")
-const parameters = require("./parameters")
-const security = require("./security")
 
 const VARIABLES = {}
 
@@ -60,7 +60,7 @@ const options = {
   apis: [join(__dirname, "..", "src", "api", "routes", "public", "*.ts")],
 }
 
-function writeFile(output, filename) {
+function writeFile(output: any, filename: string) {
   try {
     const path = join(__dirname, filename)
     let spec = output
@@ -79,7 +79,7 @@ function writeFile(output, filename) {
   }
 }
 
-function run() {
+export function run() {
   const outputJSON = swaggerJsdoc(options)
   options.format = ".yaml"
   const outputYAML = swaggerJsdoc(options)
@@ -90,5 +90,3 @@ function run() {
 if (require.main === module) {
   run()
 }
-
-module.exports = run

package/specs/openapi.json

@@ -817,7 +817,6 @@
                       "type": "string",
                       "enum": [
                         "string",
-                        "barcodeqr",
                         "longform",
                         "options",
                         "number",
@@ -829,7 +828,8 @@
                         "formula",
                         "auto",
                         "json",
-                        "internal"
+                        "internal",
+                        "barcodeqr"
                       ],
                       "description": "Defines the type of the column, most explain themselves, a link column is a relationship."
                     },
@@ -1021,7 +1021,6 @@
                           "type": "string",
                           "enum": [
                             "string",
-                            "barcodeqr",
                             "longform",
                             "options",
                             "number",
@@ -1033,7 +1032,8 @@
                             "formula",
                             "auto",
                             "json",
-                            "internal"
+                            "internal",
+                            "barcodeqr"
                           ],
                           "description": "Defines the type of the column, most explain themselves, a link column is a relationship."
                         },
@@ -1236,7 +1236,6 @@
                             "type": "string",
                             "enum": [
                               "string",
-                              "barcodeqr",
                               "longform",
                               "options",
                               "number",
@@ -1248,7 +1247,8 @@
                               "formula",
                               "auto",
                               "json",
-                              "internal"
+                              "internal",
+                              "barcodeqr"
                             ],
                             "description": "Defines the type of the column, most explain themselves, a link column is a relationship."
                           },
@@ -1829,7 +1829,7 @@
   "paths": {
     "/applications": {
       "post": {
-        "operationId": "create",
+        "operationId": "appCreate",
         "summary": "Create an application",
         "tags": [
           "applications"
@@ -1870,7 +1870,7 @@
     },
     "/applications/{appId}": {
       "put": {
-        "operationId": "update",
+        "operationId": "appUpdate",
         "summary": "Update an application",
         "tags": [
           "applications"
@@ -1909,7 +1909,7 @@
         }
       },
       "delete": {
-        "operationId": "destroy",
+        "operationId": "appDestroy",
         "summary": "Delete an application",
         "tags": [
           "applications"
@@ -1938,7 +1938,7 @@
         }
       },
       "get": {
-        "operationId": "getById",
+        "operationId": "appGetById",
         "summary": "Retrieve an application",
         "tags": [
           "applications"
@@ -1969,7 +1969,7 @@
     },
     "/applications/{appId}/unpublish": {
       "post": {
-        "operationId": "unpublish",
+        "operationId": "appUnpublish",
         "summary": "Unpublish an application",
         "tags": [
           "applications"
@@ -1988,7 +1988,7 @@
     },
     "/applications/{appId}/publish": {
       "post": {
-        "operationId": "publish",
+        "operationId": "appPublish",
         "summary": "Unpublish an application",
         "tags": [
           "applications"
@@ -2019,7 +2019,7 @@
     },
     "/applications/search": {
       "post": {
-        "operationId": "search",
+        "operationId": "appSearch",
         "summary": "Search for applications",
         "description": "Based on application properties (currently only name) search for applications.",
         "tags": [
@@ -2056,7 +2056,7 @@
     },
     "/queries/{queryId}": {
       "post": {
-        "operationId": "execute",
+        "operationId": "queryExecute",
         "summary": "Execute a query",
         "description": "Queries which have been created within a Budibase app can be executed using this,",
         "tags": [
@@ -2104,7 +2104,7 @@
     },
     "/queries/search": {
       "post": {
-        "operationId": "search",
+        "operationId": "querySearch",
         "summary": "Search for queries",
         "description": "Based on query properties (currently only name) search for queries.",
         "tags": [
@@ -2146,7 +2146,7 @@
     },
     "/tables/{tableId}/rows": {
       "post": {
-        "operationId": "create",
+        "operationId": "rowCreate",
         "summary": "Create a row",
         "description": "Creates a row within the specified table.",
         "tags": [
@@ -2196,7 +2196,7 @@
     },
     "/tables/{tableId}/rows/{rowId}": {
       "put": {
-        "operationId": "update",
+        "operationId": "rowUpdate",
         "summary": "Update a row",
         "description": "Updates a row within the specified table.",
         "tags": [
@@ -2247,7 +2247,7 @@
         }
       },
       "delete": {
-        "operationId": "destroy",
+        "operationId": "rowDestroy",
         "summary": "Delete a row",
         "description": "Deletes a row within the specified table.",
         "tags": [
@@ -2283,7 +2283,7 @@
         }
       },
       "get": {
-        "operationId": "getById",
+        "operationId": "rowGetById",
         "summary": "Retrieve a row",
         "description": "This gets a single row, it will be enriched with the full related rows, rather than the squashed \"primaryDisplay\" format returned by the search endpoint.",
         "tags": [
@@ -2321,7 +2321,7 @@
     },
     "/tables/{tableId}/rows/search": {
       "post": {
-        "operationId": "search",
+        "operationId": "rowSearch",
         "summary": "Search for rows",
         "tags": [
           "rows"
@@ -2365,7 +2365,7 @@
     },
     "/tables": {
       "post": {
-        "operationId": "create",
+        "operationId": "tableCreate",
         "summary": "Create a table",
         "description": "Create a table, this could be internal or external.",
         "tags": [
@@ -2411,7 +2411,7 @@
     },
     "/tables/{tableId}": {
       "put": {
-        "operationId": "update",
+        "operationId": "tableUpdate",
         "summary": "Update a table",
         "description": "Update a table, this could be internal or external.",
         "tags": [
@@ -2458,7 +2458,7 @@
         }
       },
       "delete": {
-        "operationId": "destroy",
+        "operationId": "tableDestroy",
         "summary": "Delete a table",
         "description": "Delete a table, this could be internal or external.",
         "tags": [
@@ -2491,7 +2491,7 @@
         }
       },
       "get": {
-        "operationId": "getById",
+        "operationId": "tableGetById",
         "summary": "Retrieve a table",
         "description": "Lookup a table, this could be internal or external.",
         "tags": [
@@ -2526,7 +2526,7 @@
     },
     "/tables/search": {
       "post": {
-        "operationId": "search",
+        "operationId": "tableSearch",
         "summary": "Search for tables",
         "description": "Based on table properties (currently only name) search for tables. This could be an internal or an external table.",
         "tags": [
@@ -2568,7 +2568,7 @@
     },
     "/users": {
       "post": {
-        "operationId": "create",
+        "operationId": "userCreate",
         "summary": "Create a user",
         "tags": [
           "users"
@@ -2604,7 +2604,7 @@
     },
     "/users/{userId}": {
       "put": {
-        "operationId": "update",
+        "operationId": "userUpdate",
         "summary": "Update a user",
         "tags": [
           "users"
@@ -2643,7 +2643,7 @@
         }
       },
       "delete": {
-        "operationId": "destroy",
+        "operationId": "userDestroy",
         "summary": "Delete a user",
         "tags": [
           "users"
@@ -2672,7 +2672,7 @@
         }
       },
       "get": {
-        "operationId": "getById",
+        "operationId": "userGetById",
         "summary": "Retrieve a user",
         "tags": [
           "users"
@@ -2703,7 +2703,7 @@
     },
     "/users/search": {
       "post": {
-        "operationId": "search",
+        "operationId": "userSearch",
         "summary": "Search for users",
         "description": "Based on user properties (currently only name) search for users.",
         "tags": [