@budibase/server 1.2.57 → 1.2.58-alpha.2

package/dist/utilities/fileSystem/index.js

@@ -14,7 +14,7 @@ const { join } = require("path");
 const uuid = require("uuid/v4");
 const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/db");
 const { ObjectStoreBuckets } = require("../../constants");
-const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, } = require("./utilities");
+const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, deleteFiles, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
 const env = require("../../environment");
 const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, } = require("../../db/utils");
@@ -291,5 +291,6 @@ exports.cleanup = appIds => {
 exports.upload = upload;
 exports.retrieve = retrieve;
 exports.retrieveToTmp = retrieveToTmp;
+exports.deleteFiles = deleteFiles;
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH;
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH;

package/dist/utilities/redis.js

@@ -30,6 +30,7 @@ exports.shutdown = () => __awaiter(void 0, void 0, void 0, function* () {
         yield debounceClient.finish();
     if (flagClient)
         yield flagClient.finish();
+    console.log("Redis shutdown");
 });
 exports.doesUserHaveLock = (devAppId, user) => __awaiter(void 0, void 0, void 0, function* () {
     const value = yield devAppClient.get(devAppId);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.2.57",
+  "version": "1.2.58-alpha.2",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^1.2.57",
-    "@budibase/client": "^1.2.57",
-    "@budibase/pro": "1.2.56",
-    "@budibase/string-templates": "^1.2.57",
-    "@budibase/types": "^1.2.57",
+    "@budibase/backend-core": "1.2.58-alpha.2",
+    "@budibase/client": "1.2.58-alpha.2",
+    "@budibase/pro": "1.2.58-alpha.1",
+    "@budibase/string-templates": "1.2.58-alpha.2",
+    "@budibase/types": "1.2.58-alpha.2",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -107,7 +107,7 @@
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
     "jimp": "0.16.1",
-    "joi": "17.2.1",
+    "joi": "17.6.0",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
     "knex": "0.95.15",
@@ -196,5 +196,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "605b91baf8bed0f1ffbb2d3304d4525c866bdd77"
+  "gitHead": "675fdd4b88456e4a663f8e2efb148556e0a35aab"
 }

package/src/api/controllers/row/internalSearch.js

@@ -1,4 +1,5 @@
 const { SearchIndexes } = require("../../../db/utils")
+const { removeKeyNumbering } = require("./utils")
 const fetch = require("node-fetch")
 const { getCouchInfo } = require("@budibase/backend-core/db")
 const { getAppId } = require("@budibase/backend-core/context")
@@ -20,6 +21,8 @@ class QueryBuilder {
       notEmpty: {},
       oneOf: {},
       contains: {},
+      notContains: {},
+      containsAny: {},
       ...base,
     }
     this.limit = 50
@@ -125,6 +128,16 @@ class QueryBuilder {
     return this
   }
 
+  addNotContains(key, value) {
+    this.query.notContains[key] = value
+    return this
+  }
+
+  addContainsAny(key, value) {
+    this.query.containsAny[key] = value
+    return this
+  }
+
   /**
    * Preprocesses a value before going into a lucene search.
    * Transforms strings to lowercase and wraps strings and bools in quotes.
@@ -170,11 +183,29 @@ class QueryBuilder {
       return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
     }
 
-    const contains = (key, value) => {
-      if (!value && value !== 0) {
+    const contains = (key, value, mode = "AND") => {
+      if (Array.isArray(value) && value.length === 0) {
         return null
       }
-      return `${key}:${builder.preprocess(value, { escape: true })}`
+      if (!Array.isArray(value)) {
+        return `${key}:${value}`
+      }
+      let statement = `${builder.preprocess(value[0], { escape: true })}`
+      for (let i = 1; i < value.length; i++) {
+        statement += ` ${mode} ${builder.preprocess(value[i], {
+          escape: true,
+        })}`
+      }
+      return `${key}:(${statement})`
+    }
+
+    const notContains = (key, value) => {
+      const allPrefix = allOr === "" ? "*:* AND" : ""
+      return allPrefix + "NOT " + contains(key, value)
+    }
+
+    const containsAny = (key, value) => {
+      return contains(key, value, "OR")
     }
 
     const oneOf = (key, value) => {
@@ -197,6 +228,8 @@ class QueryBuilder {
 
     function build(structure, queryFn) {
       for (let [key, value] of Object.entries(structure)) {
+        // check for new format - remove numbering if needed
+        key = removeKeyNumbering(key)
         key = builder.preprocess(key.replace(/ /g, "_"), {
           escape: true,
         })
@@ -275,6 +308,12 @@ class QueryBuilder {
     if (this.query.contains) {
       build(this.query.contains, contains)
     }
+    if (this.query.notContains) {
+      build(this.query.notContains, notContains)
+    }
+    if (this.query.containsAny) {
+      build(this.query.containsAny, containsAny)
+    }
     // make sure table ID is always added as an AND
     if (tableId) {
       query = `(${query})`
@@ -310,6 +349,9 @@ class QueryBuilder {
   }
 }
 
+// exported for unit testing
+exports.QueryBuilder = QueryBuilder
+
 /**
  * Executes a lucene search query.
  * @param url The query URL

package/src/api/controllers/row/utils.js

@@ -3,8 +3,9 @@ const { cloneDeep } = require("lodash/fp")
 const { InternalTables } = require("../../../db/utils")
 const userController = require("../user")
 const { FieldTypes } = require("../../../constants")
-const { makeExternalQuery } = require("../../../integrations/base/utils")
 const { getAppDB } = require("@budibase/backend-core/context")
+const { makeExternalQuery } = require("../../../integrations/base/query")
+const { removeKeyNumbering } = require("../../../integrations/base/utils")
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value) {
@@ -16,6 +17,8 @@ validateJs.extend(validateJs.validators.datetime, {
   },
 })
 
+exports.removeKeyNumbering = removeKeyNumbering
+
 exports.getDatasourceAndQuery = async json => {
   const datasourceId = json.endpoint.datasourceId
   const db = getAppDB()

package/src/api/controllers/static/index.ts

@@ -12,7 +12,7 @@ const {
 } = require("../../../utilities/fileSystem")
 const env = require("../../../environment")
 const { clientLibraryPath } = require("../../../utilities")
-const { upload } = require("../../../utilities/fileSystem")
+const { upload, deleteFiles } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentType } = require("../../../db/utils")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
@@ -97,6 +97,10 @@ export const uploadFile = async function (ctx: any) {
   ctx.body = await Promise.all(uploads)
 }
 
+export const deleteObjects = async function (ctx: any) {
+  ctx.body = await deleteFiles(ObjectStoreBuckets.APPS, ctx.request.body.keys)
+}
+
 export const serveApp = async function (ctx: any) {
   const db = getAppDB({ skip_setup: true })
   const appInfo = await db.get(DocumentType.APP_METADATA)
@@ -124,6 +128,22 @@ export const serveApp = async function (ctx: any) {
   }
 }
 
+export const serveBuilderPreview = async function (ctx: any) {
+  const db = getAppDB({ skip_setup: true })
+  const appInfo = await db.get(DocumentType.APP_METADATA)
+
+  if (!env.isJest()) {
+    let appId = getAppId()
+    const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
+    ctx.body = await processString(previewHbs, {
+      clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
+    })
+  } else {
+    // just return the app info for jest to assert on
+    ctx.body = { ...appInfo, builderPreview: true }
+  }
+}
+
 export const serveClientLibrary = async function (ctx: any) {
   return send(ctx, "budibase-client.js", {
     root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),

package/src/api/controllers/static/templates/preview.hbs

@@ -0,0 +1,103 @@
+<html lang="en">
+<head>
+  <title>Budibase Builder Preview</title>
+  <link rel="stylesheet" href="https://rsms.me/inter/inter.css" />
+  <link rel="preconnect" href="https://fonts.gstatic.com" />
+  <link
+    href="https://fonts.googleapis.com/css2?family=Source+Sans+Pro:wght@400;600;700&display=swap"
+    rel="stylesheet"
+  />
+  <link
+    href="https://cdn.jsdelivr.net/npm/remixicon@2.5.0/fonts/remixicon.css"
+    rel="stylesheet"
+  />
+  <style>
+    html, body {
+      padding: 0;
+      margin: 0;
+    }
+    html {
+      height: 100%;
+      width: 100%;
+      overflow: hidden;
+      display: flex;
+      flex-direction: column;
+      justify-content: flex-start;
+      align-items: stretch;
+    }
+    body {
+      flex: 1 1 auto;
+      overflow: hidden;
+    }
+
+    *,
+    *:before,
+    *:after {
+      box-sizing: border-box;
+    }
+  </style>
+  <script src='{{ clientLibPath }}'></script>
+  <script>
+    function receiveMessage(event) {
+      if (!event.data) {
+        return
+      }
+
+      // Parse received message
+      // If parsing fails, just ignore and wait for the next message
+      let parsed
+      try {
+        parsed = JSON.parse(event.data)
+      } catch (error) {
+        console.error("Client received invalid JSON")
+        // Ignore
+      }
+      if (!parsed || !parsed.isBudibaseEvent) {
+        return
+      }
+
+      // Extract data from message
+      const {
+        selectedComponentId,
+        layout,
+        screen,
+        appId,
+        theme,
+        customTheme,
+        previewDevice,
+        navigation,
+        hiddenComponentIds
+      } = parsed
+
+      // Set some flags so the app knows we're in the builder
+      window["##BUDIBASE_IN_BUILDER##"] = true
+      window["##BUDIBASE_APP_ID##"] = appId
+      window["##BUDIBASE_PREVIEW_LAYOUT##"] = layout
+      window["##BUDIBASE_PREVIEW_SCREEN##"] = screen
+      window["##BUDIBASE_SELECTED_COMPONENT_ID##"] = selectedComponentId
+      window["##BUDIBASE_PREVIEW_ID##"] = Math.random()
+      window["##BUDIBASE_PREVIEW_THEME##"] = theme
+      window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"] = customTheme
+      window["##BUDIBASE_PREVIEW_DEVICE##"] = previewDevice
+      window["##BUDIBASE_PREVIEW_NAVIGATION##"] = navigation
+      window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"] = hiddenComponentIds
+
+      // Initialise app
+      try {
+        if (window.loadBudibase) {
+          window.loadBudibase()
+          document.documentElement.classList.add("loaded")
+        } else {
+          throw "The client library couldn't be loaded"
+        }
+      } catch (error) {
+        window.parent.postMessage({ type: "error", error })
+      }
+    }
+
+    window.addEventListener("message", receiveMessage)
+    window.parent.postMessage({ type: "ready" })
+  </script>
+</head>
+<body></body>
+</html>

package/src/api/controllers/table/external.js

@@ -14,7 +14,7 @@ const {
   FieldTypes,
   RelationshipTypes,
 } = require("../../../constants")
-const { makeExternalQuery } = require("../../../integrations/base/utils")
+const { makeExternalQuery } = require("../../../integrations/base/query")
 const { cloneDeep } = require("lodash/fp")
 const csvParser = require("../../../utilities/csvParser")
 const { handleRequest } = require("../row/external")

package/src/api/routes/static.ts

@@ -38,6 +38,11 @@ router
   // TODO: for now this builder endpoint is not authorized/secured, will need to be
   .get("/builder/:file*", controller.serveBuilder)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
+  .post(
+    "/api/attachments/delete",
+    authorized(BUILDER),
+    controller.deleteObjects
+  )
   .post("/api/beta/:feature", controller.toggleBetaUiFeature)
   .post(
     "/api/attachments/:tableId/upload",
@@ -45,6 +50,13 @@ router
     authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
     controller.uploadFile
   )
+  .post(
+    "/api/attachments/:tableId/delete",
+    paramResource("tableId"),
+    authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
+    controller.deleteObjects
+  )
+  .get("/preview", authorized(BUILDER), controller.serveBuilderPreview)
   .get("/:appId/:path*", controller.serveApp)
   .get("/app/:appUrl/:path*", controller.serveApp)
   .post(

package/src/api/routes/tests/internalSearch.spec.js

@@ -0,0 +1,176 @@
+const search = require("../../controllers/row/internalSearch")
+// this will be mocked out for _search endpoint
+const fetch = require("node-fetch")
+const PARAMS = {
+  tableId: "ta_12345679abcdef",
+  version: "1",
+  bookmark: null,
+  sort: null,
+  sortOrder: "ascending",
+  sortType: "string",
+}
+
+function checkLucene(resp, expected, params = PARAMS) {
+  const query = resp.rows[0].query
+  const json = JSON.parse(query)
+  if (PARAMS.sort) {
+    expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
+  }
+  if (PARAMS.bookmark) {
+    expect(json.bookmark).toBe(PARAMS.bookmark)
+  }
+  expect(json.include_docs).toBe(true)
+  expect(json.q).toBe(`(${expected}) AND tableId:"${params.tableId}"`)
+  expect(json.limit).toBe(params.limit || 50)
+}
+
+describe("internal search", () => {
+  it("default query", async () => {
+    const response = await search.paginatedSearch({
+    }, PARAMS)
+    checkLucene(response, `*:*`)
+  })
+
+  it("test equal query", async () => {
+    const response = await search.paginatedSearch({
+      equal: {
+        "column": "1",
+      }
+    }, PARAMS)
+    checkLucene(response, `*:* AND column:"1"`)
+  })
+
+  it("test notEqual query", async () => {
+    const response = await search.paginatedSearch({
+      notEqual: {
+        "column": "1",
+      }
+    }, PARAMS)
+    checkLucene(response, `*:* AND !column:"1"`)
+  })
+
+  it("test OR query", async () => {
+    const response = await search.paginatedSearch({
+      allOr: true,
+      equal: {
+        "column": "2",
+      },
+      notEqual: {
+        "column": "1",
+      }
+    }, PARAMS)
+    checkLucene(response, `column:"2" OR !column:"1"`)
+  })
+
+  it("test AND query", async () => {
+    const response = await search.paginatedSearch({
+      equal: {
+        "column": "2",
+      },
+      notEqual: {
+        "column": "1",
+      }
+    }, PARAMS)
+    checkLucene(response, `*:* AND column:"2" AND !column:"1"`)
+  })
+
+  it("test pagination query", async () => {
+    const updatedParams = {
+      ...PARAMS,
+      limit: 100,
+      bookmark: "awd",
+      sort: "column",
+    }
+    const response = await search.paginatedSearch({
+      string: {
+        "column": "2",
+      },
+    }, updatedParams)
+    checkLucene(response, `*:* AND column:2*`, updatedParams)
+  })
+
+  it("test range query", async () => {
+    const response = await search.paginatedSearch({
+      range: {
+        "column": { low: 1, high: 2 },
+      },
+    }, PARAMS)
+    checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
+  })
+
+  it("test empty query", async () => {
+    const response = await search.paginatedSearch({
+      empty: {
+        "column": "",
+      },
+    }, PARAMS)
+    checkLucene(response, `*:* AND !column:["" TO *]`, PARAMS)
+  })
+
+  it("test notEmpty query", async () => {
+    const response = await search.paginatedSearch({
+      notEmpty: {
+        "column": "",
+      },
+    }, PARAMS)
+    checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
+  })
+
+  it("test oneOf query", async () => {
+    const response = await search.paginatedSearch({
+      oneOf: {
+        "column": ["a", "b"],
+      },
+    }, PARAMS)
+    checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
+  })
+
+  it("test contains query", async () => {
+    const response = await search.paginatedSearch({
+      contains: {
+        "column": "a",
+        "colArr": [1, 2, 3],
+      },
+    }, PARAMS)
+    checkLucene(response, `*:* AND column:a AND colArr:(1 AND 2 AND 3)`, PARAMS)
+  })
+
+  it("test multiple of same column", async () => {
+    const response = await search.paginatedSearch({
+      allOr: true,
+      equal: {
+        "1:column": "a",
+        "2:column": "b",
+        "3:column": "c",
+      },
+    }, PARAMS)
+    checkLucene(response, `column:"a" OR column:"b" OR column:"c"`, PARAMS)
+  })
+
+  it("check a weird case for lucene building", async () => {
+    const response = await search.paginatedSearch({
+      equal: {
+        "1:1:column": "a",
+      },
+    }, PARAMS)
+    checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
+  })
+
+  it("test containsAny query", async () => {
+    const response = await search.paginatedSearch({
+      containsAny: {
+        "column": ["a", "b", "c"]
+      },
+    }, PARAMS)
+    checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
+  })
+
+  it("test notContains query", async () => {
+    const response = await search.paginatedSearch({
+      notContains: {
+        "column": ["a", "b", "c"]
+      },
+    }, PARAMS)
+    checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
+  })
+})

package/src/api/routes/tests/static.spec.js

@@ -40,7 +40,6 @@ describe("/static", () => {
   })
 
   describe("/app", () => {
-
     beforeEach(() => {
       jest.clearAllMocks()
     })
@@ -60,7 +59,7 @@ describe("/static", () => {
     it("should serve the app by url", async () => {
       const headers = config.defaultHeaders()
       delete headers[constants.Headers.APP_ID]
-      
+
       const res = await request
         .get(`/app${config.prodApp.url}`)
         .set(headers)
@@ -82,7 +81,7 @@ describe("/static", () => {
   describe("/attachments", () => {
     describe("generateSignedUrls", () => {
       let datasource
-  
+
       beforeEach(async () => {
         datasource = await config.createDatasource({
           datasource: {
@@ -93,7 +92,7 @@ describe("/static", () => {
           },
         })
       })
-  
+
       it("should be able to generate a signed upload URL", async () => {
         const bucket = "foo"
         const key = "bar"
@@ -108,7 +107,7 @@ describe("/static", () => {
           `https://${bucket}.s3.eu-west-1.amazonaws.com/${key}`
         )
       })
-  
+
       it("should handle an invalid datasource ID", async () => {
         const res = await request
           .post(`/api/attachments/foo/url`)
@@ -123,7 +122,7 @@ describe("/static", () => {
           "The specified datasource could not be found"
         )
       })
-  
+
       it("should require a bucket parameter", async () => {
         const res = await request
           .post(`/api/attachments/${datasource._id}/url`)
@@ -136,7 +135,7 @@ describe("/static", () => {
           .expect(400)
         expect(res.body.message).toEqual("bucket and key values are required")
       })
-  
+
       it("should require a key parameter", async () => {
         const res = await request
           .post(`/api/attachments/${datasource._id}/url`)
@@ -151,4 +150,17 @@ describe("/static", () => {
     })
   })
 
+  describe("/preview", () => {
+    beforeEach(() => {
+      jest.clearAllMocks()
+    })
+
+    it("should serve the builder preview", async () => {
+      const headers = config.defaultHeaders()
+      const res = await request.get(`/preview`).set(headers).expect(200)
+
+      expect(res.body.appId).toBe(config.appId)
+      expect(res.body.builderPreview).toBe(true)
+    })
+  })
 })

package/src/app.ts

@@ -83,9 +83,7 @@ server.on("close", async () => {
     return
   }
   shuttingDown = true
-  if (!env.isTest()) {
-    console.log("Server Closed")
-  }
+  console.log("Server Closed")
   await automations.shutdown()
   await redis.shutdown()
   await events.shutdown()
@@ -167,3 +165,7 @@ process.on("uncaughtException", err => {
 process.on("SIGTERM", () => {
   shutdown()
 })
+
+process.on("SIGINT", () => {
+  shutdown()
+})

package/src/automations/bullboard.js

@@ -53,6 +53,7 @@ exports.shutdown = async () => {
     await automationQueue.close()
     automationQueue = null
   }
+  console.log("Bull shutdown")
 }
 
 exports.queue = automationQueue

package/src/automations/logging/index.ts

@@ -23,6 +23,9 @@ export async function checkAppMetadata(apps: App[]) {
     for (let [key, errors] of Object.entries(metadata.automationErrors)) {
       const updated = []
       for (let error of errors) {
+        if (!error) {
+          continue
+        }
         const startDate = error.split(dbUtils.SEPARATOR)[2]
         if (startDate > maxStartDate) {
           updated.push(error)

package/src/integrations/base/query.ts

@@ -0,0 +1,16 @@
+import { QueryJson, Datasource } from "@budibase/types"
+const { integrations } = require("../index")
+
+export async function makeExternalQuery(
+  datasource: Datasource,
+  json: QueryJson
+) {
+  const Integration = integrations[datasource.source]
+  // query is the opinionated function
+  if (Integration.prototype.query) {
+    const integration = new Integration(datasource.config)
+    return integration.query(json)
+  } else {
+    throw "Datasource does not support query."
+  }
+}