@budibase/server 1.3.13 → 1.3.14

package/dist/utilities/fileSystem/index.js

@@ -110,11 +110,11 @@ exports.defineFilter = excludeRows => {
  * data or user relationships.
  * @param {string} appId The app to backup
  * @param {object} config Config to send to export DB
- * @param {boolean} includeRows Flag to state whether the export should include data.
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
  * @returns {*} either a string or a stream of the backup
  */
-const backupAppData = (appId, config, includeRows) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: exports.defineFilter(includeRows) }));
+const backupAppData = (appId, config, excludeRows) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: exports.defineFilter(excludeRows) }));
 });
 /**
  * Takes a copy of the database state for an app to the object store.
@@ -128,11 +128,11 @@ exports.performBackup = (appId, backupName) => __awaiter(void 0, void 0, void 0,
 /**
  * Streams a backup of the database state for an app
  * @param {string} appId The ID of the app which is to be backed up.
- * @param {boolean} includeRows Flag to state whether the export should include data.
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
  * @returns {*} a readable stream of the backup which is written in real time
  */
-exports.streamBackup = (appId, includeRows) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield backupAppData(appId, { stream: true }, includeRows);
+exports.streamBackup = (appId, excludeRows) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield backupAppData(appId, { stream: true }, excludeRows);
 });
 /**
  * Exports a DB to either file or a variable (memory).

package/dist/utilities/index.js

@@ -150,3 +150,11 @@ exports.convertBookmark = bookmark => {
     }
     return bookmark;
 };
+exports.isQsTrue = param => {
+    if (typeof param === "string") {
+        return param.toLowerCase() === "true";
+    }
+    else {
+        return param === true;
+    }
+};

package/dist/utilities/rowProcessor/utils.js

@@ -64,7 +64,7 @@ exports.processDates = (table, rows) => {
         if (schema.type !== FieldTypes.DATETIME) {
             continue;
         }
-        if (!schema.ignoreTimezones) {
+        if (!schema.timeOnly && !schema.ignoreTimezones) {
             datesWithTZ.push(column);
         }
     }

package/dist/utilities/usageQuota/rows.js

@@ -72,7 +72,8 @@ exports.getUniqueRows = (appIds) => __awaiter(void 0, void 0, void 0, function*
         // ensure uniqueness on a per app pair basis
         // this can't be done on all rows because app import results in
         // duplicate row ids across apps
-        uniqueRows = uniqueRows.concat(...new Set(appRows));
+        // the array pre-concat is important to avoid stack overflow
+        uniqueRows = uniqueRows.concat([...new Set(appRows)]);
     }
     return uniqueRows;
 });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.3.13",
+  "version": "1.3.14",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^1.3.13",
-    "@budibase/client": "^1.3.13",
-    "@budibase/pro": "1.3.12",
-    "@budibase/string-templates": "^1.3.13",
-    "@budibase/types": "^1.3.13",
+    "@budibase/backend-core": "^1.3.14",
+    "@budibase/client": "^1.3.14",
+    "@budibase/pro": "1.3.13",
+    "@budibase/string-templates": "^1.3.14",
+    "@budibase/types": "^1.3.14",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -196,5 +196,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "272a2ec7c7f2fabf74153fc8573ce6745134cdde"
+  "gitHead": "a38626cf662d66aa8c04fb1d42dc7bc353ce7c31"
 }

package/src/api/controllers/application.ts

@@ -553,11 +553,7 @@ export const sync = async (ctx: any, next: any) => {
   })
   let error
   try {
-    await replication.replicate({
-      filter: function (doc: any) {
-        return doc._id !== DocumentType.APP_METADATA
-      },
-    })
+    await replication.replicate(replication.appReplicateOpts())
   } catch (err) {
     error = err
   } finally {

package/src/api/controllers/backup.js

@@ -1,11 +1,12 @@
 const { streamBackup } = require("../../utilities/fileSystem")
 const { events, context } = require("@budibase/backend-core")
 const { DocumentType } = require("../../db/utils")
+const { isQsTrue } = require("../../utilities")
 
 exports.exportAppDump = async function (ctx) {
   let { appId, excludeRows } = ctx.query
   const appName = decodeURI(ctx.query.appname)
-  excludeRows = excludeRows === "true"
+  excludeRows = isQsTrue(excludeRows)
   const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
   ctx.attachment(backupIdentifier)
   ctx.body = await streamBackup(appId, excludeRows)

package/src/api/controllers/deploy/index.ts

@@ -15,6 +15,7 @@ import {
   getAppId,
   getAppDB,
   getProdAppDB,
+  getDevAppDB,
 } from "@budibase/backend-core/context"
 import { quotas } from "@budibase/pro"
 import { events } from "@budibase/backend-core"
@@ -110,17 +111,29 @@ async function deployApp(deployment: any) {
       target: productionAppId,
     }
     replication = new Replication(config)
-
+    const devDb = getDevAppDB()
+    console.log("Compacting development DB")
+    await devDb.compact()
     console.log("Replication object created")
-    await replication.replicate()
+    await replication.replicate(replication.appReplicateOpts())
     console.log("replication complete.. replacing app meta doc")
+    // app metadata is excluded as it is likely to be in conflict
+    // replicate the app metadata document manually
     const db = getProdAppDB()
-    const appDoc = await db.get(DocumentType.APP_METADATA)
+    const appDoc = await devDb.get(DocumentType.APP_METADATA)
+    try {
+      const prodAppDoc = await db.get(DocumentType.APP_METADATA)
+      appDoc._rev = prodAppDoc._rev
+    } catch (err) {
+      delete appDoc._rev
+    }
 
+    // switch to production app ID
     deployment.appUrl = appDoc.url
-
     appDoc.appId = productionAppId
     appDoc.instance._id = productionAppId
+    // remove automation errors if they exist
+    delete appDoc.automationErrors
     await db.put(appDoc)
     await appCache.invalidateAppMetadata(productionAppId)
     console.log("New app doc written successfully.")

package/src/api/controllers/row/ExternalRequest.ts

@@ -534,7 +534,7 @@ module External {
         })
         // this is the response from knex if no rows found
         const rows = !response[0].read ? response : []
-        const storeTo = isMany ? field.throughFrom || linkPrimaryKey : manyKey
+        const storeTo = isMany ? field.throughFrom || linkPrimaryKey : fieldName
         related[storeTo] = { rows, isMany, tableId }
       }
       return related

package/src/api/routes/tests/query.spec.js

@@ -311,7 +311,7 @@ describe("/queries", () => {
         "url": "string",
         "value": "string"
       })
-      expect(res.body.rows[0].url).toContain("doctype html")
+      expect(res.body.rows[0].url).toContain("doctype%20html")
     })
 
     it("check that it automatically retries on fail with cached dynamics", async () => {
@@ -396,7 +396,7 @@ describe("/queries", () => {
         "queryHdr": userDetails.firstName,
         "secondHdr" : "1234"
       })
-      expect(res.body.rows[0].url).toEqual("http://www.google.com?email=" + userDetails.email)
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?email=" + userDetails.email.replace("@", "%40"))
     })
 
     it("should bind the current user to query parameters", async () => {
@@ -413,7 +413,7 @@ describe("/queries", () => {
         "testParam" : "1234"
       })
   
-      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=" + userDetails.email + 
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=" + userDetails.email.replace("@", "%40") +
         "&testName=" + userDetails.firstName + "&testParam=1234")
     })
 

package/src/automations/index.js

@@ -1,16 +1,19 @@
 const { processEvent } = require("./utils")
 const { queue, shutdown } = require("./bullboard")
-const { TRIGGER_DEFINITIONS } = require("./triggers")
+const { TRIGGER_DEFINITIONS, rebootTrigger } = require("./triggers")
 const { ACTION_DEFINITIONS } = require("./actions")
 
 /**
  * This module is built purely to kick off the worker farm and manage the inputs/outputs
  */
-exports.init = function () {
+exports.init = async function () {
   // this promise will not complete
-  return queue.process(async job => {
+  const promise = queue.process(async job => {
     await processEvent(job)
   })
+  // on init we need to trigger any reboot automations
+  await rebootTrigger()
+  return promise
 }
 
 exports.getQueues = () => {

package/src/automations/triggers.js

@@ -9,6 +9,7 @@ const { checkTestFlag } = require("../utilities/redis")
 const utils = require("./utils")
 const env = require("../environment")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
+const { getAllApps } = require("@budibase/backend-core/db")
 
 const TRIGGER_DEFINITIONS = definitions
 const JOB_OPTS = {
@@ -16,24 +17,27 @@ const JOB_OPTS = {
   removeOnFail: true,
 }
 
+async function getAllAutomations() {
+  const db = getAppDB()
+  let automations = await db.allDocs(
+    getAutomationParams(null, { include_docs: true })
+  )
+  return automations.rows.map(row => row.doc)
+}
+
 async function queueRelevantRowAutomations(event, eventType) {
   if (event.appId == null) {
     throw `No appId specified for ${eventType} - check event emitters.`
   }
 
   doInAppContext(event.appId, async () => {
-    const db = getAppDB()
-    let automations = await db.allDocs(
-      getAutomationParams(null, { include_docs: true })
-    )
+    let automations = await getAllAutomations()
 
     // filter down to the correct event type
-    automations = automations.rows
-      .map(automation => automation.doc)
-      .filter(automation => {
-        const trigger = automation.definition.trigger
-        return trigger && trigger.event === eventType
-      })
+    automations = automations.filter(automation => {
+      const trigger = automation.definition.trigger
+      return trigger && trigger.event === eventType
+    })
 
     for (let automation of automations) {
       let automationDef = automation.definition
@@ -110,4 +114,34 @@ exports.externalTrigger = async function (
   }
 }
 
+exports.rebootTrigger = async () => {
+  // reboot cron option is only available on the main thread at
+  // startup and only usable in self host
+  if (env.isInThread() || !env.SELF_HOSTED) {
+    return
+  }
+  // iterate through all production apps, find the reboot crons
+  // and trigger events for them
+  const appIds = await getAllApps({ dev: false, idsOnly: true })
+  for (let prodAppId of appIds) {
+    await doInAppContext(prodAppId, async () => {
+      let automations = await getAllAutomations()
+      let rebootEvents = []
+      for (let automation of automations) {
+        if (utils.isRebootTrigger(automation)) {
+          const job = {
+            automation,
+            event: {
+              appId: prodAppId,
+              timestamp: Date.now(),
+            },
+          }
+          rebootEvents.push(queue.add(job, JOB_OPTS))
+        }
+      }
+      await Promise.all(rebootEvents)
+    })
+  }
+}
+
 exports.TRIGGER_DEFINITIONS = TRIGGER_DEFINITIONS

package/src/automations/utils.ts

@@ -17,6 +17,7 @@ import { tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import { Automation } from "@budibase/types"
 
+const REBOOT_CRON = "@reboot"
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
 const Runner = new Thread(ThreadType.AUTOMATION)
@@ -109,22 +110,33 @@ export async function clearMetadata() {
   await db.bulkDocs(automationMetadata)
 }
 
+export function isCronTrigger(auto: Automation) {
+  return (
+    auto &&
+    auto.definition.trigger &&
+    auto.definition.trigger.stepId === CRON_STEP_ID
+  )
+}
+
+export function isRebootTrigger(auto: Automation) {
+  const trigger = auto ? auto.definition.trigger : null
+  return isCronTrigger(auto) && trigger?.inputs.cron === REBOOT_CRON
+}
+
 /**
  * This function handles checking of any cron jobs that need to be enabled/updated.
  * @param {string} appId The ID of the app in which we are checking for webhooks
  * @param {object|undefined} automation The automation object to be updated.
  */
-export async function enableCronTrigger(appId: any, automation: any) {
+export async function enableCronTrigger(appId: any, automation: Automation) {
   const trigger = automation ? automation.definition.trigger : null
-  function isCronTrigger(auto: any) {
-    return (
-      auto &&
-      auto.definition.trigger &&
-      auto.definition.trigger.stepId === CRON_STEP_ID
-    )
-  }
+
   // need to create cron job
-  if (isCronTrigger(automation) && trigger?.inputs.cron) {
+  if (
+    isCronTrigger(automation) &&
+    !isRebootTrigger(automation) &&
+    trigger?.inputs.cron
+  ) {
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
     const jobId = `${appId}_cron_${newid()}`
     const job: any = await queue.add(

package/src/definitions/datasource.ts

@@ -34,6 +34,7 @@ export interface RestConfig {
   defaultHeaders: {
     [key: string]: any
   }
+  legacyHttpParser: boolean
   authConfigs: AuthConfig[]
   staticVariables: {
     [key: string]: string

package/src/integrations/rest.ts

@@ -14,6 +14,7 @@ import {
   BearerAuthConfig,
 } from "../definitions/datasource"
 import { get } from "lodash"
+import qs from "querystring"
 
 const BodyTypes = {
   NONE: "none",
@@ -79,6 +80,12 @@ module RestModule {
         required: false,
         default: {},
       },
+      legacyHttpParser: {
+        display: "Legacy HTTP Support",
+        type: DatasourceFieldType.BOOLEAN,
+        required: false,
+        default: false,
+      },
     },
     query: {
       create: {
@@ -215,7 +222,8 @@ module RestModule {
         }
       }
 
-      const main = `${path}?${queryString}`
+      // make sure the query string is fully encoded
+      const main = `${path}?${qs.encode(qs.decode(queryString))}`
       let complete = main
       if (this.config.url && !main.startsWith("http")) {
         complete = !this.config.url ? main : `${this.config.url}/${main}`
@@ -377,6 +385,11 @@ module RestModule {
         paginationValues
       )
 
+      if (this.config.legacyHttpParser) {
+        // https://github.com/nodejs/node/issues/43798
+        input.extraHttpOptions = { insecureHTTPParser: true }
+      }
+
       this.startTimeMs = performance.now()
       const url = this.getUrl(path, queryString, pagination, paginationValues)
       const response = await fetch(url, input)

package/src/integrations/tests/rest.spec.js

@@ -50,7 +50,7 @@ describe("REST Integration", () => {
         name: "test",
       }),
     }
-    const response = await config.integration.create(query)
+    await config.integration.create(query)
     expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, {
       method: "POST",
       body: '{"name":"test"}',
@@ -295,7 +295,7 @@ describe("REST Integration", () => {
       }
       await config.integration.read(query)
       expect(fetch).toHaveBeenCalledWith(
-        `${BASE_URL}/api?${pageParam}=${pageValue}&${sizeParam}=${sizeValue}&`,
+        `${BASE_URL}/api?${pageParam}=${pageValue}&${sizeParam}=${sizeValue}`,
         {
           headers: {},
           method: "GET",
@@ -420,7 +420,7 @@ describe("REST Integration", () => {
       }
       const res = await config.integration.read(query)
       expect(fetch).toHaveBeenCalledWith(
-        `${BASE_URL}/api?${pageParam}=${pageValue}&${sizeParam}=${sizeValue}&`,
+        `${BASE_URL}/api?${pageParam}=${pageValue}&${sizeParam}=${sizeValue}`,
         {
           headers: {},
           method: "GET",
@@ -528,5 +528,40 @@ describe("REST Integration", () => {
       expect(sentData.get(sizeParam)).toEqual(sizeValue.toString())
       expect(res.pagination.cursor).toEqual(123)
     })
+
+    it("should encode query string correctly", async () => {
+      const query = {
+        path: "api",
+        queryString: "test=1 2",
+        headers: HEADERS,
+        bodyType: "json",
+        requestBody: JSON.stringify({
+          name: "test",
+        }),
+      }
+      await config.integration.create(query)
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1%202`, {
+        method: "POST",
+        body: '{"name":"test"}',
+        headers: HEADERS,
+      })
+    })
+  })
+
+  describe("Configuration options", () => {
+    it("Attaches insecureHttpParams when legacy HTTP Parser option is set", async () => {
+      config = new TestConfiguration({
+        url: BASE_URL,
+        legacyHttpParser: true
+      })
+      await config.integration.read({})
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/?`, {
+        method: "GET",
+        headers: {},
+        extraHttpOptions: {
+          insecureHTTPParser: true
+        }
+      })
+    })
   })
 })

package/src/threads/automation.ts

@@ -458,6 +458,9 @@ class Orchestrator {
 
 export function execute(input: AutomationEvent, callback: WorkerCallback) {
   const appId = input.data.event.appId
+  if (!appId) {
+    throw new Error("Unable to execute, event doesn't contain app ID.")
+  }
   doInAppContext(appId, async () => {
     const automationOrchestrator = new Orchestrator(
       input.data.automation,
@@ -475,6 +478,9 @@ export function execute(input: AutomationEvent, callback: WorkerCallback) {
 
 export const removeStalled = async (input: AutomationEvent) => {
   const appId = input.data.event.appId
+  if (!appId) {
+    throw new Error("Unable to execute, event doesn't contain app ID.")
+  }
   await doInAppContext(appId, async () => {
     const automationOrchestrator = new Orchestrator(
       input.data.automation,

package/src/utilities/fileSystem/index.js

@@ -125,13 +125,13 @@ exports.defineFilter = excludeRows => {
  * data or user relationships.
  * @param {string} appId The app to backup
  * @param {object} config Config to send to export DB
- * @param {boolean} includeRows Flag to state whether the export should include data.
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
  * @returns {*} either a string or a stream of the backup
  */
-const backupAppData = async (appId, config, includeRows) => {
+const backupAppData = async (appId, config, excludeRows) => {
   return await exports.exportDB(appId, {
     ...config,
-    filter: exports.defineFilter(includeRows),
+    filter: exports.defineFilter(excludeRows),
   })
 }
 
@@ -148,11 +148,11 @@ exports.performBackup = async (appId, backupName) => {
 /**
  * Streams a backup of the database state for an app
  * @param {string} appId The ID of the app which is to be backed up.
- * @param {boolean} includeRows Flag to state whether the export should include data.
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
  * @returns {*} a readable stream of the backup which is written in real time
  */
-exports.streamBackup = async (appId, includeRows) => {
-  return await backupAppData(appId, { stream: true }, includeRows)
+exports.streamBackup = async (appId, excludeRows) => {
+  return await backupAppData(appId, { stream: true }, excludeRows)
 }
 
 /**

package/src/utilities/index.js

@@ -162,3 +162,11 @@ exports.convertBookmark = bookmark => {
   }
   return bookmark
 }
+
+exports.isQsTrue = param => {
+  if (typeof param === "string") {
+    return param.toLowerCase() === "true"
+  } else {
+    return param === true
+  }
+}

package/src/utilities/rowProcessor/utils.js

@@ -76,7 +76,7 @@ exports.processDates = (table, rows) => {
     if (schema.type !== FieldTypes.DATETIME) {
       continue
     }
-    if (!schema.ignoreTimezones) {
+    if (!schema.timeOnly && !schema.ignoreTimezones) {
       datesWithTZ.push(column)
     }
   }

package/src/utilities/usageQuota/rows.js

@@ -72,7 +72,8 @@ exports.getUniqueRows = async appIds => {
     // ensure uniqueness on a per app pair basis
     // this can't be done on all rows because app import results in
     // duplicate row ids across apps
-    uniqueRows = uniqueRows.concat(...new Set(appRows))
+    // the array pre-concat is important to avoid stack overflow
+    uniqueRows = uniqueRows.concat([...new Set(appRows)])
   }
 
   return uniqueRows