@budibase/server 1.0.27-alpha.1 → 1.0.27-alpha.10

package/src/api/routes/tests/query.spec.js

@@ -1,5 +1,6 @@
 // Mock out postgres for this
 jest.mock("pg")
+jest.mock("node-fetch")
 
 // Mock isProdAppID to we can later mock the implementation and pretend we are
 // using prod app IDs
@@ -10,6 +11,7 @@ authDb.isProdAppID = mockIsProdAppID
 
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+const { checkCacheForDynamicVariable } = require("../../../threads/utils")
 const { basicQuery, basicDatasource } = setup.structures
 
 describe("/queries", () => {
@@ -226,4 +228,90 @@ describe("/queries", () => {
         .expect(400)
     })
   })
+
+  describe("test variables", () => {
+    async function restDatasource(cfg) {
+      return await config.createDatasource({
+        datasource: {
+          ...basicDatasource().datasource,
+          source: "REST",
+          config: cfg || {},
+        },
+      })
+    }
+
+    async function dynamicVariableDatasource() {
+      const datasource = await restDatasource()
+      const basedOnQuery = await config.createQuery({
+        ...basicQuery(datasource._id),
+        fields: {
+          path: "www.google.com",
+        },
+      })
+      await config.updateDatasource({
+        ...datasource,
+        config: {
+          dynamicVariables: [
+            { queryId: basedOnQuery._id, name: "variable3", value: "{{ data.0.[value] }}" }
+          ]
+        }
+      })
+      return { datasource, query: basedOnQuery }
+    }
+
+    async function preview(datasource, fields) {
+      return await request
+        .post(`/api/queries/preview`)
+        .send({
+          datasourceId: datasource._id,
+          parameters: {},
+          fields,
+          queryVerb: "read",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+    }
+
+    it("should work with static variables", async () => {
+      const datasource = await restDatasource({
+        staticVariables: {
+          variable: "google",
+          variable2: "1",
+        },
+      })
+      const res = await preview(datasource, {
+        path: "www.{{ variable }}.com",
+        queryString: "test={{ variable2 }}",
+      })
+      // these responses come from the mock
+      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
+    })
+
+    it("should work with dynamic variables", async () => {
+      const { datasource } = await dynamicVariableDatasource()
+      const res = await preview(datasource, {
+        path: "www.google.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.rows[0].url).toContain("doctype html")
+    })
+
+    it("check that it automatically retries on fail with cached dynamics", async () => {
+      const { datasource, query: base } = await dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      // check its in cache
+      const contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents.rows.length).toEqual(1)
+      const res = await preview(datasource, {
+        path: "www.failonce.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
+      expect(res.body.rows[0].fails).toEqual(1)
+    })
+  })
 })

package/src/automations/steps/executeQuery.js

@@ -35,6 +35,11 @@ exports.definition = {
           type: "object",
           description: "The response from the datasource execution",
         },
+        info: {
+          type: "object",
+          description:
+            "Some query types may return extra data, like headers from a REST query",
+        },
         success: {
           type: "boolean",
           description: "Whether the action was successful",
@@ -68,13 +73,16 @@ exports.run = async function ({ inputs, appId, emitter }) {
 
   try {
     await queryController.execute(ctx)
+    const { data, ...rest } = ctx.body
     return {
-      response: ctx.body,
+      response: data,
+      info: rest,
       success: true,
     }
   } catch (err) {
     return {
       success: false,
+      info: {},
       response: automationUtils.getError(err),
     }
   }

package/src/definitions/datasource.ts

@@ -240,6 +240,16 @@ export interface RestConfig {
     [key: string]: any
   }
   authConfigs: AuthConfig[]
+  staticVariables: {
+    [key: string]: string
+  }
+  dynamicVariables: [
+    {
+      name: string
+      queryId: string
+      value: string
+    }
+  ]
 }
 
 export interface Query {

package/src/integrations/base/sqlTable.ts

@@ -36,7 +36,7 @@ function generateSchema(
       case FieldTypes.STRING:
       case FieldTypes.OPTIONS:
       case FieldTypes.LONGFORM:
-        schema.string(key)
+        schema.text(key)
         break
       case FieldTypes.NUMBER:
         // if meta is specified then this is a junction table entry

package/src/integrations/rest.ts

@@ -48,7 +48,10 @@ module RestModule {
   const { performance } = require("perf_hooks")
   const FormData = require("form-data")
   const { URLSearchParams } = require("url")
-  const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
+  const {
+    parseStringPromise: xmlParser,
+    Builder: XmlBuilder,
+  } = require("xml2js")
 
   const SCHEMA: Integration = {
     docs: "https://github.com/node-fetch/node-fetch",
@@ -168,7 +171,7 @@ module RestModule {
     getUrl(path: string, queryString: string): string {
       const main = `${path}?${queryString}`
       let complete = main
-      if (this.config.url && !main.startsWith(this.config.url)) {
+      if (this.config.url && !main.startsWith("http")) {
         complete = !this.config.url ? main : `${this.config.url}/${main}`
       }
       if (!complete.startsWith("http")) {
@@ -211,7 +214,7 @@ module RestModule {
           break
         case BodyTypes.XML:
           if (object != null) {
-            string = (new XmlBuilder()).buildObject(object)
+            string = new XmlBuilder().buildObject(object)
           }
           input.body = string
           input.headers["Content-Type"] = "application/xml"

package/src/tests/utilities/TestConfiguration.js

@@ -316,6 +316,16 @@ class TestConfiguration {
     return this.datasource
   }
 
+  async updateDatasource(datasource) {
+    const response = await this._req(
+      datasource,
+      { datasourceId: datasource._id },
+      controllers.datasource.update
+    )
+    this.datasource = response.datasource
+    return this.datasource
+  }
+
   async createQuery(config = null) {
     if (!this.datasource && !config) {
       throw "No data source created for query."

package/src/threads/query.js

@@ -1,77 +1,169 @@
-require("./utils").threadSetup()
+const threadUtils = require("./utils")
+threadUtils.threadSetup()
 const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
+const { processStringSync } = require("@budibase/string-templates")
+const CouchDB = require("../db")
 
-function formatResponse(resp) {
-  if (typeof resp === "string") {
-    try {
-      resp = JSON.parse(resp)
-    } catch (err) {
-      resp = { response: resp }
-    }
+class QueryRunner {
+  constructor(input, flags = { noRecursiveQuery: false }) {
+    this.appId = input.appId
+    this.datasource = input.datasource
+    this.queryVerb = input.queryVerb
+    this.fields = input.fields
+    this.parameters = input.parameters
+    this.transformer = input.transformer
+    this.queryId = input.queryId
+    this.noRecursiveQuery = flags.noRecursiveQuery
+    this.cachedVariables = []
+    // allows the response from a query to be stored throughout this
+    // execution so that if it needs to be re-used for another variable
+    // it can be
+    this.queryResponse = {}
+    this.hasRerun = false
   }
-  return resp
-}
 
-function hasExtraData(response) {
-  return (
-    typeof response === "object" &&
-    !Array.isArray(response) &&
-    response.data != null &&
-    response.info != null
-  )
-}
+  async execute() {
+    let { datasource, fields, queryVerb, transformer } = this
+    // pre-query, make sure datasource variables are added to parameters
+    const parameters = await this.addDatasourceVariables()
+    const query = threadUtils.enrichQueryFields(fields, parameters)
+    const Integration = integrations[datasource.source]
+    if (!Integration) {
+      throw "Integration type does not exist."
+    }
+    const integration = new Integration(datasource.config)
 
-async function runAndTransform(datasource, queryVerb, query, transformer) {
-  const Integration = integrations[datasource.source]
-  if (!Integration) {
-    throw "Integration type does not exist."
-  }
-  const integration = new Integration(datasource.config)
+    let output = threadUtils.formatResponse(await integration[queryVerb](query))
+    let rows = output,
+      info = undefined,
+      extra = undefined
+    if (threadUtils.hasExtraData(output)) {
+      rows = output.data
+      info = output.info
+      extra = output.extra
+    }
 
-  let output = formatResponse(await integration[queryVerb](query))
-  let rows = output,
-    info = undefined,
-    extra = undefined
-  if (hasExtraData(output)) {
-    rows = output.data
-    info = output.info
-    extra = output.extra
-  }
+    // transform as required
+    if (transformer) {
+      const runner = new ScriptRunner(transformer, {
+        data: rows,
+        params: parameters,
+      })
+      rows = runner.execute()
+    }
 
-  // transform as required
-  if (transformer) {
-    const runner = new ScriptRunner(transformer, { data: rows })
-    rows = runner.execute()
-  }
+    // if the request fails we retry once, invalidating the cached value
+    if (
+      info &&
+      info.code >= 400 &&
+      this.cachedVariables.length > 0 &&
+      !this.hasRerun
+    ) {
+      this.hasRerun = true
+      // invalidate the cache value
+      await threadUtils.invalidateDynamicVariables(this.cachedVariables)
+      return this.execute()
+    }
 
-  // needs to an array for next step
-  if (!Array.isArray(rows)) {
-    rows = [rows]
-  }
+    // needs to an array for next step
+    if (!Array.isArray(rows)) {
+      rows = [rows]
+    }
 
-  // map into JSON if just raw primitive here
-  if (rows.find(row => typeof row !== "object")) {
-    rows = rows.map(value => ({ value }))
+    // map into JSON if just raw primitive here
+    if (rows.find(row => typeof row !== "object")) {
+      rows = rows.map(value => ({ value }))
+    }
+
+    // get all the potential fields in the schema
+    let keys = rows.flatMap(Object.keys)
+
+    if (integration.end) {
+      integration.end()
+    }
+
+    return { rows, keys, info, extra }
   }
 
-  // get all the potential fields in the schema
-  let keys = rows.flatMap(Object.keys)
+  async runAnotherQuery(queryId, parameters) {
+    const db = new CouchDB(this.appId)
+    const query = await db.get(queryId)
+    const datasource = await db.get(query.datasourceId)
+    return new QueryRunner(
+      {
+        appId: this.appId,
+        datasource,
+        queryVerb: query.queryVerb,
+        fields: query.fields,
+        parameters,
+        transformer: query.transformer,
+      },
+      { noRecursiveQuery: true }
+    ).execute()
+  }
 
-  if (integration.end) {
-    integration.end()
+  async getDynamicVariable(variable) {
+    let { parameters } = this
+    const queryId = variable.queryId,
+      name = variable.name
+    let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
+    if (!value) {
+      value = this.queryResponse[queryId]
+        ? this.queryResponse[queryId]
+        : await this.runAnotherQuery(queryId, parameters)
+      // store incase this query is to be called again
+      this.queryResponse[queryId] = value
+      await threadUtils.storeDynamicVariable(queryId, name, value)
+    } else {
+      this.cachedVariables.push({ queryId, name })
+    }
+    return value
   }
 
-  return { rows, keys, info, extra }
+  async addDatasourceVariables() {
+    let { datasource, parameters, fields } = this
+    if (!datasource || !datasource.config) {
+      return parameters
+    }
+    const staticVars = datasource.config.staticVariables || {}
+    const dynamicVars = datasource.config.dynamicVariables || []
+    for (let [key, value] of Object.entries(staticVars)) {
+      if (!parameters[key]) {
+        parameters[key] = value
+      }
+    }
+    if (!this.noRecursiveQuery) {
+      // need to see if this uses any variables
+      const stringFields = JSON.stringify(fields)
+      const foundVars = dynamicVars.filter(variable => {
+        // don't allow a query to use its own dynamic variable (loop)
+        if (variable.queryId === this.queryId) {
+          return false
+        }
+        // look for {{ variable }} but allow spaces between handlebars
+        const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
+        return regex.test(stringFields)
+      })
+      const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar))
+      const responses = await Promise.all(dynamics)
+      for (let i = 0; i < foundVars.length; i++) {
+        const variable = foundVars[i]
+        parameters[variable.name] = processStringSync(variable.value, {
+          data: responses[i].rows,
+          info: responses[i].extra,
+        })
+        // make sure its known that this uses dynamic variables in case it fails
+        this.hasDynamicVariables = true
+      }
+    }
+    return parameters
+  }
 }
 
 module.exports = (input, callback) => {
-  runAndTransform(
-    input.datasource,
-    input.queryVerb,
-    input.query,
-    input.transformer
-  )
+  const Runner = new QueryRunner(input)
+  Runner.execute()
     .then(response => {
       callback(null, response)
     })

package/src/threads/utils.js

@@ -1,6 +1,26 @@
 const env = require("../environment")
 const CouchDB = require("../db")
 const { init } = require("@budibase/auth")
+const redis = require("@budibase/auth/redis")
+const { SEPARATOR } = require("@budibase/auth/db")
+const { processStringSync } = require("@budibase/string-templates")
+
+const VARIABLE_TTL_SECONDS = 3600
+let client
+
+const IS_TRIPLE_BRACE = new RegExp(/^{{3}.*}{3}$/)
+const IS_HANDLEBARS = new RegExp(/^{{2}.*}{2}$/)
+
+async function getClient() {
+  if (!client) {
+    client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
+  }
+  return client
+}
+
+process.on("exit", async () => {
+  if (client) await client.finish()
+})
 
 exports.threadSetup = () => {
   // don't run this if not threading
@@ -11,3 +31,98 @@ exports.threadSetup = () => {
   env.setInThread()
   init(CouchDB)
 }
+
+function makeVariableKey(queryId, variable) {
+  return `${queryId}${SEPARATOR}${variable}`
+}
+
+exports.checkCacheForDynamicVariable = async (queryId, variable) => {
+  const cache = await getClient()
+  return cache.get(makeVariableKey(queryId, variable))
+}
+
+exports.invalidateDynamicVariables = async cachedVars => {
+  const cache = await getClient()
+  let promises = []
+  for (let variable of cachedVars) {
+    promises.push(
+      cache.delete(makeVariableKey(variable.queryId, variable.name))
+    )
+  }
+  await Promise.all(promises)
+}
+
+exports.storeDynamicVariable = async (queryId, variable, value) => {
+  const cache = await getClient()
+  await cache.store(
+    makeVariableKey(queryId, variable),
+    value,
+    VARIABLE_TTL_SECONDS
+  )
+}
+
+exports.formatResponse = resp => {
+  if (typeof resp === "string") {
+    try {
+      resp = JSON.parse(resp)
+    } catch (err) {
+      resp = { response: resp }
+    }
+  }
+  return resp
+}
+
+exports.hasExtraData = response => {
+  return (
+    typeof response === "object" &&
+    !Array.isArray(response) &&
+    response.data != null &&
+    response.info != null
+  )
+}
+
+exports.enrichQueryFields = (fields, parameters = {}) => {
+  const enrichedQuery = {}
+
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      let value = fields[key]
+      // add triple brace to avoid escaping e.g. '=' in cookie header
+      if (IS_HANDLEBARS.test(value) && !IS_TRIPLE_BRACE.test(value)) {
+        value = `{${value}}`
+      }
+      enrichedQuery[key] = processStringSync(value, parameters, {
+        noHelpers: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+
+  if (
+    enrichedQuery.json ||
+    enrichedQuery.customData ||
+    enrichedQuery.requestBody
+  ) {
+    try {
+      enrichedQuery.json = JSON.parse(
+        enrichedQuery.json ||
+          enrichedQuery.customData ||
+          enrichedQuery.requestBody
+      )
+    } catch (err) {
+      // no json found, ignore
+    }
+    delete enrichedQuery.customData
+  }
+
+  return enrichedQuery
+}

package/src/utilities/fileSystem/utilities.js

@@ -2,6 +2,7 @@ const {
   ObjectStore,
   makeSureBucketExists,
   upload,
+  deleteFiles,
   streamUpload,
   retrieve,
   retrieveToTmp,
@@ -28,3 +29,4 @@ exports.retrieveToTmp = retrieveToTmp
 exports.deleteFolder = deleteFolder
 exports.uploadDirectory = uploadDirectory
 exports.downloadTarball = downloadTarball
+exports.deleteFiles = deleteFiles

package/src/utilities/rowProcessor/index.js

@@ -3,6 +3,10 @@ const { cloneDeep } = require("lodash/fp")
 const { FieldTypes, AutoFieldSubTypes } = require("../../constants")
 const { attachmentsRelativeURL } = require("../index")
 const { processFormulas } = require("./utils")
+const { deleteFiles } = require("../../utilities/fileSystem/utilities")
+const { ObjectStoreBuckets } = require("../../constants")
+const { isProdAppID, getDeployedAppID, dbExists } = require("@budibase/auth/db")
+const CouchDB = require("../../db")
 
 const BASE_AUTO_ID = 1
 
@@ -81,6 +85,35 @@ const TYPE_TRANSFORM_MAP = {
   [FieldTypes.AUTO]: {
     parse: () => undefined,
   },
+  [FieldTypes.JSON]: {
+    parse: input => {
+      try {
+        if (input === "") {
+          return undefined
+        }
+        return JSON.parse(input)
+      } catch (err) {
+        return input
+      }
+    },
+  },
+}
+
+/**
+ * Given the old state of the row and the new one after an update, this will
+ * find the keys that have been removed in the updated row.
+ */
+function getRemovedAttachmentKeys(oldRow, row, attachmentKey) {
+  if (!oldRow[attachmentKey]) {
+    return []
+  }
+  const oldKeys = oldRow[attachmentKey].map(attachment => attachment.key)
+  // no attachments in new row, all removed
+  if (!row[attachmentKey]) {
+    return oldKeys
+  }
+  const newKeys = row[attachmentKey].map(attachment => attachment.key)
+  return oldKeys.filter(key => newKeys.indexOf(key) === -1)
 }
 
 /**
@@ -260,3 +293,45 @@ exports.outputProcessing = async (
   }
   return wasArray ? enriched : enriched[0]
 }
+
+/**
+ * Clean up any attachments that were attached to a row.
+ * @param {string} appId The ID of the app from which a row is being deleted.
+ * @param {object} table The table from which a row is being removed.
+ * @param {any} row optional - the row being removed.
+ * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
+ * @param {any} oldRow optional - if updating a row this will determine the difference.
+ * @return {Promise<void>} When all attachments have been removed this will return.
+ */
+exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
+  if (!isProdAppID(appId)) {
+    const prodAppId = getDeployedAppID(appId)
+    // if prod exists, then don't allow deleting
+    const exists = await dbExists(CouchDB, prodAppId)
+    if (exists) {
+      return
+    }
+  }
+  let files = []
+  function addFiles(row, key) {
+    if (row[key]) {
+      files = files.concat(row[key].map(attachment => attachment.key))
+    }
+  }
+  for (let [key, schema] of Object.entries(table.schema)) {
+    if (schema.type !== FieldTypes.ATTACHMENT) {
+      continue
+    }
+    // if updating, need to manage the differences
+    if (oldRow && row) {
+      files = files.concat(getRemovedAttachmentKeys(oldRow, row, key))
+    } else if (row) {
+      addFiles(row, key)
+    } else if (rows) {
+      rows.forEach(row => addFiles(row, key))
+    }
+  }
+  if (files.length > 0) {
+    return deleteFiles(ObjectStoreBuckets.APPS, files)
+  }
+}