@budibase/server 1.0.27-alpha.2 → 1.0.27-alpha.6

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.27-alpha.2",
+  "version": "1.0.27-alpha.6",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/auth": "^1.0.27-alpha.2",
-    "@budibase/client": "^1.0.27-alpha.2",
-    "@budibase/string-templates": "^1.0.27-alpha.2",
+    "@budibase/auth": "^1.0.27-alpha.6",
+    "@budibase/client": "^1.0.27-alpha.6",
+    "@budibase/string-templates": "^1.0.27-alpha.6",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -122,6 +122,7 @@
     "pouchdb-replication-stream": "1.2.9",
     "server-destroy": "1.0.1",
     "svelte": "^3.38.2",
+    "swagger-parser": "^10.0.3",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
@@ -164,5 +165,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "0c5de8cf6518f05e4b497857c23e551d293049c1"
+  "gitHead": "4ccf3f96877736296736f4e74b68c0d9e99c2e18"
 }

package/src/api/controllers/query/import/sources/curl.ts

@@ -17,8 +17,12 @@ const parseBody = (curl: any) => {
   if (curl.data) {
     const keys = Object.keys(curl.data)
     if (keys.length) {
-      const key = keys[0]
+      let key = keys[0]
       try {
+        // filter out the dollar syntax used by curl for shell support
+        if (key.startsWith("$")) {
+          key = key.substring(1)
+        }
         return JSON.parse(key)
       } catch (e) {
         // do nothing

package/src/api/controllers/query/index.js

@@ -1,4 +1,3 @@
-const { processString } = require("@budibase/string-templates")
 const CouchDB = require("../../../db")
 const {
   generateQueryID,
@@ -90,47 +89,6 @@ exports.save = async function (ctx) {
   ctx.message = `Query ${query.name} saved successfully.`
 }
 
-async function enrichQueryFields(fields, parameters = {}) {
-  const enrichedQuery = {}
-
-  // enrich the fields with dynamic parameters
-  for (let key of Object.keys(fields)) {
-    if (fields[key] == null) {
-      continue
-    }
-    if (typeof fields[key] === "object") {
-      // enrich nested fields object
-      enrichedQuery[key] = await enrichQueryFields(fields[key], parameters)
-    } else if (typeof fields[key] === "string") {
-      // enrich string value as normal
-      enrichedQuery[key] = await processString(fields[key], parameters, {
-        noHelpers: true,
-      })
-    } else {
-      enrichedQuery[key] = fields[key]
-    }
-  }
-
-  if (
-    enrichedQuery.json ||
-    enrichedQuery.customData ||
-    enrichedQuery.requestBody
-  ) {
-    try {
-      enrichedQuery.json = JSON.parse(
-        enrichedQuery.json ||
-          enrichedQuery.customData ||
-          enrichedQuery.requestBody
-      )
-    } catch (err) {
-      // no json found, ignore
-    }
-    delete enrichedQuery.customData
-  }
-
-  return enrichedQuery
-}
-
 exports.find = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const query = enrichQueries(await db.get(ctx.params.queryId))
@@ -146,16 +104,20 @@ exports.preview = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
   const datasource = await db.get(ctx.request.body.datasourceId)
-
-  const { fields, parameters, queryVerb, transformer } = ctx.request.body
-  const enrichedQuery = await enrichQueryFields(fields, parameters)
+  // preview may not have a queryId as it hasn't been saved, but if it does
+  // this stops dynamic variables from calling the same query
+  const { fields, parameters, queryVerb, transformer, queryId } =
+    ctx.request.body
 
   try {
     const { rows, keys, info, extra } = await Runner.run({
+      appId: ctx.appId,
       datasource,
       queryVerb,
-      query: enrichedQuery,
+      fields,
+      parameters,
       transformer,
+      queryId,
     })
 
     ctx.body = {
@@ -169,31 +131,41 @@ exports.preview = async function (ctx) {
   }
 }
 
-exports.execute = async function (ctx) {
+async function execute(ctx, opts = { rowsOnly: false }) {
   const db = new CouchDB(ctx.appId)
 
   const query = await db.get(ctx.params.queryId)
   const datasource = await db.get(query.datasourceId)
 
-  const enrichedQuery = await enrichQueryFields(
-    query.fields,
-    ctx.request.body.parameters
-  )
-
   // call the relevant CRUD method on the integration class
   try {
-    const { rows } = await Runner.run({
+    const { rows, extra } = await Runner.run({
+      appId: ctx.appId,
       datasource,
       queryVerb: query.queryVerb,
-      query: enrichedQuery,
+      fields: query.fields,
+      parameters: ctx.request.body.parameters,
       transformer: query.transformer,
+      queryId: ctx.params.queryId,
     })
-    ctx.body = rows
+    if (opts && opts.rowsOnly) {
+      ctx.body = rows
+    } else {
+      ctx.body = { data: rows, ...extra }
+    }
   } catch (err) {
     ctx.throw(400, err)
   }
 }
 
+exports.executeV1 = async function (ctx) {
+  return execute(ctx, { rowsOnly: true })
+}
+
+exports.executeV2 = async function (ctx) {
+  return execute(ctx, { rowsOnly: false })
+}
+
 exports.destroy = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   await db.remove(ctx.params.queryId, ctx.params.revId)

package/src/api/controllers/query/validation.js

@@ -36,6 +36,7 @@ exports.generateQueryPreviewValidation = () => {
     extra: Joi.object().optional(),
     datasourceId: Joi.string().required(),
     transformer: Joi.string().optional(),
-    parameters: Joi.object({}).required().unknown(true)
+    parameters: Joi.object({}).required().unknown(true),
+    queryId: Joi.string().optional(),
   }))
 }

package/src/api/routes/query.js

@@ -41,11 +41,18 @@ router
     authorized(PermissionTypes.QUERY, PermissionLevels.READ),
     queryController.find
   )
+  // DEPRECATED - use new query endpoint for future work
   .post(
     "/api/queries/:queryId",
     paramResource("queryId"),
     authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
-    queryController.execute
+    queryController.executeV1
+  )
+  .post(
+    "/api/v2/queries/:queryId",
+    paramResource("queryId"),
+    authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
+    queryController.executeV2
   )
   .delete(
     "/api/queries/:queryId/:revId",

package/src/api/routes/tests/query.spec.js

@@ -1,5 +1,6 @@
 // Mock out postgres for this
 jest.mock("pg")
+jest.mock("node-fetch")
 
 // Mock isProdAppID to we can later mock the implementation and pretend we are
 // using prod app IDs
@@ -10,6 +11,7 @@ authDb.isProdAppID = mockIsProdAppID
 
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+const { checkCacheForDynamicVariable } = require("../../../threads/utils")
 const { basicQuery, basicDatasource } = setup.structures
 
 describe("/queries", () => {
@@ -226,4 +228,90 @@ describe("/queries", () => {
         .expect(400)
     })
   })
+
+  describe("test variables", () => {
+    async function restDatasource(cfg) {
+      return await config.createDatasource({
+        datasource: {
+          ...basicDatasource().datasource,
+          source: "REST",
+          config: cfg || {},
+        },
+      })
+    }
+
+    async function dynamicVariableDatasource() {
+      const datasource = await restDatasource()
+      const basedOnQuery = await config.createQuery({
+        ...basicQuery(datasource._id),
+        fields: {
+          path: "www.google.com",
+        },
+      })
+      await config.updateDatasource({
+        ...datasource,
+        config: {
+          dynamicVariables: [
+            { queryId: basedOnQuery._id, name: "variable3", value: "{{ data.0.[value] }}" }
+          ]
+        }
+      })
+      return { datasource, query: basedOnQuery }
+    }
+
+    async function preview(datasource, fields) {
+      return await request
+        .post(`/api/queries/preview`)
+        .send({
+          datasourceId: datasource._id,
+          parameters: {},
+          fields,
+          queryVerb: "read",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+    }
+
+    it("should work with static variables", async () => {
+      const datasource = await restDatasource({
+        staticVariables: {
+          variable: "google",
+          variable2: "1",
+        },
+      })
+      const res = await preview(datasource, {
+        path: "www.{{ variable }}.com",
+        queryString: "test={{ variable2 }}",
+      })
+      // these responses come from the mock
+      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
+    })
+
+    it("should work with dynamic variables", async () => {
+      const { datasource } = await dynamicVariableDatasource()
+      const res = await preview(datasource, {
+        path: "www.google.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.rows[0].url).toContain("doctype html")
+    })
+
+    it("check that it automatically retries on fail with cached dynamics", async () => {
+      const { datasource, query: base } = await dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      // check its in cache
+      const contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents.rows.length).toEqual(1)
+      const res = await preview(datasource, {
+        path: "www.failonce.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
+      expect(res.body.rows[0].fails).toEqual(1)
+    })
+  })
 })

package/src/automations/steps/executeQuery.js

@@ -35,6 +35,11 @@ exports.definition = {
           type: "object",
           description: "The response from the datasource execution",
         },
+        info: {
+          type: "object",
+          description:
+            "Some query types may return extra data, like headers from a REST query",
+        },
         success: {
           type: "boolean",
           description: "Whether the action was successful",
@@ -68,13 +73,16 @@ exports.run = async function ({ inputs, appId, emitter }) {
 
   try {
     await queryController.execute(ctx)
+    const { data, ...rest } = ctx.body
     return {
-      response: ctx.body,
+      response: data,
+      info: rest,
       success: true,
     }
   } catch (err) {
     return {
       success: false,
+      info: {},
       response: automationUtils.getError(err),
     }
   }

package/src/definitions/datasource.ts

@@ -240,6 +240,16 @@ export interface RestConfig {
     [key: string]: any
   }
   authConfigs: AuthConfig[]
+  staticVariables: {
+    [key: string]: string
+  }
+  dynamicVariables: [
+    {
+      name: string
+      queryId: string
+      value: string
+    }
+  ]
 }
 
 export interface Query {

package/src/integrations/rest.ts

@@ -48,7 +48,10 @@ module RestModule {
   const { performance } = require("perf_hooks")
   const FormData = require("form-data")
   const { URLSearchParams } = require("url")
-  const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
+  const {
+    parseStringPromise: xmlParser,
+    Builder: XmlBuilder,
+  } = require("xml2js")
 
   const SCHEMA: Integration = {
     docs: "https://github.com/node-fetch/node-fetch",
@@ -211,7 +214,7 @@ module RestModule {
           break
         case BodyTypes.XML:
           if (object != null) {
-            string = (new XmlBuilder()).buildObject(object)
+            string = new XmlBuilder().buildObject(object)
           }
           input.body = string
           input.headers["Content-Type"] = "application/xml"

package/src/tests/utilities/TestConfiguration.js

@@ -316,6 +316,16 @@ class TestConfiguration {
     return this.datasource
   }
 
+  async updateDatasource(datasource) {
+    const response = await this._req(
+      datasource,
+      { datasourceId: datasource._id },
+      controllers.datasource.update
+    )
+    this.datasource = response.datasource
+    return this.datasource
+  }
+
   async createQuery(config = null) {
     if (!this.datasource && !config) {
       throw "No data source created for query."

package/src/threads/query.js

@@ -1,77 +1,166 @@
-require("./utils").threadSetup()
+const threadUtils = require("./utils")
+threadUtils.threadSetup()
 const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
+const { processStringSync } = require("@budibase/string-templates")
+const CouchDB = require("../db")
 
-function formatResponse(resp) {
-  if (typeof resp === "string") {
-    try {
-      resp = JSON.parse(resp)
-    } catch (err) {
-      resp = { response: resp }
-    }
+class QueryRunner {
+  constructor(input, flags = { noRecursiveQuery: false }) {
+    this.appId = input.appId
+    this.datasource = input.datasource
+    this.queryVerb = input.queryVerb
+    this.fields = input.fields
+    this.parameters = input.parameters
+    this.transformer = input.transformer
+    this.queryId = input.queryId
+    this.noRecursiveQuery = flags.noRecursiveQuery
+    this.cachedVariables = []
+    // allows the response from a query to be stored throughout this
+    // execution so that if it needs to be re-used for another variable
+    // it can be
+    this.queryResponse = {}
+    this.hasRerun = false
   }
-  return resp
-}
 
-function hasExtraData(response) {
-  return (
-    typeof response === "object" &&
-    !Array.isArray(response) &&
-    response.data != null &&
-    response.info != null
-  )
-}
+  async execute() {
+    let { datasource, fields, queryVerb, transformer } = this
+    // pre-query, make sure datasource variables are added to parameters
+    const parameters = await this.addDatasourceVariables()
+    const query = threadUtils.enrichQueryFields(fields, parameters)
+    const Integration = integrations[datasource.source]
+    if (!Integration) {
+      throw "Integration type does not exist."
+    }
+    const integration = new Integration(datasource.config)
 
-async function runAndTransform(datasource, queryVerb, query, transformer) {
-  const Integration = integrations[datasource.source]
-  if (!Integration) {
-    throw "Integration type does not exist."
-  }
-  const integration = new Integration(datasource.config)
+    let output = threadUtils.formatResponse(await integration[queryVerb](query))
+    let rows = output,
+      info = undefined,
+      extra = undefined
+    if (threadUtils.hasExtraData(output)) {
+      rows = output.data
+      info = output.info
+      extra = output.extra
+    }
 
-  let output = formatResponse(await integration[queryVerb](query))
-  let rows = output,
-    info = undefined,
-    extra = undefined
-  if (hasExtraData(output)) {
-    rows = output.data
-    info = output.info
-    extra = output.extra
-  }
+    // transform as required
+    if (transformer) {
+      const runner = new ScriptRunner(transformer, { data: rows })
+      rows = runner.execute()
+    }
 
-  // transform as required
-  if (transformer) {
-    const runner = new ScriptRunner(transformer, { data: rows })
-    rows = runner.execute()
-  }
+    // if the request fails we retry once, invalidating the cached value
+    if (
+      info &&
+      info.code >= 400 &&
+      this.cachedVariables.length > 0 &&
+      !this.hasRerun
+    ) {
+      this.hasRerun = true
+      // invalidate the cache value
+      await threadUtils.invalidateDynamicVariables(this.cachedVariables)
+      return this.execute()
+    }
 
-  // needs to an array for next step
-  if (!Array.isArray(rows)) {
-    rows = [rows]
-  }
+    // needs to an array for next step
+    if (!Array.isArray(rows)) {
+      rows = [rows]
+    }
 
-  // map into JSON if just raw primitive here
-  if (rows.find(row => typeof row !== "object")) {
-    rows = rows.map(value => ({ value }))
+    // map into JSON if just raw primitive here
+    if (rows.find(row => typeof row !== "object")) {
+      rows = rows.map(value => ({ value }))
+    }
+
+    // get all the potential fields in the schema
+    let keys = rows.flatMap(Object.keys)
+
+    if (integration.end) {
+      integration.end()
+    }
+
+    return { rows, keys, info, extra }
   }
 
-  // get all the potential fields in the schema
-  let keys = rows.flatMap(Object.keys)
+  async runAnotherQuery(queryId, parameters) {
+    const db = new CouchDB(this.appId)
+    const query = await db.get(queryId)
+    const datasource = await db.get(query.datasourceId)
+    return new QueryRunner(
+      {
+        appId: this.appId,
+        datasource,
+        queryVerb: query.queryVerb,
+        fields: query.fields,
+        parameters,
+        transformer: query.transformer,
+      },
+      { noRecursiveQuery: true }
+    ).execute()
+  }
 
-  if (integration.end) {
-    integration.end()
+  async getDynamicVariable(variable) {
+    let { parameters } = this
+    const queryId = variable.queryId,
+      name = variable.name
+    let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
+    if (!value) {
+      value = this.queryResponse[queryId]
+        ? this.queryResponse[queryId]
+        : await this.runAnotherQuery(queryId, parameters)
+      // store incase this query is to be called again
+      this.queryResponse[queryId] = value
+      await threadUtils.storeDynamicVariable(queryId, name, value)
+    } else {
+      this.cachedVariables.push({ queryId, name })
+    }
+    return value
   }
 
-  return { rows, keys, info, extra }
+  async addDatasourceVariables() {
+    let { datasource, parameters, fields } = this
+    if (!datasource || !datasource.config) {
+      return parameters
+    }
+    const staticVars = datasource.config.staticVariables || {}
+    const dynamicVars = datasource.config.dynamicVariables || []
+    for (let [key, value] of Object.entries(staticVars)) {
+      if (!parameters[key]) {
+        parameters[key] = value
+      }
+    }
+    if (!this.noRecursiveQuery) {
+      // need to see if this uses any variables
+      const stringFields = JSON.stringify(fields)
+      const foundVars = dynamicVars.filter(variable => {
+        // don't allow a query to use its own dynamic variable (loop)
+        if (variable.queryId === this.queryId) {
+          return false
+        }
+        // look for {{ variable }} but allow spaces between handlebars
+        const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
+        return regex.test(stringFields)
+      })
+      const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar))
+      const responses = await Promise.all(dynamics)
+      for (let i = 0; i < foundVars.length; i++) {
+        const variable = foundVars[i]
+        parameters[variable.name] = processStringSync(variable.value, {
+          data: responses[i].rows,
+          info: responses[i].extra,
+        })
+        // make sure its known that this uses dynamic variables in case it fails
+        this.hasDynamicVariables = true
+      }
+    }
+    return parameters
+  }
 }
 
 module.exports = (input, callback) => {
-  runAndTransform(
-    input.datasource,
-    input.queryVerb,
-    input.query,
-    input.transformer
-  )
+  const Runner = new QueryRunner(input)
+  Runner.execute()
     .then(response => {
       callback(null, response)
     })