@budibase/server 1.0.27-alpha.1 → 1.0.27-alpha.10

package/dist/utilities/fileSystem/utilities.js

@@ -1,5 +1,5 @@
 "use strict";
-const { ObjectStore, makeSureBucketExists, upload, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, } = require("@budibase/auth").objectStore;
+const { ObjectStore, makeSureBucketExists, upload, deleteFiles, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, } = require("@budibase/auth").objectStore;
 /***********************************
  *              NOTE               *
  * This file purely exists so that *
@@ -17,3 +17,4 @@ exports.retrieveToTmp = retrieveToTmp;
 exports.deleteFolder = deleteFolder;
 exports.uploadDirectory = uploadDirectory;
 exports.downloadTarball = downloadTarball;
+exports.deleteFiles = deleteFiles;

package/dist/utilities/rowProcessor/index.js

@@ -13,6 +13,10 @@ const { cloneDeep } = require("lodash/fp");
 const { FieldTypes, AutoFieldSubTypes } = require("../../constants");
 const { attachmentsRelativeURL } = require("../index");
 const { processFormulas } = require("./utils");
+const { deleteFiles } = require("../../utilities/fileSystem/utilities");
+const { ObjectStoreBuckets } = require("../../constants");
+const { isProdAppID, getDeployedAppID, dbExists } = require("@budibase/auth/db");
+const CouchDB = require("../../db");
 const BASE_AUTO_ID = 1;
 /**
  * A map of how we convert various properties in rows to each other based on the row type.
@@ -89,7 +93,36 @@ const TYPE_TRANSFORM_MAP = {
     [FieldTypes.AUTO]: {
         parse: () => undefined,
     },
+    [FieldTypes.JSON]: {
+        parse: input => {
+            try {
+                if (input === "") {
+                    return undefined;
+                }
+                return JSON.parse(input);
+            }
+            catch (err) {
+                return input;
+            }
+        },
+    },
 };
+/**
+ * Given the old state of the row and the new one after an update, this will
+ * find the keys that have been removed in the updated row.
+ */
+function getRemovedAttachmentKeys(oldRow, row, attachmentKey) {
+    if (!oldRow[attachmentKey]) {
+        return [];
+    }
+    const oldKeys = oldRow[attachmentKey].map(attachment => attachment.key);
+    // no attachments in new row, all removed
+    if (!row[attachmentKey]) {
+        return oldKeys;
+    }
+    const newKeys = row[attachmentKey].map(attachment => attachment.key);
+    return oldKeys.filter(key => newKeys.indexOf(key) === -1);
+}
 /**
  * This will update any auto columns that are found on the row/table with the correct information based on
  * time now and the current logged in user making the request.
@@ -240,3 +273,46 @@ exports.outputProcessing = (ctx, table, rows, opts = { squash: true }) => __awai
     }
     return wasArray ? enriched : enriched[0];
 });
+/**
+ * Clean up any attachments that were attached to a row.
+ * @param {string} appId The ID of the app from which a row is being deleted.
+ * @param {object} table The table from which a row is being removed.
+ * @param {any} row optional - the row being removed.
+ * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
+ * @param {any} oldRow optional - if updating a row this will determine the difference.
+ * @return {Promise<void>} When all attachments have been removed this will return.
+ */
+exports.cleanupAttachments = (appId, table, { row, rows, oldRow }) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!isProdAppID(appId)) {
+        const prodAppId = getDeployedAppID(appId);
+        // if prod exists, then don't allow deleting
+        const exists = yield dbExists(CouchDB, prodAppId);
+        if (exists) {
+            return;
+        }
+    }
+    let files = [];
+    function addFiles(row, key) {
+        if (row[key]) {
+            files = files.concat(row[key].map(attachment => attachment.key));
+        }
+    }
+    for (let [key, schema] of Object.entries(table.schema)) {
+        if (schema.type !== FieldTypes.ATTACHMENT) {
+            continue;
+        }
+        // if updating, need to manage the differences
+        if (oldRow && row) {
+            files = files.concat(getRemovedAttachmentKeys(oldRow, row, key));
+        }
+        else if (row) {
+            addFiles(row, key);
+        }
+        else if (rows) {
+            rows.forEach(row => addFiles(row, key));
+        }
+    }
+    if (files.length > 0) {
+        return deleteFiles(ObjectStoreBuckets.APPS, files);
+    }
+});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.27-alpha.1",
+  "version": "1.0.27-alpha.10",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/auth": "^1.0.27-alpha.1",
-    "@budibase/client": "^1.0.27-alpha.1",
-    "@budibase/string-templates": "^1.0.27-alpha.1",
+    "@budibase/auth": "^1.0.27-alpha.10",
+    "@budibase/client": "^1.0.27-alpha.10",
+    "@budibase/string-templates": "^1.0.27-alpha.10",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -122,6 +122,7 @@
     "pouchdb-replication-stream": "1.2.9",
     "server-destroy": "1.0.1",
     "svelte": "^3.38.2",
+    "swagger-parser": "^10.0.3",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
@@ -164,5 +165,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "45e6e45098a50b224d1155bcc4d8b1bfefc1fe23"
+  "gitHead": "37b3e81a0d6f99566d3b753f79b91d5b05185106"
 }

package/src/api/controllers/query/import/sources/base/index.ts

@@ -1,7 +1,7 @@
 import { Query, QueryParameter } from "../../../../../../definitions/datasource"
+import { URL } from "url"
 
 export interface ImportInfo {
-  url: string
   name: string
 }
 
@@ -23,6 +23,7 @@ export abstract class ImportSource {
     name: string,
     method: string,
     path: string,
+    url: URL,
     queryString: string,
     headers: object = {},
     parameters: QueryParameter[] = [],
@@ -33,6 +34,7 @@ export abstract class ImportSource {
     const transformer = "return data"
     const schema = {}
     path = this.processPath(path)
+    path = `${url.origin}/${path}`
     queryString = this.processQuery(queryString)
     const requestBody = JSON.stringify(body, null, 2)
 

package/src/api/controllers/query/import/sources/curl.ts

@@ -17,8 +17,12 @@ const parseBody = (curl: any) => {
   if (curl.data) {
     const keys = Object.keys(curl.data)
     if (keys.length) {
-      const key = keys[0]
+      let key = keys[0]
       try {
+        // filter out the dollar syntax used by curl for shell support
+        if (key.startsWith("$")) {
+          key = key.substring(1)
+        }
         return JSON.parse(key)
       } catch (e) {
         // do nothing
@@ -56,16 +60,19 @@ export class Curl extends ImportSource {
     return true
   }
 
+  getUrl = (): URL => {
+    return new URL(this.curl.raw_url)
+  }
+
   getInfo = async (): Promise<ImportInfo> => {
-    const url = new URL(this.curl.url)
+    const url = this.getUrl()
     return {
-      url: url.origin,
       name: url.hostname,
     }
   }
 
   getQueries = async (datasourceId: string): Promise<Query[]> => {
-    const url = new URL(this.curl.raw_url)
+    const url = this.getUrl()
     const name = url.pathname
     const path = url.pathname
     const method = this.curl.method
@@ -83,6 +90,7 @@ export class Curl extends ImportSource {
       name,
       method,
       path,
+      url,
       queryString,
       headers,
       [],

package/src/api/controllers/query/import/sources/openapi2.ts

@@ -2,6 +2,7 @@ import { ImportInfo } from "./base"
 import { Query, QueryParameter } from "../../../../../definitions/datasource"
 import { OpenAPIV2 } from "openapi-types"
 import { OpenAPISource } from "./base/openapi"
+import { URL } from "url"
 
 const parameterNotRef = (
   param: OpenAPIV2.Parameter | OpenAPIV2.ReferenceObject
@@ -55,20 +56,22 @@ export class OpenAPI2 extends OpenAPISource {
     }
   }
 
-  getInfo = async (): Promise<ImportInfo> => {
+  getUrl = (): URL => {
     const scheme = this.document.schemes?.includes("https") ? "https" : "http"
     const basePath = this.document.basePath || ""
     const host = this.document.host || "<host>"
-    const url = `${scheme}://${host}${basePath}`
-    const name = this.document.info.title || "Swagger Import"
+    return new URL(`${scheme}://${host}${basePath}`)
+  }
 
+  getInfo = async (): Promise<ImportInfo> => {
+    const name = this.document.info.title || "Swagger Import"
     return {
-      url: url,
-      name: name,
+      name,
     }
   }
 
   getQueries = async (datasourceId: string): Promise<Query[]> => {
+    const url = this.getUrl()
     const queries = []
 
     for (let [path, pathItem] of Object.entries(this.document.paths)) {
@@ -145,6 +148,7 @@ export class OpenAPI2 extends OpenAPISource {
           name,
           methodName,
           path,
+          url,
           queryString,
           headers,
           parameters,

package/src/api/controllers/query/import/sources/tests/curl/curl.spec.js

@@ -35,7 +35,6 @@ describe("Curl Import", () => {
   it("returns import info", async () => {
     await init("get")
     const info = await curl.getInfo()
-    expect(info.url).toBe("http://example.com")
     expect(info.name).toBe("example.com")
   })
   
@@ -67,8 +66,8 @@ describe("Curl Import", () => {
     }
 
     it("populates path", async () => {
-        await testPath("get", "")
-        await testPath("path", "paths/abc")
+        await testPath("get", "http://example.com/")
+        await testPath("path", "http://example.com/paths/abc")
     })
 
     const testHeaders = async (file, headers) => {

package/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js

@@ -41,7 +41,6 @@ describe("OpenAPI2 Import", () => {
   const testImportInfo = async (file, extension) => {
     await init(file, extension)
     const info = await openapi2.getInfo()
-    expect(info.url).toBe("https://petstore.swagger.io/v2")
     expect(info.name).toBe("Swagger Petstore")
   }
 
@@ -92,12 +91,12 @@ describe("OpenAPI2 Import", () => {
 
     it("populates path", async () => {
       const assertions = {
-        "createEntity" : "entities",
-        "getEntities" : "entities",
-        "getEntity" : "entities/{{entityId}}",
-        "updateEntity" : "entities/{{entityId}}",
-        "patchEntity" : "entities/{{entityId}}",
-        "deleteEntity" : "entities/{{entityId}}"
+        "createEntity" : "http://example.com/entities",
+        "getEntities" : "http://example.com/entities",
+        "getEntity" : "http://example.com/entities/{{entityId}}",
+        "updateEntity" : "http://example.com/entities/{{entityId}}",
+        "patchEntity" : "http://example.com/entities/{{entityId}}",
+        "deleteEntity" : "http://example.com/entities/{{entityId}}"
       }
       await runTests("crud", testPath, assertions)
     })

package/src/api/controllers/query/import/tests/index.spec.js

@@ -51,30 +51,24 @@ describe("Rest Importer", () => {
     await init(data)
     const info = await restImporter.getInfo()
     expect(info.name).toBe(assertions[key].name)
-    expect(info.url).toBe(assertions[key].url)
   }
 
   it("gets info", async () => {
     const assertions = {
       "oapi2CrudJson" : {
         name: "CRUD",
-        url: "http://example.com"
       },
       "oapi2CrudYaml" : {
         name: "CRUD",
-        url: "http://example.com"
       },
       "oapi2PetstoreJson" : {
         name: "Swagger Petstore",
-        url: "https://petstore.swagger.io/v2"
       },
       "oapi2PetstoreYaml" :{
         name: "Swagger Petstore",
-        url: "https://petstore.swagger.io/v2"
       },
       "curl": {
         name: "example.com",
-        url: "http://example.com"
       }
     }
     await runTest(testGetInfo, assertions)

package/src/api/controllers/query/index.js

@@ -1,4 +1,3 @@
-const { processString } = require("@budibase/string-templates")
 const CouchDB = require("../../../db")
 const {
   generateQueryID,
@@ -9,6 +8,7 @@ const { BaseQueryVerbs } = require("../../../constants")
 const { Thread, ThreadType } = require("../../../threads")
 const { save: saveDatasource } = require("../datasource")
 const { RestImporter } = require("./import")
+const { invalidateDynamicVariables } = require("../../../threads/utils")
 
 const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })
 
@@ -90,47 +90,6 @@ exports.save = async function (ctx) {
   ctx.message = `Query ${query.name} saved successfully.`
 }
 
-async function enrichQueryFields(fields, parameters = {}) {
-  const enrichedQuery = {}
-
-  // enrich the fields with dynamic parameters
-  for (let key of Object.keys(fields)) {
-    if (fields[key] == null) {
-      continue
-    }
-    if (typeof fields[key] === "object") {
-      // enrich nested fields object
-      enrichedQuery[key] = await enrichQueryFields(fields[key], parameters)
-    } else if (typeof fields[key] === "string") {
-      // enrich string value as normal
-      enrichedQuery[key] = await processString(fields[key], parameters, {
-        noHelpers: true,
-      })
-    } else {
-      enrichedQuery[key] = fields[key]
-    }
-  }
-
-  if (
-    enrichedQuery.json ||
-    enrichedQuery.customData ||
-    enrichedQuery.requestBody
-  ) {
-    try {
-      enrichedQuery.json = JSON.parse(
-        enrichedQuery.json ||
-          enrichedQuery.customData ||
-          enrichedQuery.requestBody
-      )
-    } catch (err) {
-      // no json found, ignore
-    }
-    delete enrichedQuery.customData
-  }
-
-  return enrichedQuery
-}
-
 exports.find = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const query = enrichQueries(await db.get(ctx.params.queryId))
@@ -146,16 +105,20 @@ exports.preview = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
   const datasource = await db.get(ctx.request.body.datasourceId)
-
-  const { fields, parameters, queryVerb, transformer } = ctx.request.body
-  const enrichedQuery = await enrichQueryFields(fields, parameters)
+  // preview may not have a queryId as it hasn't been saved, but if it does
+  // this stops dynamic variables from calling the same query
+  const { fields, parameters, queryVerb, transformer, queryId } =
+    ctx.request.body
 
   try {
     const { rows, keys, info, extra } = await Runner.run({
+      appId: ctx.appId,
       datasource,
       queryVerb,
-      query: enrichedQuery,
+      fields,
+      parameters,
       transformer,
+      queryId,
     })
 
     ctx.body = {
@@ -169,33 +132,63 @@ exports.preview = async function (ctx) {
   }
 }
 
-exports.execute = async function (ctx) {
+async function execute(ctx, opts = { rowsOnly: false }) {
   const db = new CouchDB(ctx.appId)
 
   const query = await db.get(ctx.params.queryId)
   const datasource = await db.get(query.datasourceId)
 
-  const enrichedQuery = await enrichQueryFields(
-    query.fields,
-    ctx.request.body.parameters
-  )
-
   // call the relevant CRUD method on the integration class
   try {
-    const { rows } = await Runner.run({
+    const { rows, extra } = await Runner.run({
+      appId: ctx.appId,
       datasource,
       queryVerb: query.queryVerb,
-      query: enrichedQuery,
+      fields: query.fields,
+      parameters: ctx.request.body.parameters,
       transformer: query.transformer,
+      queryId: ctx.params.queryId,
     })
-    ctx.body = rows
+    if (opts && opts.rowsOnly) {
+      ctx.body = rows
+    } else {
+      ctx.body = { data: rows, ...extra }
+    }
   } catch (err) {
     ctx.throw(400, err)
   }
 }
 
+exports.executeV1 = async function (ctx) {
+  return execute(ctx, { rowsOnly: true })
+}
+
+exports.executeV2 = async function (ctx) {
+  return execute(ctx, { rowsOnly: false })
+}
+
+const removeDynamicVariables = async (db, queryId) => {
+  const query = await db.get(queryId)
+  const datasource = await db.get(query.datasourceId)
+  const dynamicVariables = datasource.config.dynamicVariables
+
+  if (dynamicVariables) {
+    // delete dynamic variables from the datasource
+    const newVariables = dynamicVariables.filter(dv => dv.queryId !== queryId)
+    datasource.config.dynamicVariables = newVariables
+    await db.put(datasource)
+
+    // invalidate the deleted variables
+    const variablesToDelete = dynamicVariables.filter(
+      dv => dv.queryId === queryId
+    )
+    await invalidateDynamicVariables(variablesToDelete)
+  }
+}
+
 exports.destroy = async function (ctx) {
   const db = new CouchDB(ctx.appId)
+  await removeDynamicVariables(db, ctx.params.queryId)
   await db.remove(ctx.params.queryId, ctx.params.revId)
   ctx.message = `Query deleted.`
   ctx.status = 200

package/src/api/controllers/query/validation.js

@@ -36,6 +36,7 @@ exports.generateQueryPreviewValidation = () => {
     extra: Joi.object().optional(),
     datasourceId: Joi.string().required(),
     transformer: Joi.string().optional(),
-    parameters: Joi.object({}).required().unknown(true)
+    parameters: Joi.object({}).required().unknown(true),
+    queryId: Joi.string().optional(),
   }))
 }

package/src/api/controllers/row/internal.js

@@ -11,6 +11,7 @@ const {
   inputProcessing,
   outputProcessing,
   processAutoColumn,
+  cleanupAttachments,
 } = require("../../../utilities/rowProcessor")
 const { FieldTypes } = require("../../../constants")
 const { isEqual } = require("lodash")
@@ -25,6 +26,7 @@ const {
   getFromDesignDoc,
   getFromMemoryDoc,
 } = require("../view/utils")
+const { cloneDeep } = require("lodash/fp")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -109,14 +111,14 @@ exports.patch = async ctx => {
   const inputs = ctx.request.body
   const tableId = inputs.tableId
   const isUserTable = tableId === InternalTables.USER_METADATA
-  let dbRow
+  let oldRow
   try {
-    dbRow = await db.get(inputs._id)
+    oldRow = await db.get(inputs._id)
   } catch (err) {
     if (isUserTable) {
       // don't include the rev, it'll be the global rev
       // this time
-      dbRow = {
+      oldRow = {
         _id: inputs._id,
       }
     } else {
@@ -125,13 +127,14 @@ exports.patch = async ctx => {
   }
   let dbTable = await db.get(tableId)
   // need to build up full patch fields before coerce
+  let combinedRow = cloneDeep(oldRow)
   for (let key of Object.keys(inputs)) {
     if (!dbTable.schema[key]) continue
-    dbRow[key] = inputs[key]
+    combinedRow[key] = inputs[key]
   }
 
   // this returns the table and row incase they have been updated
-  let { table, row } = inputProcessing(ctx.user, dbTable, dbRow)
+  let { table, row } = inputProcessing(ctx.user, dbTable, combinedRow)
   const validateResult = await validate({
     row,
     table,
@@ -149,6 +152,8 @@ exports.patch = async ctx => {
     tableId: row.tableId,
     table,
   })
+  // check if any attachments removed
+  await cleanupAttachments(appId, table, { oldRow, row })
 
   if (isUserTable) {
     // the row has been updated, need to put it into the ctx
@@ -295,6 +300,8 @@ exports.destroy = async function (ctx) {
     row,
     tableId: row.tableId,
   })
+  // remove any attachments that were on the row from object storage
+  await cleanupAttachments(appId, table, { row })
 
   let response
   if (ctx.params.tableId === InternalTables.USER_METADATA) {
@@ -341,6 +348,8 @@ exports.bulkDestroy = async ctx => {
   } else {
     await db.bulkDocs(rows.map(row => ({ ...row, _deleted: true })))
   }
+  // remove any attachments that were on the rows from object storage
+  await cleanupAttachments(appId, table, { rows })
   await Promise.all(updates)
   return { response: { ok: true }, rows }
 }

package/src/api/controllers/row/internalSearch.js

@@ -191,7 +191,8 @@ class QueryBuilder {
     }
     if (this.query.equal) {
       build(this.query.equal, (key, value) => {
-        if (!value) {
+        // 0 evaluates to false, which means we would return all rows if we don't check it
+        if (!value && value !== 0) {
           return null
         }
         return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`

package/src/api/controllers/row/utils.js

@@ -50,10 +50,10 @@ exports.validate = async ({ appId, tableId, row, table }) => {
   const errors = {}
   for (let fieldName of Object.keys(table.schema)) {
     const constraints = cloneDeep(table.schema[fieldName].constraints)
+    const type = table.schema[fieldName].type
     // special case for options, need to always allow unselected (null)
     if (
-      table.schema[fieldName].type ===
-        (FieldTypes.OPTIONS || FieldTypes.ARRAY) &&
+      (type === FieldTypes.OPTIONS || type === FieldTypes.ARRAY) &&
       constraints.inclusion
     ) {
       constraints.inclusion.push(null)
@@ -61,17 +61,20 @@ exports.validate = async ({ appId, tableId, row, table }) => {
     let res
 
     // Validate.js doesn't seem to handle array
-    if (
-      table.schema[fieldName].type === FieldTypes.ARRAY &&
-      row[fieldName] &&
-      row[fieldName].length
-    ) {
+    if (type === FieldTypes.ARRAY && row[fieldName] && row[fieldName].length) {
       row[fieldName].map(val => {
         if (!constraints.inclusion.includes(val)) {
           errors[fieldName] = "Field not in list"
         }
       })
-    } else if (table.schema[fieldName].type === FieldTypes.FORMULA) {
+    } else if (type === FieldTypes.JSON && typeof row[fieldName] === "string") {
+      // this should only happen if there is an error
+      try {
+        JSON.parse(row[fieldName])
+      } catch (err) {
+        errors[fieldName] = [`Contains invalid JSON`]
+      }
+    } else if (type === FieldTypes.FORMULA) {
       res = validateJs.single(
         processStringSync(table.schema[fieldName].formula, row),
         constraints

package/src/api/routes/query.js

@@ -41,11 +41,18 @@ router
     authorized(PermissionTypes.QUERY, PermissionLevels.READ),
     queryController.find
   )
+  // DEPRECATED - use new query endpoint for future work
   .post(
     "/api/queries/:queryId",
     paramResource("queryId"),
     authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
-    queryController.execute
+    queryController.executeV1
+  )
+  .post(
+    "/api/v2/queries/:queryId",
+    paramResource("queryId"),
+    authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
+    queryController.executeV2
   )
   .delete(
     "/api/queries/:queryId/:revId",