@budibase/server 1.0.26 → 1.0.27-alpha.11

package/dist/utilities/fileSystem/utilities.js

@@ -1,5 +1,5 @@
 "use strict";
-const { ObjectStore, makeSureBucketExists, upload, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, } = require("@budibase/auth").objectStore;
+const { ObjectStore, makeSureBucketExists, upload, deleteFiles, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, } = require("@budibase/auth").objectStore;
 /***********************************
  *              NOTE               *
  * This file purely exists so that *
@@ -17,3 +17,4 @@ exports.retrieveToTmp = retrieveToTmp;
 exports.deleteFolder = deleteFolder;
 exports.uploadDirectory = uploadDirectory;
 exports.downloadTarball = downloadTarball;
+exports.deleteFiles = deleteFiles;

package/dist/utilities/index.js

@@ -142,3 +142,13 @@ exports.doesDatabaseExist = (dbName) => __awaiter(void 0, void 0, void 0, functi
         return false;
     }
 });
+exports.formatBytes = bytes => {
+    const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
+    const byteIncrements = 1024;
+    let unit = 0;
+    let size = parseInt(bytes, 10) || 0;
+    while (size >= byteIncrements && ++unit) {
+        size /= byteIncrements;
+    }
+    return `${size.toFixed(size < 10 && unit > 0 ? 1 : 0)}${units[unit]}`;
+};

package/dist/utilities/rowProcessor/index.js

@@ -13,6 +13,10 @@ const { cloneDeep } = require("lodash/fp");
 const { FieldTypes, AutoFieldSubTypes } = require("../../constants");
 const { attachmentsRelativeURL } = require("../index");
 const { processFormulas } = require("./utils");
+const { deleteFiles } = require("../../utilities/fileSystem/utilities");
+const { ObjectStoreBuckets } = require("../../constants");
+const { isProdAppID, getDeployedAppID, dbExists } = require("@budibase/auth/db");
+const CouchDB = require("../../db");
 const BASE_AUTO_ID = 1;
 /**
  * A map of how we convert various properties in rows to each other based on the row type.
@@ -89,7 +93,36 @@ const TYPE_TRANSFORM_MAP = {
     [FieldTypes.AUTO]: {
         parse: () => undefined,
     },
+    [FieldTypes.JSON]: {
+        parse: input => {
+            try {
+                if (input === "") {
+                    return undefined;
+                }
+                return JSON.parse(input);
+            }
+            catch (err) {
+                return input;
+            }
+        },
+    },
 };
+/**
+ * Given the old state of the row and the new one after an update, this will
+ * find the keys that have been removed in the updated row.
+ */
+function getRemovedAttachmentKeys(oldRow, row, attachmentKey) {
+    if (!oldRow[attachmentKey]) {
+        return [];
+    }
+    const oldKeys = oldRow[attachmentKey].map(attachment => attachment.key);
+    // no attachments in new row, all removed
+    if (!row[attachmentKey]) {
+        return oldKeys;
+    }
+    const newKeys = row[attachmentKey].map(attachment => attachment.key);
+    return oldKeys.filter(key => newKeys.indexOf(key) === -1);
+}
 /**
  * This will update any auto columns that are found on the row/table with the correct information based on
  * time now and the current logged in user making the request.
@@ -240,3 +273,46 @@ exports.outputProcessing = (ctx, table, rows, opts = { squash: true }) => __awai
     }
     return wasArray ? enriched : enriched[0];
 });
+/**
+ * Clean up any attachments that were attached to a row.
+ * @param {string} appId The ID of the app from which a row is being deleted.
+ * @param {object} table The table from which a row is being removed.
+ * @param {any} row optional - the row being removed.
+ * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
+ * @param {any} oldRow optional - if updating a row this will determine the difference.
+ * @return {Promise<void>} When all attachments have been removed this will return.
+ */
+exports.cleanupAttachments = (appId, table, { row, rows, oldRow }) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!isProdAppID(appId)) {
+        const prodAppId = getDeployedAppID(appId);
+        // if prod exists, then don't allow deleting
+        const exists = yield dbExists(CouchDB, prodAppId);
+        if (exists) {
+            return;
+        }
+    }
+    let files = [];
+    function addFiles(row, key) {
+        if (row[key]) {
+            files = files.concat(row[key].map(attachment => attachment.key));
+        }
+    }
+    for (let [key, schema] of Object.entries(table.schema)) {
+        if (schema.type !== FieldTypes.ATTACHMENT) {
+            continue;
+        }
+        // if updating, need to manage the differences
+        if (oldRow && row) {
+            files = files.concat(getRemovedAttachmentKeys(oldRow, row, key));
+        }
+        else if (row) {
+            addFiles(row, key);
+        }
+        else if (rows) {
+            rows.forEach(row => addFiles(row, key));
+        }
+    }
+    if (files.length > 0) {
+        return deleteFiles(ObjectStoreBuckets.APPS, files);
+    }
+});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.26",
+  "version": "1.0.27-alpha.11",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -69,9 +69,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/auth": "^1.0.26",
-    "@budibase/client": "^1.0.26",
-    "@budibase/string-templates": "^1.0.26",
+    "@apidevtools/swagger-parser": "^10.0.3",
+    "@budibase/auth": "^1.0.27-alpha.11",
+    "@budibase/client": "^1.0.27-alpha.11",
+    "@budibase/string-templates": "^1.0.27-alpha.11",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -85,12 +86,15 @@
     "bull": "^3.22.4",
     "chmodr": "1.2.0",
     "csvtojson": "2.0.10",
+    "curlconverter": "^3.21.0",
     "dotenv": "8.2.0",
     "download": "8.0.0",
     "fix-path": "3.0.0",
+    "form-data": "^4.0.0",
     "fs-extra": "8.1.0",
     "jimp": "0.16.1",
     "joi": "17.2.1",
+    "js-yaml": "^4.1.0",
     "jsonschema": "1.4.0",
     "knex": "^0.95.6",
     "koa": "2.7.0",
@@ -118,11 +122,13 @@
     "pouchdb-replication-stream": "1.2.9",
     "server-destroy": "1.0.1",
     "svelte": "^3.38.2",
+    "swagger-parser": "^10.0.3",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
     "vm2": "^3.9.3",
     "worker-farm": "^1.7.0",
+    "xml2js": "^0.4.23",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -146,6 +152,7 @@
     "eslint": "^6.8.0",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
+    "openapi-types": "^9.3.1",
     "path-to-regexp": "^6.2.0",
     "prettier": "^2.3.1",
     "rimraf": "^3.0.2",
@@ -158,5 +165,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "6ebc345b6523efdd0ac77ea4c7e392918aff0757"
+  "gitHead": "45631b235ddba52942183e312b9154fbdb4ea862"
 }

package/src/api/controllers/automation.js

@@ -5,11 +5,15 @@ const { getAutomationParams, generateAutomationID } = require("../../db/utils")
 const {
   checkForWebhooks,
   updateTestHistory,
+  removeDeprecated,
 } = require("../../automations/utils")
 const { deleteEntityMetadata } = require("../../utilities")
 const { MetadataTypes } = require("../../constants")
 const { setTestFlag, clearTestFlag } = require("../../utilities/redis")
 
+const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
+const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
+
 /*************************
  *                       *
  *   BUILDER FUNCTIONS   *
@@ -155,17 +159,17 @@ exports.destroy = async function (ctx) {
 }
 
 exports.getActionList = async function (ctx) {
-  ctx.body = actions.ACTION_DEFINITIONS
+  ctx.body = ACTION_DEFS
 }
 
 exports.getTriggerList = async function (ctx) {
-  ctx.body = triggers.TRIGGER_DEFINITIONS
+  ctx.body = TRIGGER_DEFS
 }
 
 module.exports.getDefinitionList = async function (ctx) {
   ctx.body = {
-    trigger: triggers.TRIGGER_DEFINITIONS,
-    action: actions.ACTION_DEFINITIONS,
+    trigger: TRIGGER_DEFS,
+    action: ACTION_DEFS,
   }
 }
 

package/src/api/controllers/datasource.js

@@ -10,6 +10,7 @@ const {
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
 const { integrations } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
+const { invalidateDynamicVariables } = require("../../threads/utils")
 
 exports.fetch = async function (ctx) {
   const database = new CouchDB(ctx.appId)
@@ -57,10 +58,43 @@ exports.buildSchemaFromDb = async function (ctx) {
   ctx.body = response
 }
 
+/**
+ * Check for variables that have been updated or removed and invalidate them.
+ */
+const invalidateVariables = async (existingDatasource, updatedDatasource) => {
+  const existingVariables = existingDatasource.config.dynamicVariables
+  const updatedVariables = updatedDatasource.config.dynamicVariables
+  const toInvalidate = []
+
+  if (!existingVariables) {
+    return
+  }
+
+  if (!updatedVariables) {
+    // invalidate all
+    toInvalidate.push(...existingVariables)
+  } else {
+    // invaldate changed / removed
+    existingVariables.forEach(existing => {
+      const unchanged = updatedVariables.find(
+        updated =>
+          existing.name === updated.name &&
+          existing.queryId === updated.queryId &&
+          existing.value === updated.value
+      )
+      if (!unchanged) {
+        toInvalidate.push(existing)
+      }
+    })
+  }
+  await invalidateDynamicVariables(toInvalidate)
+}
+
 exports.update = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const datasourceId = ctx.params.datasourceId
   let datasource = await db.get(datasourceId)
+  await invalidateVariables(datasource, ctx.request.body)
   datasource = { ...datasource, ...ctx.request.body }
 
   const response = await db.put(datasource)

package/src/api/controllers/query/import/index.ts

@@ -0,0 +1,85 @@
+import CouchDB from "../../../../db"
+import { queryValidation } from "../validation"
+import { generateQueryID } from "../../../../db/utils"
+import { ImportInfo, ImportSource } from "./sources/base"
+import { OpenAPI2 } from "./sources/openapi2"
+import { Query } from "./../../../../definitions/common"
+import { Curl } from "./sources/curl"
+interface ImportResult {
+  errorQueries: Query[]
+  queries: Query[]
+}
+
+export class RestImporter {
+  data: string
+  sources: ImportSource[]
+  source!: ImportSource
+
+  constructor(data: string) {
+    this.data = data
+    this.sources = [new OpenAPI2(), new Curl()]
+  }
+
+  init = async () => {
+    for (let source of this.sources) {
+      if (await source.isSupported(this.data)) {
+        this.source = source
+        break
+      }
+    }
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    return this.source.getInfo()
+  }
+
+  importQueries = async (
+    appId: string,
+    datasourceId: string
+  ): Promise<ImportResult> => {
+    // constuct the queries
+    let queries = await this.source.getQueries(datasourceId)
+
+    // validate queries
+    const errorQueries: Query[] = []
+    const schema = queryValidation()
+    queries = queries
+      .filter(query => {
+        const validation = schema.validate(query)
+        if (validation.error) {
+          errorQueries.push(query)
+          return false
+        }
+        return true
+      })
+      .map(query => {
+        query._id = generateQueryID(query.datasourceId)
+        return query
+      })
+
+    // persist queries
+    const db = new CouchDB(appId)
+    const response = await db.bulkDocs(queries)
+
+    // create index to seperate queries and errors
+    const queryIndex = queries.reduce((acc, query) => {
+      if (query._id) {
+        acc[query._id] = query
+      }
+      return acc
+    }, {} as { [key: string]: Query })
+
+    // check for failed writes
+    response.forEach((query: any) => {
+      if (!query.ok) {
+        errorQueries.push(queryIndex[query.id])
+        delete queryIndex[query.id]
+      }
+    })
+
+    return {
+      errorQueries,
+      queries: Object.values(queryIndex),
+    }
+  }
+}

package/src/api/controllers/query/import/sources/base/index.ts

@@ -0,0 +1,87 @@
+import { Query, QueryParameter } from "../../../../../../definitions/datasource"
+import { URL } from "url"
+
+export interface ImportInfo {
+  name: string
+}
+
+enum MethodToVerb {
+  get = "read",
+  post = "create",
+  put = "update",
+  patch = "patch",
+  delete = "delete",
+}
+
+export abstract class ImportSource {
+  abstract isSupported(data: string): Promise<boolean>
+  abstract getInfo(): Promise<ImportInfo>
+  abstract getQueries(datasourceId: string): Promise<Query[]>
+
+  constructQuery = (
+    datasourceId: string,
+    name: string,
+    method: string,
+    path: string,
+    url: URL,
+    queryString: string,
+    headers: object = {},
+    parameters: QueryParameter[] = [],
+    body: object | undefined = undefined
+  ): Query => {
+    const readable = true
+    const queryVerb = this.verbFromMethod(method)
+    const transformer = "return data"
+    const schema = {}
+    path = this.processPath(path)
+    path = `${url.origin}/${path}`
+    queryString = this.processQuery(queryString)
+    const requestBody = JSON.stringify(body, null, 2)
+
+    const query: Query = {
+      datasourceId,
+      name,
+      parameters,
+      fields: {
+        headers,
+        queryString,
+        path,
+        requestBody,
+      },
+      transformer,
+      schema,
+      readable,
+      queryVerb,
+    }
+
+    return query
+  }
+
+  verbFromMethod = (method: string) => {
+    const verb = (<any>MethodToVerb)[method]
+    if (!verb) {
+      throw new Error(`Unsupported method: ${method}`)
+    }
+    return verb
+  }
+
+  processPath = (path: string): string => {
+    if (path?.startsWith("/")) {
+      path = path.substring(1)
+    }
+
+    // add extra braces around params for binding
+    path = path.replace(/[{]/g, "{{")
+    path = path.replace(/[}]/g, "}}")
+
+    return path
+  }
+
+  processQuery = (queryString: string): string => {
+    if (queryString?.startsWith("?")) {
+      return queryString.substring(1)
+    }
+
+    return queryString
+  }
+}

package/src/api/controllers/query/import/sources/base/openapi.ts

@@ -0,0 +1,24 @@
+import { ImportSource } from "."
+import SwaggerParser from "@apidevtools/swagger-parser"
+import { OpenAPI } from "openapi-types"
+const yaml = require("js-yaml")
+
+export abstract class OpenAPISource extends ImportSource {
+  parseData = async (data: string): Promise<OpenAPI.Document> => {
+    let json: OpenAPI.Document
+    try {
+      json = JSON.parse(data)
+    } catch (jsonErr) {
+      // couldn't parse json
+      // try to convert yaml -> json
+      try {
+        json = yaml.load(data)
+      } catch (yamlErr) {
+        // couldn't parse yaml
+        throw new Error("Could not parse JSON or YAML")
+      }
+    }
+
+    return SwaggerParser.validate(json, {})
+  }
+}

package/src/api/controllers/query/import/sources/curl.ts

@@ -0,0 +1,102 @@
+import { ImportSource, ImportInfo } from "./base"
+import { Query } from "../../../../../definitions/common"
+import { URL } from "url"
+const curlconverter = require("curlconverter")
+
+const parseCurl = (data: string): any => {
+  const curlJson = curlconverter.toJsonString(data)
+  return JSON.parse(curlJson)
+}
+
+/**
+ * The curl converter parses the request body into the key field of an object
+ * e.g. --d '{"key":"val"}' produces an object { "{"key":"val"}" : "" }
+ * This is not what we want, so we need to parse out the key from the object
+ */
+const parseBody = (curl: any) => {
+  if (curl.data) {
+    const keys = Object.keys(curl.data)
+    if (keys.length) {
+      let key = keys[0]
+      try {
+        // filter out the dollar syntax used by curl for shell support
+        if (key.startsWith("$")) {
+          key = key.substring(1)
+        }
+        return JSON.parse(key)
+      } catch (e) {
+        // do nothing
+      }
+    }
+  }
+  return undefined
+}
+
+const parseCookie = (curl: any) => {
+  if (curl.cookies) {
+    return Object.entries(curl.cookies).reduce((acc, entry) => {
+      const [key, value] = entry
+      return acc + `${key}=${value}; `
+    }, "")
+  }
+
+  return null
+}
+
+/**
+ * Curl
+ * https://curl.se/docs/manpage.html
+ */
+export class Curl extends ImportSource {
+  curl: any
+
+  isSupported = async (data: string): Promise<boolean> => {
+    try {
+      const curl = parseCurl(data)
+      this.curl = curl
+    } catch (err) {
+      return false
+    }
+    return true
+  }
+
+  getUrl = (): URL => {
+    return new URL(this.curl.raw_url)
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    const url = this.getUrl()
+    return {
+      name: url.hostname,
+    }
+  }
+
+  getQueries = async (datasourceId: string): Promise<Query[]> => {
+    const url = this.getUrl()
+    const name = url.pathname
+    const path = url.pathname
+    const method = this.curl.method
+    const queryString = url.search
+    const headers = this.curl.headers
+    const requestBody = parseBody(this.curl)
+
+    const cookieHeader = parseCookie(this.curl)
+    if (cookieHeader) {
+      headers["Cookie"] = cookieHeader
+    }
+
+    const query = this.constructQuery(
+      datasourceId,
+      name,
+      method,
+      path,
+      url,
+      queryString,
+      headers,
+      [],
+      requestBody
+    )
+
+    return [query]
+  }
+}