@budibase/server 1.0.25 → 1.0.27-alpha.10

package/dist/utilities/fileSystem/utilities.js

@@ -1,5 +1,5 @@
 "use strict";
-const { ObjectStore, makeSureBucketExists, upload, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, } = require("@budibase/auth").objectStore;
+const { ObjectStore, makeSureBucketExists, upload, deleteFiles, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, } = require("@budibase/auth").objectStore;
 /***********************************
  *              NOTE               *
  * This file purely exists so that *
@@ -17,3 +17,4 @@ exports.retrieveToTmp = retrieveToTmp;
 exports.deleteFolder = deleteFolder;
 exports.uploadDirectory = uploadDirectory;
 exports.downloadTarball = downloadTarball;
+exports.deleteFiles = deleteFiles;

package/dist/utilities/index.js

@@ -142,3 +142,13 @@ exports.doesDatabaseExist = (dbName) => __awaiter(void 0, void 0, void 0, functi
         return false;
     }
 });
+exports.formatBytes = bytes => {
+    const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
+    const byteIncrements = 1024;
+    let unit = 0;
+    let size = parseInt(bytes, 10) || 0;
+    while (size >= byteIncrements && ++unit) {
+        size /= byteIncrements;
+    }
+    return `${size.toFixed(size < 10 && unit > 0 ? 1 : 0)}${units[unit]}`;
+};

package/dist/utilities/rowProcessor/index.js

@@ -13,6 +13,10 @@ const { cloneDeep } = require("lodash/fp");
 const { FieldTypes, AutoFieldSubTypes } = require("../../constants");
 const { attachmentsRelativeURL } = require("../index");
 const { processFormulas } = require("./utils");
+const { deleteFiles } = require("../../utilities/fileSystem/utilities");
+const { ObjectStoreBuckets } = require("../../constants");
+const { isProdAppID, getDeployedAppID, dbExists } = require("@budibase/auth/db");
+const CouchDB = require("../../db");
 const BASE_AUTO_ID = 1;
 /**
  * A map of how we convert various properties in rows to each other based on the row type.
@@ -89,7 +93,36 @@ const TYPE_TRANSFORM_MAP = {
     [FieldTypes.AUTO]: {
         parse: () => undefined,
     },
+    [FieldTypes.JSON]: {
+        parse: input => {
+            try {
+                if (input === "") {
+                    return undefined;
+                }
+                return JSON.parse(input);
+            }
+            catch (err) {
+                return input;
+            }
+        },
+    },
 };
+/**
+ * Given the old state of the row and the new one after an update, this will
+ * find the keys that have been removed in the updated row.
+ */
+function getRemovedAttachmentKeys(oldRow, row, attachmentKey) {
+    if (!oldRow[attachmentKey]) {
+        return [];
+    }
+    const oldKeys = oldRow[attachmentKey].map(attachment => attachment.key);
+    // no attachments in new row, all removed
+    if (!row[attachmentKey]) {
+        return oldKeys;
+    }
+    const newKeys = row[attachmentKey].map(attachment => attachment.key);
+    return oldKeys.filter(key => newKeys.indexOf(key) === -1);
+}
 /**
  * This will update any auto columns that are found on the row/table with the correct information based on
  * time now and the current logged in user making the request.
@@ -240,3 +273,46 @@ exports.outputProcessing = (ctx, table, rows, opts = { squash: true }) => __awai
     }
     return wasArray ? enriched : enriched[0];
 });
+/**
+ * Clean up any attachments that were attached to a row.
+ * @param {string} appId The ID of the app from which a row is being deleted.
+ * @param {object} table The table from which a row is being removed.
+ * @param {any} row optional - the row being removed.
+ * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
+ * @param {any} oldRow optional - if updating a row this will determine the difference.
+ * @return {Promise<void>} When all attachments have been removed this will return.
+ */
+exports.cleanupAttachments = (appId, table, { row, rows, oldRow }) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!isProdAppID(appId)) {
+        const prodAppId = getDeployedAppID(appId);
+        // if prod exists, then don't allow deleting
+        const exists = yield dbExists(CouchDB, prodAppId);
+        if (exists) {
+            return;
+        }
+    }
+    let files = [];
+    function addFiles(row, key) {
+        if (row[key]) {
+            files = files.concat(row[key].map(attachment => attachment.key));
+        }
+    }
+    for (let [key, schema] of Object.entries(table.schema)) {
+        if (schema.type !== FieldTypes.ATTACHMENT) {
+            continue;
+        }
+        // if updating, need to manage the differences
+        if (oldRow && row) {
+            files = files.concat(getRemovedAttachmentKeys(oldRow, row, key));
+        }
+        else if (row) {
+            addFiles(row, key);
+        }
+        else if (rows) {
+            rows.forEach(row => addFiles(row, key));
+        }
+    }
+    if (files.length > 0) {
+        return deleteFiles(ObjectStoreBuckets.APPS, files);
+    }
+});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.25",
+  "version": "1.0.27-alpha.10",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -69,9 +69,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/auth": "^1.0.25",
-    "@budibase/client": "^1.0.25",
-    "@budibase/string-templates": "^1.0.25",
+    "@apidevtools/swagger-parser": "^10.0.3",
+    "@budibase/auth": "^1.0.27-alpha.10",
+    "@budibase/client": "^1.0.27-alpha.10",
+    "@budibase/string-templates": "^1.0.27-alpha.10",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -85,12 +86,15 @@
     "bull": "^3.22.4",
     "chmodr": "1.2.0",
     "csvtojson": "2.0.10",
+    "curlconverter": "^3.21.0",
     "dotenv": "8.2.0",
     "download": "8.0.0",
     "fix-path": "3.0.0",
+    "form-data": "^4.0.0",
     "fs-extra": "8.1.0",
     "jimp": "0.16.1",
     "joi": "17.2.1",
+    "js-yaml": "^4.1.0",
     "jsonschema": "1.4.0",
     "knex": "^0.95.6",
     "koa": "2.7.0",
@@ -118,11 +122,13 @@
     "pouchdb-replication-stream": "1.2.9",
     "server-destroy": "1.0.1",
     "svelte": "^3.38.2",
+    "swagger-parser": "^10.0.3",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
     "vm2": "^3.9.3",
     "worker-farm": "^1.7.0",
+    "xml2js": "^0.4.23",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -146,6 +152,7 @@
     "eslint": "^6.8.0",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
+    "openapi-types": "^9.3.1",
     "path-to-regexp": "^6.2.0",
     "prettier": "^2.3.1",
     "rimraf": "^3.0.2",
@@ -158,5 +165,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "382b86689a90ef561bd06d9020ee3d1400052d19"
+  "gitHead": "37b3e81a0d6f99566d3b753f79b91d5b05185106"
 }

package/src/api/controllers/automation.js

@@ -5,11 +5,15 @@ const { getAutomationParams, generateAutomationID } = require("../../db/utils")
 const {
   checkForWebhooks,
   updateTestHistory,
+  removeDeprecated,
 } = require("../../automations/utils")
 const { deleteEntityMetadata } = require("../../utilities")
 const { MetadataTypes } = require("../../constants")
 const { setTestFlag, clearTestFlag } = require("../../utilities/redis")
 
+const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
+const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
+
 /*************************
  *                       *
  *   BUILDER FUNCTIONS   *
@@ -155,17 +159,17 @@ exports.destroy = async function (ctx) {
 }
 
 exports.getActionList = async function (ctx) {
-  ctx.body = actions.ACTION_DEFINITIONS
+  ctx.body = ACTION_DEFS
 }
 
 exports.getTriggerList = async function (ctx) {
-  ctx.body = triggers.TRIGGER_DEFINITIONS
+  ctx.body = TRIGGER_DEFS
 }
 
 module.exports.getDefinitionList = async function (ctx) {
   ctx.body = {
-    trigger: triggers.TRIGGER_DEFINITIONS,
-    action: actions.ACTION_DEFINITIONS,
+    trigger: TRIGGER_DEFS,
+    action: ACTION_DEFS,
   }
 }
 

package/src/api/controllers/query/import/index.ts

@@ -0,0 +1,85 @@
+import CouchDB from "../../../../db"
+import { queryValidation } from "../validation"
+import { generateQueryID } from "../../../../db/utils"
+import { ImportInfo, ImportSource } from "./sources/base"
+import { OpenAPI2 } from "./sources/openapi2"
+import { Query } from "./../../../../definitions/common"
+import { Curl } from "./sources/curl"
+interface ImportResult {
+  errorQueries: Query[]
+  queries: Query[]
+}
+
+export class RestImporter {
+  data: string
+  sources: ImportSource[]
+  source!: ImportSource
+
+  constructor(data: string) {
+    this.data = data
+    this.sources = [new OpenAPI2(), new Curl()]
+  }
+
+  init = async () => {
+    for (let source of this.sources) {
+      if (await source.isSupported(this.data)) {
+        this.source = source
+        break
+      }
+    }
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    return this.source.getInfo()
+  }
+
+  importQueries = async (
+    appId: string,
+    datasourceId: string
+  ): Promise<ImportResult> => {
+    // constuct the queries
+    let queries = await this.source.getQueries(datasourceId)
+
+    // validate queries
+    const errorQueries: Query[] = []
+    const schema = queryValidation()
+    queries = queries
+      .filter(query => {
+        const validation = schema.validate(query)
+        if (validation.error) {
+          errorQueries.push(query)
+          return false
+        }
+        return true
+      })
+      .map(query => {
+        query._id = generateQueryID(query.datasourceId)
+        return query
+      })
+
+    // persist queries
+    const db = new CouchDB(appId)
+    const response = await db.bulkDocs(queries)
+
+    // create index to seperate queries and errors
+    const queryIndex = queries.reduce((acc, query) => {
+      if (query._id) {
+        acc[query._id] = query
+      }
+      return acc
+    }, {} as { [key: string]: Query })
+
+    // check for failed writes
+    response.forEach((query: any) => {
+      if (!query.ok) {
+        errorQueries.push(queryIndex[query.id])
+        delete queryIndex[query.id]
+      }
+    })
+
+    return {
+      errorQueries,
+      queries: Object.values(queryIndex),
+    }
+  }
+}

package/src/api/controllers/query/import/sources/base/index.ts

@@ -0,0 +1,87 @@
+import { Query, QueryParameter } from "../../../../../../definitions/datasource"
+import { URL } from "url"
+
+export interface ImportInfo {
+  name: string
+}
+
+enum MethodToVerb {
+  get = "read",
+  post = "create",
+  put = "update",
+  patch = "patch",
+  delete = "delete",
+}
+
+export abstract class ImportSource {
+  abstract isSupported(data: string): Promise<boolean>
+  abstract getInfo(): Promise<ImportInfo>
+  abstract getQueries(datasourceId: string): Promise<Query[]>
+
+  constructQuery = (
+    datasourceId: string,
+    name: string,
+    method: string,
+    path: string,
+    url: URL,
+    queryString: string,
+    headers: object = {},
+    parameters: QueryParameter[] = [],
+    body: object | undefined = undefined
+  ): Query => {
+    const readable = true
+    const queryVerb = this.verbFromMethod(method)
+    const transformer = "return data"
+    const schema = {}
+    path = this.processPath(path)
+    path = `${url.origin}/${path}`
+    queryString = this.processQuery(queryString)
+    const requestBody = JSON.stringify(body, null, 2)
+
+    const query: Query = {
+      datasourceId,
+      name,
+      parameters,
+      fields: {
+        headers,
+        queryString,
+        path,
+        requestBody,
+      },
+      transformer,
+      schema,
+      readable,
+      queryVerb,
+    }
+
+    return query
+  }
+
+  verbFromMethod = (method: string) => {
+    const verb = (<any>MethodToVerb)[method]
+    if (!verb) {
+      throw new Error(`Unsupported method: ${method}`)
+    }
+    return verb
+  }
+
+  processPath = (path: string): string => {
+    if (path?.startsWith("/")) {
+      path = path.substring(1)
+    }
+
+    // add extra braces around params for binding
+    path = path.replace(/[{]/g, "{{")
+    path = path.replace(/[}]/g, "}}")
+
+    return path
+  }
+
+  processQuery = (queryString: string): string => {
+    if (queryString?.startsWith("?")) {
+      return queryString.substring(1)
+    }
+
+    return queryString
+  }
+}

package/src/api/controllers/query/import/sources/base/openapi.ts

@@ -0,0 +1,24 @@
+import { ImportSource } from "."
+import SwaggerParser from "@apidevtools/swagger-parser"
+import { OpenAPI } from "openapi-types"
+const yaml = require("js-yaml")
+
+export abstract class OpenAPISource extends ImportSource {
+  parseData = async (data: string): Promise<OpenAPI.Document> => {
+    let json: OpenAPI.Document
+    try {
+      json = JSON.parse(data)
+    } catch (jsonErr) {
+      // couldn't parse json
+      // try to convert yaml -> json
+      try {
+        json = yaml.load(data)
+      } catch (yamlErr) {
+        // couldn't parse yaml
+        throw new Error("Could not parse JSON or YAML")
+      }
+    }
+
+    return SwaggerParser.validate(json, {})
+  }
+}

package/src/api/controllers/query/import/sources/curl.ts

@@ -0,0 +1,102 @@
+import { ImportSource, ImportInfo } from "./base"
+import { Query } from "../../../../../definitions/common"
+import { URL } from "url"
+const curlconverter = require("curlconverter")
+
+const parseCurl = (data: string): any => {
+  const curlJson = curlconverter.toJsonString(data)
+  return JSON.parse(curlJson)
+}
+
+/**
+ * The curl converter parses the request body into the key field of an object
+ * e.g. --d '{"key":"val"}' produces an object { "{"key":"val"}" : "" }
+ * This is not what we want, so we need to parse out the key from the object
+ */
+const parseBody = (curl: any) => {
+  if (curl.data) {
+    const keys = Object.keys(curl.data)
+    if (keys.length) {
+      let key = keys[0]
+      try {
+        // filter out the dollar syntax used by curl for shell support
+        if (key.startsWith("$")) {
+          key = key.substring(1)
+        }
+        return JSON.parse(key)
+      } catch (e) {
+        // do nothing
+      }
+    }
+  }
+  return undefined
+}
+
+const parseCookie = (curl: any) => {
+  if (curl.cookies) {
+    return Object.entries(curl.cookies).reduce((acc, entry) => {
+      const [key, value] = entry
+      return acc + `${key}=${value}; `
+    }, "")
+  }
+
+  return null
+}
+
+/**
+ * Curl
+ * https://curl.se/docs/manpage.html
+ */
+export class Curl extends ImportSource {
+  curl: any
+
+  isSupported = async (data: string): Promise<boolean> => {
+    try {
+      const curl = parseCurl(data)
+      this.curl = curl
+    } catch (err) {
+      return false
+    }
+    return true
+  }
+
+  getUrl = (): URL => {
+    return new URL(this.curl.raw_url)
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    const url = this.getUrl()
+    return {
+      name: url.hostname,
+    }
+  }
+
+  getQueries = async (datasourceId: string): Promise<Query[]> => {
+    const url = this.getUrl()
+    const name = url.pathname
+    const path = url.pathname
+    const method = this.curl.method
+    const queryString = url.search
+    const headers = this.curl.headers
+    const requestBody = parseBody(this.curl)
+
+    const cookieHeader = parseCookie(this.curl)
+    if (cookieHeader) {
+      headers["Cookie"] = cookieHeader
+    }
+
+    const query = this.constructQuery(
+      datasourceId,
+      name,
+      method,
+      path,
+      url,
+      queryString,
+      headers,
+      [],
+      requestBody
+    )
+
+    return [query]
+  }
+}

package/src/api/controllers/query/import/sources/openapi2.ts

@@ -0,0 +1,163 @@
+import { ImportInfo } from "./base"
+import { Query, QueryParameter } from "../../../../../definitions/datasource"
+import { OpenAPIV2 } from "openapi-types"
+import { OpenAPISource } from "./base/openapi"
+import { URL } from "url"
+
+const parameterNotRef = (
+  param: OpenAPIV2.Parameter | OpenAPIV2.ReferenceObject
+): param is OpenAPIV2.Parameter => {
+  // all refs are deferenced by parser library
+  return true
+}
+
+const isOpenAPI2 = (document: any): document is OpenAPIV2.Document => {
+  if (document.swagger === "2.0") {
+    return true
+  } else {
+    return false
+  }
+}
+
+const methods: string[] = Object.values(OpenAPIV2.HttpMethods)
+
+const isOperation = (
+  key: string,
+  pathItem: any
+): pathItem is OpenAPIV2.OperationObject => {
+  return methods.includes(key)
+}
+
+const isParameter = (
+  key: string,
+  pathItem: any
+): pathItem is OpenAPIV2.Parameter => {
+  return !isOperation(key, pathItem)
+}
+
+/**
+ * OpenAPI Version 2.0 - aka "Swagger"
+ * https://github.com/OAI/OpenAPI-Specification/blob/main/versions/2.0.md
+ */
+export class OpenAPI2 extends OpenAPISource {
+  document!: OpenAPIV2.Document
+
+  isSupported = async (data: string): Promise<boolean> => {
+    try {
+      const document: any = await this.parseData(data)
+      if (isOpenAPI2(document)) {
+        this.document = document
+        return true
+      } else {
+        return false
+      }
+    } catch (err) {
+      return false
+    }
+  }
+
+  getUrl = (): URL => {
+    const scheme = this.document.schemes?.includes("https") ? "https" : "http"
+    const basePath = this.document.basePath || ""
+    const host = this.document.host || "<host>"
+    return new URL(`${scheme}://${host}${basePath}`)
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    const name = this.document.info.title || "Swagger Import"
+    return {
+      name,
+    }
+  }
+
+  getQueries = async (datasourceId: string): Promise<Query[]> => {
+    const url = this.getUrl()
+    const queries = []
+
+    for (let [path, pathItem] of Object.entries(this.document.paths)) {
+      // parameters that apply to every operation in the path
+      let pathParams: OpenAPIV2.Parameter[] = []
+
+      for (let [key, opOrParams] of Object.entries(pathItem)) {
+        if (isParameter(key, opOrParams)) {
+          const pathParameters = opOrParams as OpenAPIV2.Parameter[]
+          pathParams.push(...pathParameters)
+          continue
+        }
+        // can not be a parameter, must be an operation
+        const operation = opOrParams as OpenAPIV2.OperationObject
+
+        const methodName = key
+        const name = operation.operationId || path
+        let queryString = ""
+        const headers: any = {}
+        let requestBody = undefined
+        const parameters: QueryParameter[] = []
+
+        if (operation.consumes) {
+          headers["Content-Type"] = operation.consumes[0]
+        }
+
+        // combine the path parameters with the operation parameters
+        const operationParams = operation.parameters || []
+        const allParams = [...pathParams, ...operationParams]
+
+        for (let param of allParams) {
+          if (parameterNotRef(param)) {
+            switch (param.in) {
+              case "query":
+                let prefix = ""
+                if (queryString) {
+                  prefix = "&"
+                }
+                queryString = `${queryString}${prefix}${param.name}={{${param.name}}}`
+                break
+              case "header":
+                headers[param.name] = `{{${param.name}}}`
+                break
+              case "path":
+                // do nothing: param is already in the path
+                break
+              case "formData":
+                // future enhancement
+                break
+              case "body":
+                // set the request body to the example provided
+                // future enhancement: generate an example from the schema
+                let bodyParam: OpenAPIV2.InBodyParameterObject =
+                  param as OpenAPIV2.InBodyParameterObject
+                if (param.schema.example) {
+                  const schema = bodyParam.schema as OpenAPIV2.SchemaObject
+                  requestBody = schema.example
+                }
+                break
+            }
+
+            // add the parameter if it can be bound in our config
+            if (["query", "header", "path"].includes(param.in)) {
+              parameters.push({
+                name: param.name,
+                default: param.default || "",
+              })
+            }
+          }
+        }
+
+        const query = this.constructQuery(
+          datasourceId,
+          name,
+          methodName,
+          path,
+          url,
+          queryString,
+          headers,
+          parameters,
+          requestBody
+        )
+        queries.push(query)
+      }
+    }
+
+    return queries
+  }
+}