@budibase/server 0.9.190-alpha.7 → 1.0.1

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.190-alpha.7",
+  "version": "1.0.1",
   "description": "Budibase Web Server",
-  "main": "src/index.js",
+  "main": "src/index.ts",
   "repository": {
     "type": "git",
     "url": "https://github.com/Budibase/budibase.git"
@@ -15,6 +15,7 @@
     "test:watch": "jest --watch",
     "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "build:docs": "node ./scripts/docs/generate.js open",
     "run:docker": "node dist/index.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
@@ -68,9 +69,9 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/auth": "^0.9.190-alpha.7",
-    "@budibase/client": "^0.9.190-alpha.7",
-    "@budibase/string-templates": "^0.9.190-alpha.7",
+    "@budibase/auth": "^1.0.1",
+    "@budibase/client": "^1.0.1",
+    "@budibase/string-templates": "^1.0.1",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -106,7 +107,7 @@
     "mssql": "6.2.3",
     "mysql2": "^2.3.1",
     "node-fetch": "2.6.0",
-    "open": "7.3.0",
+    "open": "^8.4.0",
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",
     "posthog-node": "^1.1.4",
@@ -130,6 +131,7 @@
     "@babel/preset-env": "^7.14.4",
     "@budibase/standard-components": "^0.9.139",
     "@jest/test-sequencer": "^24.8.0",
+    "@types/apidoc": "^0.50.0",
     "@types/bull": "^3.15.1",
     "@types/jest": "^26.0.23",
     "@types/koa": "^2.13.3",
@@ -137,12 +139,14 @@
     "@types/node": "^15.12.4",
     "@types/oracledb": "^5.2.1",
     "@typescript-eslint/parser": "4.28.0",
+    "apidoc": "^0.50.2",
     "babel-jest": "^27.0.2",
     "copyfiles": "^2.4.1",
     "docker-compose": "^0.23.6",
     "eslint": "^6.8.0",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
+    "path-to-regexp": "^6.2.0",
     "prettier": "^2.3.1",
     "rimraf": "^3.0.2",
     "supertest": "^4.0.2",
@@ -154,5 +158,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "c5c9206ab2703a604cf6553d0e865cc457a1474f"
+  "gitHead": "b349e1c27fd9d23c0b3f4251f53b07b5c2ef7590"
 }

package/scripts/docs/README.md

@@ -0,0 +1,31 @@
+### Documentation
+
+This directory contains the scripts required to generate the APIDoc based documentation.
+You can find the docs about comment structure at the [APIDocs page](https://apidocjs.com/).
+
+In general most API endpoints will look like:
+```js
+  /**
+   * @api {post} /api/:param/url Give it a name 
+   * @apiName Give it a name
+   * @apiGroup group
+   * @apiPermission permission
+   * @apiDescription Describe what the endpoint does, any special cases the user
+   * should be aware of.
+   *
+   * @apiParam {string} param describe a URL parameter.
+   *
+   * @apiParam (Body) input describe a field on the body.
+   *
+   * @apiSuccess {object} output describe the output.
+   */
+```
+
+There are a few key points to note when writing API docs:
+1. Don't use `@apiBody` - this isn't currently supported by our swagger converter.
+2. Make sure to always have an `@api` definition at the start, which must always have the
+HTTP verb, the endpoint URL and the name.
+3. There are three ways you can specify parameters used as inputs for your endpoint,
+`@apiParam` for a URL param, `@apiParam (Body)` for a field on the request body and `@apiParam (Query)`
+for query string parameters.
+4. The `@apiGroup` should be the same for all API Doc comments in a route file.

package/scripts/docs/generate.js

@@ -0,0 +1,74 @@
+const fs = require("fs")
+const { join } = require("path")
+const { createDoc } = require("apidoc")
+const packageJson = require("../../package.json")
+const toSwagger = require("./toSwagger")
+const open = require("open")
+
+const config = {
+  name: "Budibase API",
+  version: packageJson.version,
+  description: "Documenting the Budibase backend API",
+  title: "Budibase app service API",
+}
+
+const shouldOpen = process.argv[2]
+const disallowed = []
+
+function filter(parsedRouteFiles) {
+  const tagToSearch = "url"
+  for (let routeFile of parsedRouteFiles) {
+    for (let route of routeFile) {
+      let routeInfo = route["local"]
+      if (disallowed.includes(routeInfo[tagToSearch])) {
+        const idx = routeFile.indexOf(route)
+        routeFile.splice(idx, 1)
+      }
+    }
+  }
+}
+
+async function generate() {
+  // start by writing a config file
+  const configPath = join(__dirname, "config.json")
+  fs.writeFileSync(configPath, JSON.stringify(config))
+  const mainPath = join(__dirname, "..", "..")
+  const srcPath = join(mainPath, "src", "api", "routes")
+  const assetsPath = join(mainPath, "builder", "assets", "docs")
+  if (!fs.existsSync(assetsPath)) {
+    fs.mkdirSync(assetsPath, { recursive: true })
+  }
+  const options = {
+    src: [srcPath],
+    dest: assetsPath,
+    filters: {
+      main: {
+        postFilter: filter,
+      },
+    },
+    config: configPath,
+  }
+  const doc = createDoc(options)
+  if (typeof doc !== "boolean") {
+    const swagger = toSwagger(JSON.parse(doc.data), JSON.parse(doc.project))
+    fs.writeFileSync(join(assetsPath, "swagger.json"), JSON.stringify(swagger))
+    fs.writeFileSync(join(assetsPath, "apidoc.json"), doc.data)
+    fs.writeFileSync(join(assetsPath, "project.json"), doc.project)
+    console.log(
+      `Docs generated successfully, find in ${assetsPath}, swagger.json, apidoc.json and project.json`
+    )
+  } else {
+    throw "Unable to generate docs."
+  }
+  // delete the temporary config file
+  fs.unlinkSync(configPath)
+  setTimeout(async () => {
+    if (shouldOpen === "open") {
+      await open(join(assetsPath, "index.html"), { wait: false })
+    }
+  }, 2000)
+}
+
+generate().catch(err => {
+  console.error(err)
+})

package/scripts/docs/toSwagger.js

@@ -0,0 +1,320 @@
+let _ = require("lodash")
+let { pathToRegexp } = require("path-to-regexp")
+
+/********************************************************
+ * Based on: https://github.com/fsbahman/apidoc-swagger *
+ ********************************************************/
+
+let swagger = {
+  swagger: "2.0",
+  info: {},
+  paths: {},
+  definitions: {},
+}
+
+function toSwagger(apidocJson, projectJson) {
+  swagger.info = addInfo(projectJson)
+  swagger.paths = extractPaths(apidocJson)
+  return swagger
+}
+
+let tagsRegex = /(<([^>]+)>)/gi
+// Removes <p> </p> tags from text
+function removeTags(text) {
+  return text ? text.replace(tagsRegex, "") : text
+}
+
+function addInfo(projectJson) {
+  let info = {}
+  info["title"] = projectJson.title || projectJson.name
+  info["version"] = projectJson.version
+  info["description"] = projectJson.description
+  return info
+}
+
+/**
+ * Extracts paths provided in json format
+ * post, patch, put request parameters are extracted in body
+ * get and delete are extracted to path parameters
+ * @param apidocJson
+ * @returns {{}}
+ */
+function extractPaths(apidocJson) {
+  let apiPaths = groupByUrl(apidocJson)
+  let paths = {}
+  for (let i = 0; i < apiPaths.length; i++) {
+    let verbs = apiPaths[i].verbs
+    let url = verbs[0].url
+    let pattern = pathToRegexp(url, null)
+    let matches = pattern.exec(url)
+
+    // Surrounds URL parameters with curly brackets -> :email with {email}
+    let pathKeys = []
+    for (let j = 1; j < matches.length; j++) {
+      let key = matches[j].substr(1)
+      url = url.replace(matches[j], "{" + key + "}")
+      pathKeys.push(key)
+    }
+
+    for (let j = 0; j < verbs.length; j++) {
+      let verb = verbs[j]
+      let type = verb.type
+
+      let obj = (paths[url] = paths[url] || {})
+
+      if (type === "post" || type === "patch" || type === "put") {
+        _.extend(
+          obj,
+          createPostPushPutOutput(verb, swagger.definitions, pathKeys)
+        )
+      } else {
+        _.extend(obj, createGetDeleteOutput(verb, swagger.definitions))
+      }
+    }
+  }
+  return paths
+}
+
+function createPostPushPutOutput(verbs, definitions, pathKeys) {
+  let pathItemObject = {}
+  let verbDefinitionResult = createVerbDefinitions(verbs, definitions)
+
+  let params = []
+  let pathParams = createPathParameters(verbs, pathKeys)
+  pathParams = _.filter(pathParams, function (param) {
+    let hasKey = pathKeys.indexOf(param.name) !== -1
+    return !(param.in === "path" && !hasKey)
+  })
+
+  params = params.concat(pathParams)
+  let required =
+    verbs.parameter &&
+    verbs.parameter.fields &&
+    verbs.parameter.fields.Parameter &&
+    verbs.parameter.fields.Parameter.length > 0
+
+  params.push({
+    in: "body",
+    name: "body",
+    description: removeTags(verbs.description),
+    required: required,
+    schema: {
+      $ref: "#/definitions/" + verbDefinitionResult.topLevelParametersRef,
+    },
+  })
+
+  pathItemObject[verbs.type] = {
+    tags: [verbs.group],
+    summary: removeTags(verbs.description),
+    consumes: ["application/json"],
+    produces: ["application/json"],
+    parameters: params,
+  }
+
+  if (verbDefinitionResult.topLevelSuccessRef) {
+    pathItemObject[verbs.type].responses = {
+      200: {
+        description: "successful operation",
+        schema: {
+          type: verbDefinitionResult.topLevelSuccessRefType,
+          items: {
+            $ref: "#/definitions/" + verbDefinitionResult.topLevelSuccessRef,
+          },
+        },
+      },
+    }
+  }
+
+  return pathItemObject
+}
+
+function createVerbDefinitions(verbs, definitions) {
+  let result = {
+    topLevelParametersRef: null,
+    topLevelSuccessRef: null,
+    topLevelSuccessRefType: null,
+  }
+  let defaultObjectName = verbs.name
+
+  let fieldArrayResult = {}
+  if (verbs && verbs.parameter && verbs.parameter.fields) {
+    fieldArrayResult = createFieldArrayDefinitions(
+      verbs.parameter.fields.Parameter,
+      definitions,
+      verbs.name,
+      defaultObjectName
+    )
+    result.topLevelParametersRef = fieldArrayResult.topLevelRef
+  }
+
+  if (verbs && verbs.success && verbs.success.fields) {
+    fieldArrayResult = createFieldArrayDefinitions(
+      verbs.success.fields["Success 200"],
+      definitions,
+      verbs.name,
+      defaultObjectName
+    )
+    result.topLevelSuccessRef = fieldArrayResult.topLevelRef
+    result.topLevelSuccessRefType = fieldArrayResult.topLevelRefType
+  }
+
+  return result
+}
+
+function createFieldArrayDefinitions(
+  fieldArray,
+  definitions,
+  topLevelRef,
+  defaultObjectName
+) {
+  let result = {
+    topLevelRef: topLevelRef,
+    topLevelRefType: null,
+  }
+
+  if (!fieldArray) {
+    return result
+  }
+
+  for (let i = 0; i < fieldArray.length; i++) {
+    let parameter = fieldArray[i]
+
+    let nestedName = createNestedName(parameter.field)
+    let objectName = nestedName.objectName
+    if (!objectName) {
+      objectName = defaultObjectName
+    }
+    let type = parameter.type
+    if (i === 0) {
+      result.topLevelRefType = type
+      if (parameter.type === "Object") {
+        objectName = nestedName.propertyName
+        nestedName.propertyName = null
+      } else if (parameter.type === "Array") {
+        objectName = nestedName.propertyName
+        nestedName.propertyName = null
+        result.topLevelRefType = "array"
+      }
+      result.topLevelRef = objectName
+    }
+
+    definitions[objectName] = definitions[objectName] || {
+      properties: {},
+      required: [],
+    }
+
+    if (nestedName.propertyName) {
+      let prop = {
+        type: (parameter.type || "").toLowerCase(),
+        description: removeTags(parameter.description),
+      }
+      if (parameter.type === "Object") {
+        prop.$ref = "#/definitions/" + parameter.field
+      }
+
+      let typeIndex = type.indexOf("[]")
+      if (typeIndex !== -1 && typeIndex === type.length - 2) {
+        prop.type = "array"
+        prop.items = {
+          type: type.slice(0, type.length - 2),
+        }
+      }
+
+      definitions[objectName]["properties"][nestedName.propertyName] = prop
+      if (!parameter.optional) {
+        let arr = definitions[objectName]["required"]
+        if (arr.indexOf(nestedName.propertyName) === -1) {
+          arr.push(nestedName.propertyName)
+        }
+      }
+    }
+  }
+
+  return result
+}
+
+function createNestedName(field) {
+  let propertyName = field
+  let objectName
+  let propertyNames = field.split(".")
+  if (propertyNames && propertyNames.length > 1) {
+    propertyName = propertyNames[propertyNames.length - 1]
+    propertyNames.pop()
+    objectName = propertyNames.join(".")
+  }
+
+  return {
+    propertyName: propertyName,
+    objectName: objectName,
+  }
+}
+
+/**
+ * Generate get, delete method output
+ * @param verbs
+ * @param definitions
+ * @returns {{}}
+ */
+function createGetDeleteOutput(verbs, definitions) {
+  let pathItemObject = {}
+  verbs.type = verbs.type === "del" ? "delete" : verbs.type
+
+  let verbDefinitionResult = createVerbDefinitions(verbs, definitions)
+  pathItemObject[verbs.type] = {
+    tags: [verbs.group],
+    summary: removeTags(verbs.description),
+    consumes: ["application/json"],
+    produces: ["application/json"],
+    parameters: createPathParameters(verbs),
+  }
+  if (verbDefinitionResult.topLevelSuccessRef) {
+    pathItemObject[verbs.type].responses = {
+      200: {
+        description: "successful operation",
+        schema: {
+          type: verbDefinitionResult.topLevelSuccessRefType,
+          items: {
+            $ref: "#/definitions/" + verbDefinitionResult.topLevelSuccessRef,
+          },
+        },
+      },
+    }
+  }
+  return pathItemObject
+}
+
+/**
+ * Iterate through all method parameters and create array of parameter objects which are stored as path parameters
+ * @param verbs
+ * @returns {Array}
+ */
+function createPathParameters(verbs) {
+  let pathItemObject = []
+  if (verbs.parameter && verbs.parameter.fields.Parameter) {
+    for (let i = 0; i < verbs.parameter.fields.Parameter.length; i++) {
+      let param = verbs.parameter.fields.Parameter[i]
+      let field = param.field
+      let type = param.type
+      pathItemObject.push({
+        name: field,
+        in: type === "file" ? "formData" : "path",
+        required: !param.optional,
+        type: param.type.toLowerCase(),
+        description: removeTags(param.description),
+      })
+    }
+  }
+  return pathItemObject
+}
+
+function groupByUrl(apidocJson) {
+  return _.chain(apidocJson)
+    .groupBy("url")
+    .toPairs()
+    .map(function (element) {
+      return _.zipObject(["url", "verbs"], element)
+    })
+    .value()
+}
+
+module.exports = toSwagger

package/scripts/integrations/oracle/docker-compose.yml

@@ -4,7 +4,7 @@
 version: "3.8"
 services:
   db:
-    container_name: oracle-xe
+    restart: always
     platform: linux/x86_64
     image: container-registry.oracle.com/database/express:18.4.0-xe
     environment:

package/src/api/controllers/row/ExternalRequest.ts

@@ -1,4 +1,5 @@
 import {
+  FilterTypes,
   IncludeRelationships,
   Operation,
   PaginationJson,
@@ -118,8 +119,13 @@ module External {
     }
     // check the row and filters to make sure they aren't a key of some sort
     if (config.filters) {
-      for (let filter of Object.values(config.filters)) {
-        if (typeof filter !== "object" || Object.keys(filter).length === 0) {
+      for (let [key, filter] of Object.entries(config.filters)) {
+        // oneOf is an array, don't iterate it
+        if (
+          typeof filter !== "object" ||
+          Object.keys(filter).length === 0 ||
+          key === FilterTypes.ONE_OF
+        ) {
           continue
         }
         iterateObject(filter)

package/src/api/controllers/table/internal.js

@@ -44,14 +44,10 @@ exports.save = async function (ctx) {
   // the column if you want to change the type
   if (oldTable && oldTable.schema) {
     for (let propKey of Object.keys(tableToSave.schema)) {
-      let column = tableToSave.schema[propKey]
       let oldColumn = oldTable.schema[propKey]
       if (oldColumn && oldColumn.type === "internal") {
         oldColumn.type = "auto"
       }
-      if (oldColumn && oldColumn.type !== column.type) {
-        ctx.throw(400, "Cannot change the type of a column")
-      }
     }
   }