@budibase/server 0.9.190-alpha.0 → 0.9.190-alpha.12

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.190-alpha.0",
+  "version": "0.9.190-alpha.12",
   "description": "Budibase Web Server",
-  "main": "src/index.js",
+  "main": "src/index.ts",
   "repository": {
     "type": "git",
     "url": "https://github.com/Budibase/budibase.git"
@@ -15,6 +15,7 @@
     "test:watch": "jest --watch",
     "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "build:docs": "node ./scripts/docs/generate.js open",
     "run:docker": "node dist/index.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
@@ -68,9 +69,9 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/auth": "^0.9.190-alpha.0",
-    "@budibase/client": "^0.9.190-alpha.0",
-    "@budibase/string-templates": "^0.9.190-alpha.0",
+    "@budibase/auth": "^0.9.190-alpha.12",
+    "@budibase/client": "^0.9.190-alpha.12",
+    "@budibase/string-templates": "^0.9.190-alpha.12",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -106,7 +107,7 @@
     "mssql": "6.2.3",
     "mysql2": "^2.3.1",
     "node-fetch": "2.6.0",
-    "open": "7.3.0",
+    "open": "^8.4.0",
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",
     "posthog-node": "^1.1.4",
@@ -130,6 +131,7 @@
     "@babel/preset-env": "^7.14.4",
     "@budibase/standard-components": "^0.9.139",
     "@jest/test-sequencer": "^24.8.0",
+    "@types/apidoc": "^0.50.0",
     "@types/bull": "^3.15.1",
     "@types/jest": "^26.0.23",
     "@types/koa": "^2.13.3",
@@ -137,12 +139,14 @@
     "@types/node": "^15.12.4",
     "@types/oracledb": "^5.2.1",
     "@typescript-eslint/parser": "4.28.0",
+    "apidoc": "^0.50.2",
     "babel-jest": "^27.0.2",
     "copyfiles": "^2.4.1",
     "docker-compose": "^0.23.6",
     "eslint": "^6.8.0",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
+    "path-to-regexp": "^6.2.0",
     "prettier": "^2.3.1",
     "rimraf": "^3.0.2",
     "supertest": "^4.0.2",
@@ -154,5 +158,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "58c1927f6a5dcbab760fbdb9d581b1ef1bb9dabf"
+  "gitHead": "4e607feda0a258f486df6fe138940b7faab1db99"
 }

package/scripts/docs/README.md

@@ -0,0 +1,31 @@
+### Documentation
+
+This directory contains the scripts required to generate the APIDoc based documentation.
+You can find the docs about comment structure at the [APIDocs page](https://apidocjs.com/).
+
+In general most API endpoints will look like:
+```js
+  /**
+   * @api {post} /api/:param/url Give it a name 
+   * @apiName Give it a name
+   * @apiGroup group
+   * @apiPermission permission
+   * @apiDescription Describe what the endpoint does, any special cases the user
+   * should be aware of.
+   *
+   * @apiParam {string} param describe a URL parameter.
+   *
+   * @apiParam (Body) input describe a field on the body.
+   *
+   * @apiSuccess {object} output describe the output.
+   */
+```
+
+There are a few key points to note when writing API docs:
+1. Don't use `@apiBody` - this isn't currently supported by our swagger converter.
+2. Make sure to always have an `@api` definition at the start, which must always have the
+HTTP verb, the endpoint URL and the name.
+3. There are three ways you can specify parameters used as inputs for your endpoint,
+`@apiParam` for a URL param, `@apiParam (Body)` for a field on the request body and `@apiParam (Query)`
+for query string parameters.
+4. The `@apiGroup` should be the same for all API Doc comments in a route file.

package/scripts/docs/generate.js

@@ -0,0 +1,74 @@
+const fs = require("fs")
+const { join } = require("path")
+const { createDoc } = require("apidoc")
+const packageJson = require("../../package.json")
+const toSwagger = require("./toSwagger")
+const open = require("open")
+
+const config = {
+  name: "Budibase API",
+  version: packageJson.version,
+  description: "Documenting the Budibase backend API",
+  title: "Budibase app service API",
+}
+
+const shouldOpen = process.argv[2]
+const disallowed = []
+
+function filter(parsedRouteFiles) {
+  const tagToSearch = "url"
+  for (let routeFile of parsedRouteFiles) {
+    for (let route of routeFile) {
+      let routeInfo = route["local"]
+      if (disallowed.includes(routeInfo[tagToSearch])) {
+        const idx = routeFile.indexOf(route)
+        routeFile.splice(idx, 1)
+      }
+    }
+  }
+}
+
+async function generate() {
+  // start by writing a config file
+  const configPath = join(__dirname, "config.json")
+  fs.writeFileSync(configPath, JSON.stringify(config))
+  const mainPath = join(__dirname, "..", "..")
+  const srcPath = join(mainPath, "src", "api", "routes")
+  const assetsPath = join(mainPath, "builder", "assets", "docs")
+  if (!fs.existsSync(assetsPath)) {
+    fs.mkdirSync(assetsPath, { recursive: true })
+  }
+  const options = {
+    src: [srcPath],
+    dest: assetsPath,
+    filters: {
+      main: {
+        postFilter: filter,
+      },
+    },
+    config: configPath,
+  }
+  const doc = createDoc(options)
+  if (typeof doc !== "boolean") {
+    const swagger = toSwagger(JSON.parse(doc.data), JSON.parse(doc.project))
+    fs.writeFileSync(join(assetsPath, "swagger.json"), JSON.stringify(swagger))
+    fs.writeFileSync(join(assetsPath, "apidoc.json"), doc.data)
+    fs.writeFileSync(join(assetsPath, "project.json"), doc.project)
+    console.log(
+      `Docs generated successfully, find in ${assetsPath}, swagger.json, apidoc.json and project.json`
+    )
+  } else {
+    throw "Unable to generate docs."
+  }
+  // delete the temporary config file
+  fs.unlinkSync(configPath)
+  setTimeout(async () => {
+    if (shouldOpen === "open") {
+      await open(join(assetsPath, "index.html"), { wait: false })
+    }
+  }, 2000)
+}
+
+generate().catch(err => {
+  console.error(err)
+})

package/scripts/docs/toSwagger.js

@@ -0,0 +1,320 @@
+let _ = require("lodash")
+let { pathToRegexp } = require("path-to-regexp")
+
+/********************************************************
+ * Based on: https://github.com/fsbahman/apidoc-swagger *
+ ********************************************************/
+
+let swagger = {
+  swagger: "2.0",
+  info: {},
+  paths: {},
+  definitions: {},
+}
+
+function toSwagger(apidocJson, projectJson) {
+  swagger.info = addInfo(projectJson)
+  swagger.paths = extractPaths(apidocJson)
+  return swagger
+}
+
+let tagsRegex = /(<([^>]+)>)/gi
+// Removes <p> </p> tags from text
+function removeTags(text) {
+  return text ? text.replace(tagsRegex, "") : text
+}
+
+function addInfo(projectJson) {
+  let info = {}
+  info["title"] = projectJson.title || projectJson.name
+  info["version"] = projectJson.version
+  info["description"] = projectJson.description
+  return info
+}
+
+/**
+ * Extracts paths provided in json format
+ * post, patch, put request parameters are extracted in body
+ * get and delete are extracted to path parameters
+ * @param apidocJson
+ * @returns {{}}
+ */
+function extractPaths(apidocJson) {
+  let apiPaths = groupByUrl(apidocJson)
+  let paths = {}
+  for (let i = 0; i < apiPaths.length; i++) {
+    let verbs = apiPaths[i].verbs
+    let url = verbs[0].url
+    let pattern = pathToRegexp(url, null)
+    let matches = pattern.exec(url)
+
+    // Surrounds URL parameters with curly brackets -> :email with {email}
+    let pathKeys = []
+    for (let j = 1; j < matches.length; j++) {
+      let key = matches[j].substr(1)
+      url = url.replace(matches[j], "{" + key + "}")
+      pathKeys.push(key)
+    }
+
+    for (let j = 0; j < verbs.length; j++) {
+      let verb = verbs[j]
+      let type = verb.type
+
+      let obj = (paths[url] = paths[url] || {})
+
+      if (type === "post" || type === "patch" || type === "put") {
+        _.extend(
+          obj,
+          createPostPushPutOutput(verb, swagger.definitions, pathKeys)
+        )
+      } else {
+        _.extend(obj, createGetDeleteOutput(verb, swagger.definitions))
+      }
+    }
+  }
+  return paths
+}
+
+function createPostPushPutOutput(verbs, definitions, pathKeys) {
+  let pathItemObject = {}
+  let verbDefinitionResult = createVerbDefinitions(verbs, definitions)
+
+  let params = []
+  let pathParams = createPathParameters(verbs, pathKeys)
+  pathParams = _.filter(pathParams, function (param) {
+    let hasKey = pathKeys.indexOf(param.name) !== -1
+    return !(param.in === "path" && !hasKey)
+  })
+
+  params = params.concat(pathParams)
+  let required =
+    verbs.parameter &&
+    verbs.parameter.fields &&
+    verbs.parameter.fields.Parameter &&
+    verbs.parameter.fields.Parameter.length > 0
+
+  params.push({
+    in: "body",
+    name: "body",
+    description: removeTags(verbs.description),
+    required: required,
+    schema: {
+      $ref: "#/definitions/" + verbDefinitionResult.topLevelParametersRef,
+    },
+  })
+
+  pathItemObject[verbs.type] = {
+    tags: [verbs.group],
+    summary: removeTags(verbs.description),
+    consumes: ["application/json"],
+    produces: ["application/json"],
+    parameters: params,
+  }
+
+  if (verbDefinitionResult.topLevelSuccessRef) {
+    pathItemObject[verbs.type].responses = {
+      200: {
+        description: "successful operation",
+        schema: {
+          type: verbDefinitionResult.topLevelSuccessRefType,
+          items: {
+            $ref: "#/definitions/" + verbDefinitionResult.topLevelSuccessRef,
+          },
+        },
+      },
+    }
+  }
+
+  return pathItemObject
+}
+
+function createVerbDefinitions(verbs, definitions) {
+  let result = {
+    topLevelParametersRef: null,
+    topLevelSuccessRef: null,
+    topLevelSuccessRefType: null,
+  }
+  let defaultObjectName = verbs.name
+
+  let fieldArrayResult = {}
+  if (verbs && verbs.parameter && verbs.parameter.fields) {
+    fieldArrayResult = createFieldArrayDefinitions(
+      verbs.parameter.fields.Parameter,
+      definitions,
+      verbs.name,
+      defaultObjectName
+    )
+    result.topLevelParametersRef = fieldArrayResult.topLevelRef
+  }
+
+  if (verbs && verbs.success && verbs.success.fields) {
+    fieldArrayResult = createFieldArrayDefinitions(
+      verbs.success.fields["Success 200"],
+      definitions,
+      verbs.name,
+      defaultObjectName
+    )
+    result.topLevelSuccessRef = fieldArrayResult.topLevelRef
+    result.topLevelSuccessRefType = fieldArrayResult.topLevelRefType
+  }
+
+  return result
+}
+
+function createFieldArrayDefinitions(
+  fieldArray,
+  definitions,
+  topLevelRef,
+  defaultObjectName
+) {
+  let result = {
+    topLevelRef: topLevelRef,
+    topLevelRefType: null,
+  }
+
+  if (!fieldArray) {
+    return result
+  }
+
+  for (let i = 0; i < fieldArray.length; i++) {
+    let parameter = fieldArray[i]
+
+    let nestedName = createNestedName(parameter.field)
+    let objectName = nestedName.objectName
+    if (!objectName) {
+      objectName = defaultObjectName
+    }
+    let type = parameter.type
+    if (i === 0) {
+      result.topLevelRefType = type
+      if (parameter.type === "Object") {
+        objectName = nestedName.propertyName
+        nestedName.propertyName = null
+      } else if (parameter.type === "Array") {
+        objectName = nestedName.propertyName
+        nestedName.propertyName = null
+        result.topLevelRefType = "array"
+      }
+      result.topLevelRef = objectName
+    }
+
+    definitions[objectName] = definitions[objectName] || {
+      properties: {},
+      required: [],
+    }
+
+    if (nestedName.propertyName) {
+      let prop = {
+        type: (parameter.type || "").toLowerCase(),
+        description: removeTags(parameter.description),
+      }
+      if (parameter.type === "Object") {
+        prop.$ref = "#/definitions/" + parameter.field
+      }
+
+      let typeIndex = type.indexOf("[]")
+      if (typeIndex !== -1 && typeIndex === type.length - 2) {
+        prop.type = "array"
+        prop.items = {
+          type: type.slice(0, type.length - 2),
+        }
+      }
+
+      definitions[objectName]["properties"][nestedName.propertyName] = prop
+      if (!parameter.optional) {
+        let arr = definitions[objectName]["required"]
+        if (arr.indexOf(nestedName.propertyName) === -1) {
+          arr.push(nestedName.propertyName)
+        }
+      }
+    }
+  }
+
+  return result
+}
+
+function createNestedName(field) {
+  let propertyName = field
+  let objectName
+  let propertyNames = field.split(".")
+  if (propertyNames && propertyNames.length > 1) {
+    propertyName = propertyNames[propertyNames.length - 1]
+    propertyNames.pop()
+    objectName = propertyNames.join(".")
+  }
+
+  return {
+    propertyName: propertyName,
+    objectName: objectName,
+  }
+}
+
+/**
+ * Generate get, delete method output
+ * @param verbs
+ * @param definitions
+ * @returns {{}}
+ */
+function createGetDeleteOutput(verbs, definitions) {
+  let pathItemObject = {}
+  verbs.type = verbs.type === "del" ? "delete" : verbs.type
+
+  let verbDefinitionResult = createVerbDefinitions(verbs, definitions)
+  pathItemObject[verbs.type] = {
+    tags: [verbs.group],
+    summary: removeTags(verbs.description),
+    consumes: ["application/json"],
+    produces: ["application/json"],
+    parameters: createPathParameters(verbs),
+  }
+  if (verbDefinitionResult.topLevelSuccessRef) {
+    pathItemObject[verbs.type].responses = {
+      200: {
+        description: "successful operation",
+        schema: {
+          type: verbDefinitionResult.topLevelSuccessRefType,
+          items: {
+            $ref: "#/definitions/" + verbDefinitionResult.topLevelSuccessRef,
+          },
+        },
+      },
+    }
+  }
+  return pathItemObject
+}
+
+/**
+ * Iterate through all method parameters and create array of parameter objects which are stored as path parameters
+ * @param verbs
+ * @returns {Array}
+ */
+function createPathParameters(verbs) {
+  let pathItemObject = []
+  if (verbs.parameter && verbs.parameter.fields.Parameter) {
+    for (let i = 0; i < verbs.parameter.fields.Parameter.length; i++) {
+      let param = verbs.parameter.fields.Parameter[i]
+      let field = param.field
+      let type = param.type
+      pathItemObject.push({
+        name: field,
+        in: type === "file" ? "formData" : "path",
+        required: !param.optional,
+        type: param.type.toLowerCase(),
+        description: removeTags(param.description),
+      })
+    }
+  }
+  return pathItemObject
+}
+
+function groupByUrl(apidocJson) {
+  return _.chain(apidocJson)
+    .groupBy("url")
+    .toPairs()
+    .map(function (element) {
+      return _.zipObject(["url", "verbs"], element)
+    })
+    .value()
+}
+
+module.exports = toSwagger

package/scripts/integrations/oracle/docker-compose.yml

@@ -4,7 +4,7 @@
 version: "3.8"
 services:
   db:
-    container_name: oracle-xe
+    restart: always
     platform: linux/x86_64
     image: container-registry.oracle.com/database/express:18.4.0-xe
     environment:

package/src/api/controllers/application.js

@@ -15,14 +15,12 @@ const {
   generateAppID,
   getLayoutParams,
   getScreenParams,
-  generateScreenID,
   generateDevAppID,
   DocumentTypes,
   AppStatus,
 } = require("../../db/utils")
 const { BUILTIN_ROLE_IDS, AccessController } = require("@budibase/auth/roles")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
-const { createHomeScreen } = require("../../constants/screens")
 const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
 const {
@@ -409,9 +407,5 @@ const createEmptyAppPackage = async (ctx, app) => {
     screensAndLayouts.push(await processObject(cloned, app))
   }
 
-  const homeScreen = createHomeScreen(app)
-  homeScreen._id = generateScreenID()
-  screensAndLayouts.push(homeScreen)
-
   await db.bulkDocs(screensAndLayouts)
 }

package/src/api/controllers/query.js

@@ -1,8 +1,11 @@
 const { processString } = require("@budibase/string-templates")
 const CouchDB = require("../../db")
-const { generateQueryID, getQueryParams } = require("../../db/utils")
+const {
+  generateQueryID,
+  getQueryParams,
+  isProdAppID,
+} = require("../../db/utils")
 const { BaseQueryVerbs } = require("../../constants")
-const env = require("../../environment")
 const { Thread, ThreadType } = require("../../threads")
 
 const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })
@@ -90,10 +93,9 @@ exports.find = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const query = enrichQueries(await db.get(ctx.params.queryId))
   // remove properties that could be dangerous in real app
-  if (env.isProd()) {
+  if (isProdAppID(ctx.appId)) {
     delete query.fields
     delete query.parameters
-    delete query.schema
   }
   ctx.body = query
 }

package/src/api/controllers/row/ExternalRequest.ts

@@ -1,4 +1,5 @@
 import {
+  FilterTypes,
   IncludeRelationships,
   Operation,
   PaginationJson,
@@ -118,8 +119,13 @@ module External {
     }
     // check the row and filters to make sure they aren't a key of some sort
     if (config.filters) {
-      for (let filter of Object.values(config.filters)) {
-        if (typeof filter !== "object" || Object.keys(filter).length === 0) {
+      for (let [key, filter] of Object.entries(config.filters)) {
+        // oneOf is an array, don't iterate it
+        if (
+          typeof filter !== "object" ||
+          Object.keys(filter).length === 0 ||
+          key === FilterTypes.ONE_OF
+        ) {
           continue
         }
         iterateObject(filter)
@@ -197,9 +203,9 @@ module External {
     return row
   }
 
-  function isMany(field: FieldSchema) {
+  function isOneSide(field: FieldSchema) {
     return (
-      field.relationshipType && field.relationshipType.split("-")[0] === "many"
+      field.relationshipType && field.relationshipType.split("-")[0] === "one"
     )
   }
 
@@ -274,25 +280,37 @@ module External {
         const linkTable = this.tables[linkTableName]
         // @ts-ignore
         const linkTablePrimary = linkTable.primary[0]
-        if (!isMany(field)) {
+        // one to many
+        if (isOneSide(field)) {
           newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(
             row[key][0]
           )[0]
-        } else {
+        }
+        // many to many
+        else if (field.through) {
           // we're not inserting a doc, will be a bunch of update calls
-          const isUpdate = !field.through
-          const thisKey: string = isUpdate
-            ? "id"
-            : field.throughTo || linkTablePrimary
-          // @ts-ignore
-          const otherKey: string = isUpdate
-            ? field.fieldName
-            : field.throughFrom || tablePrimary
+          const otherKey: string = field.throughFrom || linkTablePrimary
+          const thisKey: string = field.throughTo || tablePrimary
           row[key].map((relationship: any) => {
-            // we don't really support composite keys for relationships, this is why [0] is used
             manyRelationships.push({
               tableId: field.through || field.tableId,
-              isUpdate,
+              isUpdate: false,
+              key: otherKey,
+              [otherKey]: breakRowIdField(relationship)[0],
+              // leave the ID for enrichment later
+              [thisKey]: `{{ literal ${tablePrimary} }}`,
+            })
+          })
+        }
+        // many to one
+        else {
+          const thisKey: string = "id"
+          // @ts-ignore
+          const otherKey: string = field.fieldName
+          row[key].map((relationship: any) => {
+            manyRelationships.push({
+              tableId: field.tableId,
+              isUpdate: true,
               key: otherKey,
               [thisKey]: breakRowIdField(relationship)[0],
               // leave the ID for enrichment later
@@ -425,8 +443,8 @@ module External {
           )
           definition.through = throughTableName
           // don't support composite keys for relationships
-          definition.from = field.throughFrom || table.primary[0]
-          definition.to = field.throughTo || linkTable.primary[0]
+          definition.from = field.throughTo || table.primary[0]
+          definition.to = field.throughFrom || linkTable.primary[0]
           definition.fromPrimary = table.primary[0]
           definition.toPrimary = linkTable.primary[0]
         }
@@ -448,24 +466,36 @@ module External {
       // make a new request to get the row with all its relationships
       // we need this to work out if any relationships need removed
       for (let field of Object.values(table.schema)) {
-        if (field.type !== FieldTypes.LINK || !field.fieldName) {
+        if (
+          field.type !== FieldTypes.LINK ||
+          !field.fieldName ||
+          isOneSide(field)
+        ) {
           continue
         }
         const isMany = field.relationshipType === RelationshipTypes.MANY_TO_MANY
         const tableId = isMany ? field.through : field.tableId
-        const manyKey = field.throughFrom || primaryKey
+        const { tableName: relatedTableName } = breakExternalTableId(tableId)
+        // @ts-ignore
+        const linkPrimaryKey = this.tables[relatedTableName].primary[0]
+        const manyKey = field.throughTo || primaryKey
+        const lookupField = isMany ? primaryKey : field.foreignKey
         const fieldName = isMany ? manyKey : field.fieldName
+        if (!lookupField || !row[lookupField]) {
+          continue
+        }
         const response = await getDatasourceAndQuery(this.appId, {
           endpoint: getEndpoint(tableId, DataSourceOperation.READ),
           filters: {
             equal: {
-              [fieldName]: row[primaryKey],
+              [fieldName]: row[lookupField],
             },
           },
         })
         // this is the response from knex if no rows found
         const rows = !response[0].read ? response : []
-        related[fieldName] = { rows, isMany, tableId }
+        const storeTo = isMany ? field.throughFrom || linkPrimaryKey : manyKey
+        related[storeTo] = { rows, isMany, tableId }
       }
       return related
     }