@budibase/server 0.0.32 → 0.0.999-alpha.30

package/package.json

@@ -1,45 +1,180 @@
 {
   "name": "@budibase/server",
-  "version": "0.0.32",
+  "email": "hi@budibase.com",
+  "version": "0.0.999-alpha.30",
   "description": "Budibase Web Server",
-  "main": "index.js",
+  "main": "src/index.ts",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Budibase/budibase.git"
+  },
   "scripts": {
-    "test": "jest apps && jest builder",
-    "build": "cd appPackages/_master && yarn && cd ../testApp && yarn && cd ../testApp2 && yarn",
-    "initialise": "node ./initialise/initialiseBudibase init -d ./myapps -c contributors -u admin -p admin",
-    "budi": "node ../cli/bin/budi",
-    "dev:builder": "node index --enable-source-maps"
+    "prebuild": "rimraf dist/",
+    "build": "tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/",
+    "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
+    "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
+    "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
+    "test": "bash scripts/test.sh",
+    "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
+    "test:watch": "jest --watch",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
+    "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "build:docs": "node ./scripts/docs/generate.js open",
+    "run:docker": "node dist/index.js",
+    "run:docker:cluster": "pm2-runtime start pm2.config.js",
+    "dev:stack:up": "node scripts/dev/manage.js up",
+    "dev:stack:down": "node scripts/dev/manage.js down",
+    "dev:stack:nuke": "node scripts/dev/manage.js nuke",
+    "dev:builder": "yarn run dev:stack:up && nodemon",
+    "dev:built": "yarn run dev:stack:up && yarn run run:docker",
+    "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
+    "initialise": "node scripts/initialise.js",
+    "env:multi:enable": "node scripts/multiTenancy.js enable",
+    "env:multi:disable": "node scripts/multiTenancy.js disable",
+    "env:selfhost:enable": "node scripts/selfhost.js enable",
+    "env:selfhost:disable": "node scripts/selfhost.js disable",
+    "env:localdomain:enable": "node scripts/localdomain.js enable",
+    "env:localdomain:disable": "node scripts/localdomain.js disable",
+    "env:account:enable": "node scripts/account.js enable",
+    "env:account:disable": "node scripts/account.js disable"
   },
   "keywords": [
     "budibase"
   ],
-  "author": "Michael Shanks",
-  "license": "AGPL-3.0-or-later",
+  "author": "Budibase",
+  "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/client": "^0.0.32",
-    "@budibase/core": "^0.0.32",
-    "@koa/router": "^8.0.0",
-    "fs-extra": "^8.1.0",
-    "koa": "^2.7.0",
-    "koa-body": "^4.1.0",
-    "koa-send": "^5.0.0",
-    "koa-session": "^5.12.0",
-    "koa-static": "^5.0.0",
-    "lodash": "^4.17.13",
-    "squirrelly": "^7.5.0",
-    "tar-fs": "^2.0.0",
-    "uuid": "^3.3.2",
-    "yargs": "^13.2.4"
+    "@apidevtools/swagger-parser": "10.0.3",
+    "@budibase/backend-core": "0.0.999-alpha.30",
+    "@budibase/client": "0.0.999-alpha.30",
+    "@budibase/pro": "0.0.999-alpha.30",
+    "@budibase/shared-core": "0.0.999-alpha.30",
+    "@budibase/string-templates": "0.0.999-alpha.30",
+    "@budibase/types": "0.0.999-alpha.30",
+    "@bull-board/api": "3.7.0",
+    "@bull-board/koa": "3.9.4",
+    "@elastic/elasticsearch": "7.10.0",
+    "@google-cloud/firestore": "5.0.2",
+    "@koa/router": "8.0.8",
+    "@sendgrid/mail": "7.1.1",
+    "@sentry/node": "6.17.7",
+    "airtable": "0.10.1",
+    "arangojs": "7.2.0",
+    "aws-sdk": "2.1030.0",
+    "bcryptjs": "2.4.3",
+    "bull": "4.10.1",
+    "chmodr": "1.2.0",
+    "chokidar": "3.5.3",
+    "cookies": "0.8.0",
+    "csvtojson": "2.0.10",
+    "curlconverter": "3.21.0",
+    "dd-trace": "3.13.2",
+    "dotenv": "8.2.0",
+    "download": "8.0.0",
+    "elastic-apm-node": "3.38.0",
+    "fix-path": "3.0.0",
+    "form-data": "4.0.0",
+    "fs-extra": "8.1.0",
+    "global-agent": "3.0.0",
+    "google-auth-library": "7.12.0",
+    "google-spreadsheet": "3.2.0",
+    "jimp": "0.16.1",
+    "joi": "17.6.0",
+    "js-yaml": "4.1.0",
+    "jsonschema": "1.4.0",
+    "knex": "2.4.0",
+    "koa": "2.13.4",
+    "koa-body": "4.2.0",
+    "koa-compress": "4.0.1",
+    "koa-connect": "2.1.0",
+    "koa-send": "5.0.0",
+    "koa-session": "5.12.0",
+    "koa-static": "5.0.0",
+    "koa-useragent": "^4.1.0",
+    "koa2-ratelimit": "1.1.1",
+    "lodash": "4.17.21",
+    "memorystream": "0.3.1",
+    "mongodb": "4.9",
+    "mssql": "6.2.3",
+    "mysql2": "2.3.3",
+    "node-fetch": "2.6.7",
+    "open": "8.4.0",
+    "pg": "8.5.1",
+    "posthog-node": "1.3.0",
+    "pouchdb": "7.3.0",
+    "pouchdb-adapter-memory": "7.2.2",
+    "pouchdb-all-dbs": "1.0.2",
+    "pouchdb-find": "7.2.2",
+    "pouchdb-replication-stream": "1.2.9",
+    "redis": "4",
+    "server-destroy": "1.0.1",
+    "snowflake-promise": "^4.5.0",
+    "socket.io": "4.6.1",
+    "svelte": "3.49.0",
+    "swagger-parser": "10.0.3",
+    "tar": "6.1.11",
+    "to-json-schema": "0.2.5",
+    "uuid": "3.3.2",
+    "validate.js": "0.13.1",
+    "vm2": "3.9.16",
+    "worker-farm": "1.7.0",
+    "xml2js": "0.5.0",
+    "yargs": "13.2.4"
   },
   "devDependencies": {
-    "@jest/test-sequencer": "^24.8.0",
-    "jest": "^24.8.0",
-    "nodemon": "^2.0.2",
-    "server-destroy": "^1.0.1",
-    "supertest": "^4.0.2"
+    "@babel/core": "7.17.4",
+    "@babel/preset-env": "7.16.11",
+    "@budibase/standard-components": "^0.9.139",
+    "@jest/test-sequencer": "29.5.0",
+    "@swc/core": "^1.3.25",
+    "@swc/jest": "^0.2.24",
+    "@trendyol/jest-testcontainers": "^2.1.1",
+    "@types/apidoc": "0.50.0",
+    "@types/bson": "4.2.0",
+    "@types/global-agent": "2.1.1",
+    "@types/google-spreadsheet": "3.1.5",
+    "@types/ioredis": "4.28.10",
+    "@types/jest": "29.5.0",
+    "@types/koa": "2.13.4",
+    "@types/koa__router": "8.0.8",
+    "@types/lodash": "4.14.180",
+    "@types/node": "14.18.20",
+    "@types/node-fetch": "2.6.1",
+    "@types/oracledb": "5.2.2",
+    "@types/pouchdb": "6.4.0",
+    "@types/redis": "4.0.11",
+    "@types/server-destroy": "1.0.1",
+    "@types/supertest": "2.0.12",
+    "@types/tar": "6.1.3",
+    "@typescript-eslint/parser": "5.45.0",
+    "apidoc": "0.50.4",
+    "babel-jest": "29.5.0",
+    "copyfiles": "2.4.1",
+    "docker-compose": "0.23.17",
+    "eslint": "6.8.0",
+    "ioredis-mock": "7.2.0",
+    "is-wsl": "2.2.0",
+    "jest": "29.5.0",
+    "jest-openapi": "0.14.2",
+    "jest-runner": "29.5.0",
+    "jest-serial-runner": "^1.2.1",
+    "nodemon": "2.0.15",
+    "openapi-types": "9.3.1",
+    "openapi-typescript": "5.2.0",
+    "path-to-regexp": "6.2.0",
+    "prettier": "2.5.1",
+    "rimraf": "3.0.2",
+    "supertest": "6.2.2",
+    "swagger-jsdoc": "6.1.0",
+    "timekeeper": "2.2.0",
+    "ts-jest": "29.0.5",
+    "ts-node": "10.8.1",
+    "tsconfig-paths": "4.0.0",
+    "typescript": "4.7.3",
+    "update-dotenv": "1.1.1"
   },
-  "jest": {
-    "testEnvironment": "node"
+  "optionalDependencies": {
+    "oracledb": "5.3.0"
   },
-  "gitHead": "b1f4f90927d9e494e513220ef060af28d2d42455"
+  "gitHead": "107657a4f410801f80746f2cb526515d115ec5f2"
 }

package/pm2.config.js

@@ -0,0 +1,9 @@
+module.exports = {
+  apps: [
+    {
+      script: "dist/index.js",
+      instances: "max",
+      exec_mode: "cluster",
+    },
+  ],
+}

package/scripts/account.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+const updateDotEnv = require("update-dotenv")
+
+const arg = process.argv.slice(2)[0]
+
+updateDotEnv({
+  DISABLE_ACCOUNT_PORTAL: arg === "enable" ? "" : "1",
+}).then(() => console.log("Updated server!"))

package/scripts/dev/manage.js

@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+const compose = require("docker-compose")
+const path = require("path")
+const fs = require("fs")
+
+// This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands.
+const CONFIG = {
+  cwd: path.resolve(process.cwd(), "../../hosting"),
+  config: "docker-compose.dev.yaml",
+  log: true,
+}
+
+const Commands = {
+  Up: "up",
+  Down: "down",
+  Nuke: "nuke",
+}
+
+async function init() {
+  const envFilePath = path.join(process.cwd(), ".env")
+  if (!fs.existsSync(envFilePath)) {
+    const envFileJson = {
+      PORT: 4001,
+      MINIO_URL: "http://localhost:4004",
+      COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
+      REDIS_URL: "localhost:6379",
+      WORKER_URL: "http://localhost:4002",
+      INTERNAL_API_KEY: "budibase",
+      ACCOUNT_PORTAL_URL: "http://localhost:10001",
+      ACCOUNT_PORTAL_API_KEY: "budibase",
+      JWT_SECRET: "testsecret",
+      ENCRYPTION_KEY: "testsecret",
+      REDIS_PASSWORD: "budibase",
+      MINIO_ACCESS_KEY: "budibase",
+      MINIO_SECRET_KEY: "budibase",
+      COUCH_DB_PASSWORD: "budibase",
+      COUCH_DB_USER: "budibase",
+      SELF_HOSTED: 1,
+      DISABLE_ACCOUNT_PORTAL: 1,
+      MULTI_TENANCY: "",
+      DISABLE_THREADING: 1,
+      SERVICE: "app-service",
+      DEPLOYMENT_ENVIRONMENT: "development",
+      BB_ADMIN_USER_EMAIL: "",
+      BB_ADMIN_USER_PASSWORD: "",
+      PLUGINS_DIR: "",
+      TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
+      HTTP_MIGRATIONS: "0",
+      HTTP_LOGGING: "0",
+    }
+    let envFile = ""
+    Object.keys(envFileJson).forEach(key => {
+      envFile += `${key}=${envFileJson[key]}\n`
+    })
+    fs.writeFileSync(envFilePath, envFile)
+  }
+}
+
+async function up() {
+  console.log("Spinning up your budibase dev environment... 🔧✨")
+  await init()
+  await compose.upAll(CONFIG)
+
+  // We always ensure to restart the proxy service in case of nginx conf changes
+  await compose.restartOne("proxy-service", CONFIG)
+}
+
+async function down() {
+  console.log("Spinning down your budibase dev environment... 🌇")
+  await compose.stop(CONFIG)
+}
+
+async function nuke() {
+  console.log(
+    "Clearing down your budibase dev environment, including all containers and volumes... 💥"
+  )
+  await compose.down({
+    ...CONFIG,
+    // stop containers, delete volumes
+    commandOptions: ["-v", "--remove-orphans"],
+  })
+}
+
+const managementCommand = process.argv.slice(2)[0]
+
+if (
+  !managementCommand ||
+  !Object.values(Commands).some(command => managementCommand === command)
+) {
+  throw new Error(
+    "You must supply either an 'up', 'down' or 'nuke' commmand to manage the budibase development environment."
+  )
+}
+
+let command
+switch (managementCommand) {
+  case Commands.Up:
+    command = up
+    break
+  case Commands.Down:
+    command = down
+    break
+  case Commands.Nuke:
+    command = nuke
+    break
+  default:
+    command = up
+}
+
+command()
+  .then(() => {
+    console.log("Done! 🎉")
+  })
+  .catch(err => {
+    console.error(
+      "Something went wrong while managing budibase dev environment:",
+      err.message
+    )
+  })

package/scripts/docs/README.md

@@ -0,0 +1,31 @@
+### Documentation
+
+This directory contains the scripts required to generate the APIDoc based documentation.
+You can find the docs about comment structure at the [APIDocs page](https://apidocjs.com/).
+
+In general most API endpoints will look like:
+```js
+  /**
+   * @api {post} /api/:param/url Give it a name 
+   * @apiName Give it a name
+   * @apiGroup group
+   * @apiPermission permission
+   * @apiDescription Describe what the endpoint does, any special cases the user
+   * should be aware of.
+   *
+   * @apiParam {string} param describe a URL parameter.
+   *
+   * @apiParam (Body) input describe a field on the body.
+   *
+   * @apiSuccess {object} output describe the output.
+   */
+```
+
+There are a few key points to note when writing API docs:
+1. Don't use `@apiBody` - this isn't currently supported by our swagger converter.
+2. Make sure to always have an `@api` definition at the start, which must always have the
+HTTP verb, the endpoint URL and the name.
+3. There are three ways you can specify parameters used as inputs for your endpoint,
+`@apiParam` for a URL param, `@apiParam (Body)` for a field on the request body and `@apiParam (Query)`
+for query string parameters.
+4. The `@apiGroup` should be the same for all API Doc comments in a route file.

package/scripts/docs/generate.js

@@ -0,0 +1,74 @@
+const fs = require("fs")
+const { join } = require("path")
+const { createDoc } = require("apidoc")
+const packageJson = require("../../package.json")
+const toSwagger = require("./toSwagger")
+const open = require("open")
+
+const config = {
+  name: "Budibase API",
+  version: packageJson.version,
+  description: "Documenting the Budibase backend API",
+  title: "Budibase app service API",
+}
+
+const shouldOpen = process.argv[2]
+const disallowed = []
+
+function filter(parsedRouteFiles) {
+  const tagToSearch = "url"
+  for (let routeFile of parsedRouteFiles) {
+    for (let route of routeFile) {
+      let routeInfo = route["local"]
+      if (disallowed.includes(routeInfo[tagToSearch])) {
+        const idx = routeFile.indexOf(route)
+        routeFile.splice(idx, 1)
+      }
+    }
+  }
+}
+
+async function generate() {
+  // start by writing a config file
+  const configPath = join(__dirname, "config.json")
+  fs.writeFileSync(configPath, JSON.stringify(config))
+  const mainPath = join(__dirname, "..", "..")
+  const srcPath = join(mainPath, "src", "api", "routes")
+  const assetsPath = join(mainPath, "builder", "assets", "docs")
+  if (!fs.existsSync(assetsPath)) {
+    fs.mkdirSync(assetsPath, { recursive: true })
+  }
+  const options = {
+    src: [srcPath],
+    dest: assetsPath,
+    filters: {
+      main: {
+        postFilter: filter,
+      },
+    },
+    config: configPath,
+  }
+  const doc = createDoc(options)
+  if (typeof doc !== "boolean") {
+    const swagger = toSwagger(JSON.parse(doc.data), JSON.parse(doc.project))
+    fs.writeFileSync(join(assetsPath, "swagger.json"), JSON.stringify(swagger))
+    fs.writeFileSync(join(assetsPath, "apidoc.json"), doc.data)
+    fs.writeFileSync(join(assetsPath, "project.json"), doc.project)
+    console.log(
+      `Docs generated successfully, find in ${assetsPath}, swagger.json, apidoc.json and project.json`
+    )
+  } else {
+    throw "Unable to generate docs."
+  }
+  // delete the temporary config file
+  fs.unlinkSync(configPath)
+  setTimeout(async () => {
+    if (shouldOpen === "open") {
+      await open(join(assetsPath, "index.html"), { wait: false })
+    }
+  }, 2000)
+}
+
+generate().catch(err => {
+  console.error(err)
+})