@budibase/server 1.0.49-alpha.1 → 1.0.49-alpha.2

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.49-alpha.1",
-    "@budibase/client": "^1.0.49-alpha.1",
-    "@budibase/string-templates": "^1.0.49-alpha.1",
+    "@budibase/backend-core": "^1.0.49-alpha.2",
+    "@budibase/client": "^1.0.49-alpha.2",
+    "@budibase/string-templates": "^1.0.49-alpha.2",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -168,5 +168,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "3f8d81837dd8ba0ebf17dbd8106c3a5d3bec3e46"
+  "gitHead": "cb6189e76b22266d2f55c9ad2e4f38639272cd20"
 }

package/src/api/controllers/application.js

@@ -75,7 +75,7 @@ function getUserRoleId(ctx) {
     : ctx.user.role._id
 }
 
-async function getAppUrl(ctx) {
+exports.getAppUrl = ctx => {
   // construct the url
   let url
   if (ctx.request.body.url) {
@@ -219,7 +219,7 @@ exports.create = async ctx => {
   const apps = await getAllApps(CouchDB, { dev: true })
   const name = ctx.request.body.name
   checkAppName(ctx, apps, name)
-  const url = await getAppUrl(ctx)
+  const url = exports.getAppUrl(ctx)
   checkAppUrl(ctx, apps, url)
 
   const { useTemplate, templateKey, templateString } = ctx.request.body
@@ -286,7 +286,7 @@ exports.update = async ctx => {
   if (name) {
     checkAppName(ctx, apps, name, ctx.params.appId)
   }
-  const url = await getAppUrl(ctx)
+  const url = await exports.getAppUrl(ctx)
   if (url) {
     checkAppUrl(ctx, apps, url, ctx.params.appId)
     ctx.request.body.url = url

package/src/api/controllers/migrations.js

@@ -0,0 +1,13 @@
+const { migrate, MIGRATIONS } = require("../../migrations")
+
+exports.migrate = async ctx => {
+  const options = ctx.request.body
+  // don't await as can take a while, just return
+  migrate(options)
+  ctx.status = 200
+}
+
+exports.fetchDefinitions = async ctx => {
+  ctx.body = MIGRATIONS
+  ctx.status = 200
+}

package/src/api/routes/index.js

@@ -24,6 +24,7 @@ const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
 const cloudRoutes = require("./cloud")
+const migrationRoutes = require("./migrations")
 
 exports.mainRoutes = [
   authRoutes,
@@ -53,6 +54,7 @@ exports.mainRoutes = [
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,
   rowRoutes,
+  migrationRoutes,
 ]
 
 exports.staticRoutes = staticRoutes

package/src/api/routes/migrations.js

@@ -0,0 +1,14 @@
+const Router = require("@koa/router")
+const migrationsController = require("../controllers/migrations")
+const router = Router()
+const { internalApi } = require("@budibase/backend-core/auth")
+
+router
+  .post("/api/migrations/run", internalApi, migrationsController.migrate)
+  .get(
+    "/api/migrations/definitions",
+    internalApi,
+    migrationsController.fetchDefinitions
+  )
+
+module.exports = router

package/src/app.ts

@@ -1,7 +1,7 @@
 // need to load environment first
 import { ExtendableContext } from "koa"
 
-const env = require("./environment")
+import * as env from "./environment"
 const CouchDB = require("./db")
 require("@budibase/backend-core").init(CouchDB)
 const Koa = require("koa")
@@ -16,6 +16,7 @@ const Sentry = require("@sentry/node")
 const fileSystem = require("./utilities/fileSystem")
 const bullboard = require("./automations/bullboard")
 const redis = require("./utilities/redis")
+import * as migrations from "./migrations"
 
 const app = new Koa()
 
@@ -84,13 +85,25 @@ module.exports = server.listen(env.PORT || 0, async () => {
   await automations.init()
 })
 
-process.on("uncaughtException", err => {
-  console.error(err)
+const shutdown = () => {
   server.close()
   server.destroy()
+}
+
+process.on("uncaughtException", err => {
+  console.error(err)
+  shutdown()
 })
 
 process.on("SIGTERM", () => {
-  server.close()
-  server.destroy()
+  shutdown()
 })
+
+// run migrations on startup if not done via http
+// not recommended in a clustered environment
+if (!env.HTTP_MIGRATIONS) {
+  migrations.migrate().catch(err => {
+    console.error("Error performing migrations. Exiting.\n", err)
+    shutdown()
+  })
+}

package/src/environment.js

@@ -43,6 +43,7 @@ module.exports = {
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
+  HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
   // environment
   NODE_ENV: process.env.NODE_ENV,
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,

package/src/middleware/usageQuota.js

@@ -5,7 +5,6 @@ const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
-const migration = require("../migrations/usageQuotas")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -74,7 +73,6 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await migration.run()
     await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)

package/src/migrations/functions/appUrls.ts

@@ -0,0 +1,25 @@
+const { DocumentTypes } = require("@budibase/backend-core/db")
+import { getAppUrl } from "../../api/controllers/application"
+
+/**
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Add the url to the app metadata if it doesn't exist
+ */
+export const run = async (appDb: any) => {
+  const metadata = await appDb.get(DocumentTypes.APP_METADATA)
+  if (!metadata.url) {
+    const context = {
+      request: {
+        body: {
+          name: metadata.name,
+        },
+      },
+    }
+    metadata.url = getAppUrl(context)
+    console.log(`Adding url to app: ${metadata.url}`)
+  }
+  await appDb.put(metadata)
+}

package/src/migrations/functions/quotas1.ts

@@ -0,0 +1,20 @@
+import { runQuotaMigration } from "./usageQuotas"
+import * as syncApps from "./usageQuotas/syncApps"
+import * as syncRows from "./usageQuotas/syncRows"
+
+/**
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Synchronise the app and row quotas to the state of the db after it was
+ * discovered that the quota resets were still in place and the row quotas
+ * weren't being decremented correctly.
+ */
+
+export const run = async () => {
+  await runQuotaMigration(async () => {
+    await syncApps.run()
+    await syncRows.run()
+  })
+}

package/src/migrations/functions/tests/appUrls.spec.js

@@ -0,0 +1,29 @@
+const { DocumentTypes } = require("@budibase/backend-core/db")
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const migration = require("../appUrls")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+  const CouchDB = config.getCouch()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    const app = await config.createApp("testApp")
+    const appDb = new CouchDB(app.appId)
+    let metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    delete metadata.url
+    await appDb.put(metadata)
+    
+    await migration.run(appDb)
+
+    metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    expect(metadata.url).toEqual("/testapp")
+  })
+})

package/src/migrations/{tests/usageQuotas/index.spec.js → functions/tests/quotas1.spec.js}

@@ -4,10 +4,10 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const syncApps = jest.fn()
 const syncRows = jest.fn()
 
-jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
-jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
+jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
+jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
 
-const migrations = require("../../usageQuotas")
+const migration = require("../quotas1")
 
 describe("run", () => {
   let config = new TestConfig(false)
@@ -19,8 +19,8 @@ describe("run", () => {
 
   afterAll(config.end)  
 
-  it("runs the required migrations", async () => {
-    await migrations.run()
+  it("runs ", async () => {
+    await migration.run()
     expect(syncApps).toHaveBeenCalledTimes(1)
     expect(syncRows).toHaveBeenCalledTimes(1)
   })

package/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -0,0 +1,25 @@
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+
+// mock email view creation
+const coreDb = require("@budibase/backend-core/db")
+const createUserEmailView = jest.fn()
+coreDb.createUserEmailView = createUserEmailView
+
+const migration = require("../userEmailViewCasing")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+  const globalDb = getGlobalDB()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    await migration.run(globalDb)
+    expect(createUserEmailView).toHaveBeenCalledTimes(1)
+  })
+})

package/src/migrations/functions/usageQuotas/index.ts

@@ -0,0 +1,8 @@
+const { useQuotas } = require("../../../utilities/usageQuota")
+
+export const runQuotaMigration = async (migration: Function) => {
+  if (!useQuotas()) {
+    return
+  }
+  await migration()
+}

package/src/migrations/{usageQuotas/syncApps.js → functions/usageQuotas/syncApps.ts}

@@ -1,9 +1,9 @@
 const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../../db")
-const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+import CouchDB from "../../../db"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
 
-exports.run = async () => {
+export const run = async () => {
   const db = getGlobalDB()
   // get app count
   const devApps = await getAllApps(CouchDB, { dev: true })

package/src/migrations/{usageQuotas/syncRows.js → functions/usageQuotas/syncRows.ts}

@@ -1,14 +1,14 @@
 const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../../db")
-const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
-const { getUniqueRows } = require("../../utilities/usageQuota/rows")
+import CouchDB from "../../../db"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
+import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 
-exports.run = async () => {
+export const run = async () => {
   const db = getGlobalDB()
   // get all rows in all apps
   const allApps = await getAllApps(CouchDB, { all: true })
-  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
   const rows = await getUniqueRows(appIds)
   const rowCount = rows ? rows.length : 0
 

package/src/migrations/{tests/usageQuotas → functions/usageQuotas/tests}/syncApps.spec.js

@@ -1,8 +1,8 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
-const syncApps = require("../../usageQuotas/syncApps")
-const env = require("../../../environment")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
+const syncApps = require("../syncApps")
+const env = require("../../../../environment")
 
 describe("syncApps", () => {
   let config = new TestConfig(false)

package/src/migrations/{tests/usageQuotas → functions/usageQuotas/tests}/syncRows.spec.js

@@ -1,8 +1,8 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
-const syncRows = require("../../usageQuotas/syncRows")
-const env = require("../../../environment")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
+const syncRows = require("../syncRows")
+const env = require("../../../../environment")
 
 describe("syncRows", () => {
   let config = new TestConfig(false)

package/src/migrations/functions/userEmailViewCasing.ts

@@ -0,0 +1,13 @@
+const { createUserEmailView } = require("@budibase/backend-core/db")
+
+/**
+ * Date:
+ * October 2021
+ *
+ * Description:
+ * Recreate the user email view to include latest changes i.e. lower casing the email address
+ */
+
+export const run = async (db: any) => {
+  await createUserEmailView(db)
+}

package/src/migrations/index.ts

@@ -0,0 +1,56 @@
+import CouchDB from "../db"
+const {
+  MIGRATION_TYPES,
+  runMigrations,
+} = require("@budibase/backend-core/migrations")
+
+// migration functions
+import * as userEmailViewCasing from "./functions/userEmailViewCasing"
+import * as quota1 from "./functions/quotas1"
+import * as appUrls from "./functions/appUrls"
+
+export interface Migration {
+  type: string
+  name: string
+  opts?: object
+  fn: Function
+}
+
+/**
+ * e.g.
+ * {
+ *   tenantIds: ['bb'],
+ *   force: {
+ *    global: ['quota_1']
+ *   }
+ * }
+ */
+export interface MigrationOptions {
+  tenantIds?: string[]
+  forced?: {
+    [type: string]: string[]
+  }
+}
+
+export const MIGRATIONS: Migration[] = [
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "user_email_view_casing",
+    fn: userEmailViewCasing.run,
+  },
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "quotas_1",
+    fn: quota1.run,
+  },
+  {
+    type: MIGRATION_TYPES.APP,
+    name: "app_urls",
+    opts: { all: true },
+    fn: appUrls.run,
+  },
+]
+
+export const migrate = async (options?: MigrationOptions) => {
+  await runMigrations(CouchDB, MIGRATIONS, options)
+}

package/src/tests/utilities/TestConfiguration.js

@@ -50,6 +50,10 @@ class TestConfiguration {
     return this.appId
   }
 
+  getCouch() {
+    return CouchDB
+  }
+
   async _req(config, params, controlFunc) {
     const request = {}
     // fake cookies, we don't need them

package/coverage/lcov-report/migrations/usageQuotas/index.html

@@ -1,141 +0,0 @@
-
-<!doctype html>
-<html lang="en">
-
-<head>
-    <title>Code coverage report for migrations/usageQuotas</title>
-    <meta charset="utf-8" />
-    <link rel="stylesheet" href="../../prettify.css" />
-    <link rel="stylesheet" href="../../base.css" />
-    <link rel="shortcut icon" type="image/x-icon" href="../../favicon.png" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <style type='text/css'>
-        .coverage-summary .sorter {
-            background-image: url(../../sort-arrow-sprite.png);
-        }
-    </style>
-</head>
-    
-<body>
-<div class='wrapper'>
-    <div class='pad1'>
-        <h1><a href="../../index.html">All files</a> migrations/usageQuotas</h1>
-        <div class='clearfix'>
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">100% </span>
-                <span class="quiet">Statements</span>
-                <span class='fraction'>40/40</span>
-            </div>
-        
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">62.5% </span>
-                <span class="quiet">Branches</span>
-                <span class='fraction'>5/8</span>
-            </div>
-        
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">100% </span>
-                <span class="quiet">Functions</span>
-                <span class='fraction'>5/5</span>
-            </div>
-        
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">100% </span>
-                <span class="quiet">Lines</span>
-                <span class='fraction'>39/39</span>
-            </div>
-        
-            
-        </div>
-        <p class="quiet">
-            Press <em>n</em> or <em>j</em> to go to the next uncovered block, <em>b</em>, <em>p</em> or <em>k</em> for the previous block.
-        </p>
-    </div>
-    <div class='status-line high'></div>
-    <div class="pad1">
-<table class="coverage-summary">
-<thead>
-<tr>
-   <th data-col="file" data-fmt="html" data-html="true" class="file">File</th>
-   <th data-col="pic" data-type="number" data-fmt="html" data-html="true" class="pic"></th>
-   <th data-col="statements" data-type="number" data-fmt="pct" class="pct">Statements</th>
-   <th data-col="statements_raw" data-type="number" data-fmt="html" class="abs"></th>
-   <th data-col="branches" data-type="number" data-fmt="pct" class="pct">Branches</th>
-   <th data-col="branches_raw" data-type="number" data-fmt="html" class="abs"></th>
-   <th data-col="functions" data-type="number" data-fmt="pct" class="pct">Functions</th>
-   <th data-col="functions_raw" data-type="number" data-fmt="html" class="abs"></th>
-   <th data-col="lines" data-type="number" data-fmt="pct" class="pct">Lines</th>
-   <th data-col="lines_raw" data-type="number" data-fmt="html" class="abs"></th>
-</tr>
-</thead>
-<tbody><tr>
-	<td class="file high" data-value="index.js"><a href="index.js.html">index.js</a></td>
-	<td data-value="100" class="pic high">
-	<div class="chart"><div class="cover-fill cover-full" style="width: 100%"></div><div class="cover-empty" style="width: 0%"></div></div>
-	</td>
-	<td data-value="100" class="pct high">100%</td>
-	<td data-value="10" class="abs high">10/10</td>
-	<td data-value="100" class="pct high">100%</td>
-	<td data-value="2" class="abs high">2/2</td>
-	<td data-value="100" class="pct high">100%</td>
-	<td data-value="2" class="abs high">2/2</td>
-	<td data-value="100" class="pct high">100%</td>
-	<td data-value="10" class="abs high">10/10</td>
-	</tr>
-
-<tr>
-	<td class="file high" data-value="syncApps.js"><a href="syncApps.js.html">syncApps.js</a></td>
-	<td data-value="100" class="pic high">
-	<div class="chart"><div class="cover-fill cover-full" style="width: 100%"></div><div class="cover-empty" style="width: 0%"></div></div>
-	</td>
-	<td data-value="100" class="pct high">100%</td>
-	<td data-value="13" class="abs high">13/13</td>
-	<td data-value="50" class="pct medium">50%</td>
-	<td data-value="2" class="abs medium">1/2</td>
-	<td data-value="100" class="pct high">100%</td>
-	<td data-value="1" class="abs high">1/1</td>
-	<td data-value="100" class="pct high">100%</td>
-	<td data-value="13" class="abs high">13/13</td>
-	</tr>
-
-<tr>
-	<td class="file high" data-value="syncRows.js"><a href="syncRows.js.html">syncRows.js</a></td>
-	<td data-value="100" class="pic high">
-	<div class="chart"><div class="cover-fill cover-full" style="width: 100%"></div><div class="cover-empty" style="width: 0%"></div></div>
-	</td>
-	<td data-value="100" class="pct high">100%</td>
-	<td data-value="17" class="abs high">17/17</td>
-	<td data-value="50" class="pct medium">50%</td>
-	<td data-value="4" class="abs medium">2/4</td>
-	<td data-value="100" class="pct high">100%</td>
-	<td data-value="2" class="abs high">2/2</td>
-	<td data-value="100" class="pct high">100%</td>
-	<td data-value="16" class="abs high">16/16</td>
-	</tr>
-
-</tbody>
-</table>
-</div>
-                <div class='push'></div><!-- for sticky footer -->
-            </div><!-- /wrapper -->
-            <div class='footer quiet pad2 space-top1 center small'>
-                Code coverage generated by
-                <a href="https://istanbul.js.org/" target="_blank">istanbul</a>
-                at Mon Jan 31 2022 16:01:47 GMT+0000 (Coordinated Universal Time)
-            </div>
-        </div>
-        <script src="../../prettify.js"></script>
-        <script>
-            window.onload = function () {
-                prettyPrint();
-            };
-        </script>
-        <script src="../../sorter.js"></script>
-        <script src="../../block-navigation.js"></script>
-    </body>
-</html>
-