@budibase/server 1.0.115 → 1.0.118

package/dist/utilities/fileSystem/index.js

@@ -12,7 +12,7 @@ const { budibaseTempDir } = require("../budibaseDir");
 const fs = require("fs");
 const { join } = require("path");
 const uuid = require("uuid/v4");
-const CouchDB = require("../../db");
+const { doWithDB } = require("@budibase/backend-core/db");
 const { ObjectStoreBuckets } = require("../../constants");
 const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
@@ -135,32 +135,33 @@ exports.streamBackup = (appId) => __awaiter(void 0, void 0, void 0, function* ()
  * @return {*} either a readable stream or a string
  */
 exports.exportDB = (dbName, { stream, filter, exportName } = {}) => __awaiter(void 0, void 0, void 0, function* () {
-    const instanceDb = new CouchDB(dbName);
-    // Stream the dump if required
-    if (stream) {
-        const memStream = new MemoryStream();
-        instanceDb.dump(memStream, { filter });
-        return memStream;
-    }
-    // Write the dump to file if required
-    if (exportName) {
-        const path = join(budibaseTempDir(), exportName);
-        const writeStream = fs.createWriteStream(path);
-        yield instanceDb.dump(writeStream, { filter });
-        // Upload the dump to the object store if self hosted
-        if (env.SELF_HOSTED) {
-            yield streamUpload(ObjectStoreBuckets.BACKUPS, join(dbName, exportName), fs.createReadStream(path));
+    return doWithDB(dbName, (db) => __awaiter(void 0, void 0, void 0, function* () {
+        // Stream the dump if required
+        if (stream) {
+            const memStream = new MemoryStream();
+            db.dump(memStream, { filter });
+            return memStream;
         }
-        return fs.createReadStream(path);
-    }
-    // Stringify the dump in memory if required
-    const memStream = new MemoryStream();
-    let appString = "";
-    memStream.on("data", chunk => {
-        appString += chunk.toString();
-    });
-    yield instanceDb.dump(memStream, { filter });
-    return appString;
+        // Write the dump to file if required
+        if (exportName) {
+            const path = join(budibaseTempDir(), exportName);
+            const writeStream = fs.createWriteStream(path);
+            yield db.dump(writeStream, { filter });
+            // Upload the dump to the object store if self hosted
+            if (env.SELF_HOSTED) {
+                yield streamUpload(ObjectStoreBuckets.BACKUPS, join(dbName, exportName), fs.createReadStream(path));
+            }
+            return fs.createReadStream(path);
+        }
+        // Stringify the dump in memory if required
+        const memStream = new MemoryStream();
+        let appString = "";
+        memStream.on("data", chunk => {
+            appString += chunk.toString();
+        });
+        yield db.dump(memStream, { filter });
+        return appString;
+    }));
 });
 /**
  * Writes the provided contents to a temporary file, which can be used briefly.

package/dist/utilities/usageQuota/rows.js

@@ -9,8 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils");
-const CouchDB = require("../../db");
-const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db");
+const { isDevAppID, getDevelopmentAppID, doWithDB, } = require("@budibase/backend-core/db");
 const ROW_EXCLUSIONS = [USER_METDATA_PREFIX];
 const getAppPairs = appIds => {
     // collect the app ids into dev / prod pairs
@@ -32,20 +31,21 @@ const getAppPairs = appIds => {
 };
 const getAppRows = (appId) => __awaiter(void 0, void 0, void 0, function* () {
     // need to specify the app ID, as this is used for different apps in one call
-    const appDb = new CouchDB(appId);
-    const response = yield appDb.allDocs(getRowParams(null, null, {
-        include_docs: false,
-    }));
-    return response.rows
-        .map(r => r.id)
-        .filter(id => {
-        for (let exclusion of ROW_EXCLUSIONS) {
-            if (id.startsWith(exclusion)) {
-                return false;
+    return doWithDB(appId, (db) => __awaiter(void 0, void 0, void 0, function* () {
+        const response = yield db.allDocs(getRowParams(null, null, {
+            include_docs: false,
+        }));
+        return response.rows
+            .map(r => r.id)
+            .filter(id => {
+            for (let exclusion of ROW_EXCLUSIONS) {
+                if (id.startsWith(exclusion)) {
+                    return false;
+                }
             }
-        }
-        return true;
-    });
+            return true;
+        });
+    }));
 });
 /**
  * Return a set of all rows in the given app ids.
@@ -62,7 +62,7 @@ exports.getUniqueRows = (appIds) => __awaiter(void 0, void 0, void 0, function*
                 continue;
             }
             try {
-                appRows.push(yield getAppRows(appId));
+                appRows = appRows.concat(yield getAppRows(appId));
             }
             catch (e) {
                 console.error(e);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.115",
+  "version": "1.0.118",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -68,9 +68,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.115",
-    "@budibase/client": "^1.0.115",
-    "@budibase/string-templates": "^1.0.115",
+    "@budibase/backend-core": "^1.0.118",
+    "@budibase/client": "^1.0.118",
+    "@budibase/string-templates": "^1.0.118",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -117,7 +117,7 @@
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",
     "posthog-node": "^1.1.4",
-    "pouchdb": "7.2.1",
+    "pouchdb": "7.3.0",
     "pouchdb-adapter-memory": "^7.2.1",
     "pouchdb-all-dbs": "1.0.2",
     "pouchdb-find": "^7.2.2",
@@ -174,5 +174,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "732a547c2cc3940ecfe144433105960ca94423c8"
+  "gitHead": "ef0d516dcb713a215b5bfcc008b3fe9d9145b693"
 }

package/scripts/exportAppTemplate.js

@@ -2,7 +2,8 @@
 const yargs = require("yargs")
 const fs = require("fs")
 const { join } = require("path")
-const CouchDB = require("../src/db")
+require("../src/db").init()
+const { doWithDB } = require("@budibase/backend-core/db")
 // load environment
 const env = require("../src/environment")
 const {
@@ -47,13 +48,14 @@ yargs
       const writeStream = fs.createWriteStream(join(exportPath, "dump.text"))
       // perform couch dump
 
-      const instanceDb = new CouchDB(appId)
-      await instanceDb.dump(writeStream, {
-        filter: doc =>
-          !(
-            doc._id.includes(USER_METDATA_PREFIX) ||
-            doc.includes(LINK_USER_METADATA_PREFIX)
-          ),
+      await doWithDB(appId, async db => {
+        return db.dump(writeStream, {
+          filter: doc =>
+            !(
+              doc._id.includes(USER_METDATA_PREFIX) ||
+              doc.includes(LINK_USER_METADATA_PREFIX)
+            ),
+        })
       })
       console.log(`Template ${name} exported to ${exportPath}`)
     }

package/src/api/controllers/application.js

@@ -123,7 +123,7 @@ async function createInstance(template) {
   const tenantId = isMultiTenant() ? getTenantId() : null
   const baseAppId = generateAppID(tenantId)
   const appId = generateDevAppID(baseAppId)
-  updateAppId(appId)
+  await updateAppId(appId)
 
   const db = getAppDB()
   await db.put({
@@ -404,6 +404,8 @@ exports.sync = async (ctx, next) => {
     })
   } catch (err) {
     error = err
+  } finally {
+    await replication.close()
   }
 
   // sync the users

package/src/api/controllers/deploy/index.js

@@ -94,18 +94,16 @@ async function initDeployedApp(prodAppId) {
 }
 
 async function deployApp(deployment) {
+  let replication
   try {
     const appId = getAppId()
     const devAppId = getDevelopmentAppID(appId)
     const productionAppId = getProdAppID(appId)
-
-    const replication = new Replication({
+    replication = new Replication({
       source: devAppId,
       target: productionAppId,
     })
-
     console.log("Replication object created")
-
     await replication.replicate()
     console.log("replication complete.. replacing app meta doc")
     const db = getProdAppDB()
@@ -126,6 +124,8 @@ async function deployApp(deployment) {
       ...err,
       message: `Deployment Failed: ${err.message}`,
     }
+  } finally {
+    await replication.close()
   }
 }
 

package/src/api/controllers/dev.js

@@ -83,7 +83,9 @@ exports.revert = async ctx => {
   try {
     const db = getProdAppDB({ skip_setup: true })
     const info = await db.info()
-    if (info.error) throw info.error
+    if (info.error) {
+      throw info.error
+    }
     const deploymentDoc = await db.get(DocumentTypes.DEPLOYMENTS)
     if (
       !deploymentDoc.history ||
@@ -95,12 +97,11 @@ exports.revert = async ctx => {
     return ctx.throw(400, "App has not yet been deployed")
   }
 
+  const replication = new Replication({
+    source: productionAppId,
+    target: appId,
+  })
   try {
-    const replication = new Replication({
-      source: productionAppId,
-      target: appId,
-    })
-
     await replication.rollback()
     // update appID in reverted app to be dev version again
     const db = getAppDB()
@@ -114,6 +115,8 @@ exports.revert = async ctx => {
     }
   } catch (err) {
     ctx.throw(400, `Unable to revert. ${err}`)
+  } finally {
+    await replication.close()
   }
 }
 

package/src/api/controllers/query/import/tests/index.spec.js

@@ -1,12 +1,4 @@
-
-const bulkDocs = jest.fn()
-const db = jest.fn(() => {
-  return {
-    bulkDocs
-  }
-})
-jest.mock("../../../../../db", () => db)
-require("@budibase/backend-core").init(require("../../../../../db"))
+const TestConfig = require("../../../../../tests/utilities/TestConfiguration")
 
 const { RestImporter } = require("../index")
 
@@ -48,6 +40,12 @@ const datasets = {
 }
 
 describe("Rest Importer", () => {
+  const config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
   let restImporter 
 
   const init = async (data) => {
@@ -105,11 +103,9 @@ describe("Rest Importer", () => {
 
   const testImportQueries = async (key, data, assertions) => {
     await init(data)
-    bulkDocs.mockReturnValue([])
     const importResult = await restImporter.importQueries("datasourceId")
     expect(importResult.errorQueries.length).toBe(0)
     expect(importResult.queries.length).toBe(assertions[key].count)
-    expect(bulkDocs).toHaveBeenCalledTimes(1)
     jest.clearAllMocks()
   }
 

package/src/api/controllers/row/ExternalRequest.ts

@@ -323,6 +323,28 @@ module External {
       return { row: newRow, manyRelationships }
     }
 
+    squashRelationshipColumns(
+      table: Table,
+      row: Row,
+      relationships: RelationshipsJson[]
+    ): Row {
+      for (let relationship of relationships) {
+        const linkedTable = this.tables[relationship.tableName]
+        if (!linkedTable) {
+          continue
+        }
+        const display = linkedTable.primaryDisplay
+        for (let key of Object.keys(row[relationship.column])) {
+          const related: Row = row[relationship.column][key]
+          row[relationship.column][key] = {
+            primaryDisplay: display ? related[display] : undefined,
+            _id: related._id,
+          }
+        }
+      }
+      return row
+    }
+
     /**
      * This iterates through the returned rows and works out what elements of the rows
      * actually match up to another row (based on primary keys) - this is pretty specific
@@ -354,12 +376,6 @@ module External {
         if (!linked._id) {
           continue
         }
-        // if not returning full docs then get the minimal links out
-        const display = linkedTable.primaryDisplay
-        linked = {
-          primaryDisplay: display ? linked[display] : undefined,
-          _id: linked._id,
-        }
         columns[relationship.column] = linked
       }
       for (let [column, related] of Object.entries(columns)) {
@@ -417,7 +433,9 @@ module External {
           relationships
         )
       }
-      return processFormulas(table, Object.values(finalRows))
+      return processFormulas(table, Object.values(finalRows)).map((row: Row) =>
+        this.squashRelationshipColumns(table, row, relationships)
+      )
     }
 
     /**

package/src/api/controllers/row/internal.js

@@ -6,6 +6,7 @@ const {
   DocumentTypes,
   InternalTables,
 } = require("../../../db/utils")
+const { dangerousGetDB } = require("@budibase/backend-core/db")
 const userController = require("../user")
 const {
   inputProcessing,
@@ -250,7 +251,7 @@ exports.fetch = async ctx => {
 }
 
 exports.find = async ctx => {
-  const db = getAppDB()
+  const db = dangerousGetDB(ctx.appId)
   const table = await db.get(ctx.params.tableId)
   let row = await findRow(ctx, ctx.params.tableId, ctx.params.rowId)
   row = await outputProcessing(table, row)

package/src/api/controllers/webhook.js

@@ -54,7 +54,7 @@ exports.destroy = async ctx => {
 }
 
 exports.buildSchema = async ctx => {
-  updateAppId(ctx.params.instance)
+  await updateAppId(ctx.params.instance)
   const db = getAppDB()
   const webhook = await db.get(ctx.params.id)
   webhook.bodySchema = toJsonSchema(ctx.request.body)
@@ -80,7 +80,7 @@ exports.buildSchema = async ctx => {
 
 exports.trigger = async ctx => {
   const prodAppId = getProdAppID(ctx.params.instance)
-  updateAppId(prodAppId)
+  await updateAppId(prodAppId)
   try {
     const db = getAppDB()
     const webhook = await db.get(ctx.params.id)

package/src/api/routes/tests/row.spec.js

@@ -2,6 +2,7 @@ const { outputProcessing } = require("../../../utilities/rowProcessor")
 const setup = require("./utilities")
 const { basicRow } = setup.structures
 const { doInAppContext } = require("@budibase/backend-core/context")
+const { doInTenant } = require("@budibase/backend-core/tenancy")
 
 // mock the fetch for the search system
 jest.mock("node-fetch")
@@ -340,17 +341,20 @@ describe("/rows", () => {
 
   describe("fetchEnrichedRows", () => {
     it("should allow enriching some linked rows", async () => {
-      const table = await config.createLinkedTable()
-      const firstRow = await config.createRow({
-        name: "Test Contact",
-        description: "original description",
-        tableId: table._id
-      })
-      const secondRow = await config.createRow({
-        name: "Test 2",
-        description: "og desc",
-        link: [{_id: firstRow._id}],
-        tableId: table._id,
+      const { table, firstRow, secondRow } = await doInTenant(setup.structures.TENANT_ID, async () => {
+        const table = await config.createLinkedTable()
+        const firstRow = await config.createRow({
+          name: "Test Contact",
+          description: "original description",
+          tableId: table._id
+        })
+        const secondRow = await config.createRow({
+          name: "Test 2",
+          description: "og desc",
+          link: [{_id: firstRow._id}],
+          tableId: table._id,
+        })
+        return { table, firstRow, secondRow }
       })
 
       // test basic enrichment

package/src/api/routes/tests/table.spec.js

@@ -1,4 +1,5 @@
-const { checkBuilderEndpoint, getDB } = require("./utilities/TestFunctions")
+const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+const { getAppDB } = require("@budibase/backend-core/context")
 const setup = require("./utilities")
 const { basicTable } = setup.structures
 
@@ -122,7 +123,7 @@ describe("/tables", () => {
 
   describe("indexing", () => {
     it("should be able to create a table with indexes", async () => {
-      const db = getDB(config)
+      const db = getAppDB(config)
       const indexCount = (await db.getIndexes()).total_rows
       const table = basicTable()
       table.indexes = ["name"]

package/src/api/routes/tests/utilities/TestFunctions.js

@@ -3,7 +3,7 @@ const appController = require("../../../controllers/application")
 const { AppStatus } = require("../../../../db/utils")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { TENANT_ID } = require("../../../../tests/utilities/structures")
-const { getAppDB, doInAppContext } = require("@budibase/backend-core/context")
+const { doInAppContext } = require("@budibase/backend-core/context")
 const env = require("../../../../environment")
 
 function Request(appId, params) {
@@ -106,10 +106,6 @@ exports.checkPermissionsEndpoint = async ({
     .expect(403)
 }
 
-exports.getDB = () => {
-  return getAppDB()
-}
-
 exports.testAutomation = async (config, automation) => {
   return runRequest(automation.appId, async () => {
     return await config.request

package/src/app.ts

@@ -2,8 +2,8 @@
 import { ExtendableContext } from "koa"
 
 import * as env from "./environment"
-const CouchDB = require("./db")
-require("@budibase/backend-core").init(CouchDB)
+import db from "./db"
+db.init()
 const Koa = require("koa")
 const destroyable = require("server-destroy")
 const koaBody = require("koa-body")

package/src/automations/tests/deleteRow.spec.js

@@ -27,7 +27,7 @@ describe("test the delete row action", () => {
     expect(res.row._id).toEqual(row._id)
     let error
     try {
-      await config.getRow(table._id, res.id)
+      await config.getRow(table._id, res.row._id)
     } catch (err) {
       error = err
     }

package/src/automations/utils.js

@@ -1,13 +1,13 @@
 const { Thread, ThreadType } = require("../threads")
 const { definitions } = require("./triggerInfo")
 const webhooks = require("../api/controllers/webhook")
-const CouchDB = require("../db")
 const { queue } = require("./bullboard")
 const newid = require("../db/newid")
 const { updateEntityMetadata } = require("../utilities")
 const { MetadataTypes, WebhookType } = require("../constants")
 const { getProdAppID } = require("@budibase/backend-core/db")
 const { cloneDeep } = require("lodash/fp")
+const { doWithDB } = require("@budibase/backend-core/db")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
@@ -101,10 +101,11 @@ exports.enableCronTrigger = async (appId, automation) => {
     // can't use getAppDB here as this is likely to be called from dev app,
     // but this call could be for dev app or prod app, need to just use what
     // was passed in
-    const db = new CouchDB(appId)
-    const response = await db.put(automation)
-    automation._id = response.id
-    automation._rev = response.rev
+    await doWithDB(appId, async db => {
+      const response = await db.put(automation)
+      automation._id = response.id
+      automation._rev = response.rev
+    })
   }
   return automation
 }

package/src/db/inMemoryView.js

@@ -1,48 +1,49 @@
-const PouchDB = require("pouchdb")
-const memory = require("pouchdb-adapter-memory")
 const newid = require("./newid")
 
-PouchDB.plugin(memory)
-const Pouch = PouchDB.defaults({
-  prefix: undefined,
-  adapter: "memory",
-})
+// bypass the main application db config
+// use in memory pouchdb directly
+const { getPouch, closeDB } = require("@budibase/backend-core/db")
+const Pouch = getPouch({ inMemory: true })
 
 exports.runView = async (view, calculation, group, data) => {
   // use a different ID each time for the DB, make sure they
   // are always unique for each query, don't want overlap
   // which could cause 409s
   const db = new Pouch(newid())
-  // write all the docs to the in memory Pouch (remove revs)
-  await db.bulkDocs(
-    data.map(row => ({
-      ...row,
-      _rev: undefined,
-    }))
-  )
-  let fn = (doc, emit) => emit(doc._id)
-  eval("fn = " + view.map.replace("function (doc)", "function (doc, emit)"))
-  const queryFns = {
-    meta: view.meta,
-    map: fn,
-  }
-  if (view.reduce) {
-    queryFns.reduce = view.reduce
-  }
-  const response = await db.query(queryFns, {
-    include_docs: !calculation,
-    group: !!group,
-  })
-  // need to fix the revs to be totally accurate
-  for (let row of response.rows) {
-    if (!row._rev || !row._id) {
-      continue
+  try {
+    // write all the docs to the in memory Pouch (remove revs)
+    await db.bulkDocs(
+      data.map(row => ({
+        ...row,
+        _rev: undefined,
+      }))
+    )
+    let fn = (doc, emit) => emit(doc._id)
+    eval("fn = " + view.map.replace("function (doc)", "function (doc, emit)"))
+    const queryFns = {
+      meta: view.meta,
+      map: fn,
+    }
+    if (view.reduce) {
+      queryFns.reduce = view.reduce
     }
-    const found = data.find(possible => possible._id === row._id)
-    if (found) {
-      row._rev = found._rev
+    const response = await db.query(queryFns, {
+      include_docs: !calculation,
+      group: !!group,
+    })
+    // need to fix the revs to be totally accurate
+    for (let row of response.rows) {
+      if (!row._rev || !row._id) {
+        continue
+      }
+      const found = data.find(possible => possible._id === row._id)
+      if (found) {
+        row._rev = found._rev
+      }
     }
+    return response
+  } finally {
+    await db.destroy()
+    await closeDB(db)
   }
-  await db.destroy()
-  return response
 }

package/src/db/index.js

@@ -1,3 +1,16 @@
-const client = require("./client")
+const core = require("@budibase/backend-core")
+const env = require("../environment")
 
-module.exports = client
+exports.init = () => {
+  const dbConfig = {
+    replication: true,
+    find: true,
+  }
+
+  if (env.isTest()) {
+    dbConfig.inMemory = true
+    dbConfig.allDbs = true
+  }
+
+  core.init({ db: dbConfig })
+}