@budibase/backend-core 2.9.40-alpha.6 → 2.10.1

package/src/db/couch/DatabaseImpl.ts

@@ -0,0 +1,238 @@
+import Nano from "@budibase/nano"
+import {
+  AllDocsResponse,
+  AnyDocument,
+  Database,
+  DatabaseOpts,
+  DatabaseQueryOpts,
+  DatabasePutOpts,
+  DatabaseCreateIndexOpts,
+  DatabaseDeleteIndexOpts,
+  Document,
+  isDocument,
+} from "@budibase/types"
+import { getCouchInfo } from "./connections"
+import { directCouchUrlCall } from "./utils"
+import { getPouchDB } from "./pouchDB"
+import { WriteStream, ReadStream } from "fs"
+import { newid } from "../../docIds/newid"
+
+function buildNano(couchInfo: { url: string; cookie: string }) {
+  return Nano({
+    url: couchInfo.url,
+    requestDefaults: {
+      headers: {
+        Authorization: couchInfo.cookie,
+      },
+    },
+    parseUrl: false,
+  })
+}
+
+export function DatabaseWithConnection(
+  dbName: string,
+  connection: string,
+  opts?: DatabaseOpts
+) {
+  if (!connection) {
+    throw new Error("Must provide connection details")
+  }
+  return new DatabaseImpl(dbName, opts, connection)
+}
+
+export class DatabaseImpl implements Database {
+  public readonly name: string
+  private static nano: Nano.ServerScope
+  private readonly instanceNano?: Nano.ServerScope
+  private readonly pouchOpts: DatabaseOpts
+
+  private readonly couchInfo = getCouchInfo()
+
+  constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) {
+    if (dbName == null) {
+      throw new Error("Database name cannot be undefined.")
+    }
+    this.name = dbName
+    this.pouchOpts = opts || {}
+    if (connection) {
+      this.couchInfo = getCouchInfo(connection)
+      this.instanceNano = buildNano(this.couchInfo)
+    }
+    if (!DatabaseImpl.nano) {
+      DatabaseImpl.init()
+    }
+  }
+
+  static init() {
+    const couchInfo = getCouchInfo()
+    DatabaseImpl.nano = buildNano(couchInfo)
+  }
+
+  async exists() {
+    const response = await directCouchUrlCall({
+      url: `${this.couchInfo.url}/${this.name}`,
+      method: "HEAD",
+      cookie: this.couchInfo.cookie,
+    })
+    return response.status === 200
+  }
+
+  private nano() {
+    return this.instanceNano || DatabaseImpl.nano
+  }
+
+  async checkSetup() {
+    let shouldCreate = !this.pouchOpts?.skip_setup
+    // check exists in a lightweight fashion
+    let exists = await this.exists()
+    if (!shouldCreate && !exists) {
+      throw new Error("DB does not exist")
+    }
+    if (!exists) {
+      try {
+        await this.nano().db.create(this.name)
+      } catch (err: any) {
+        // Handling race conditions
+        if (err.statusCode !== 412) {
+          throw err
+        }
+      }
+    }
+    return this.nano().db.use(this.name)
+  }
+
+  private async updateOutput(fnc: any) {
+    try {
+      return await fnc()
+    } catch (err: any) {
+      if (err.statusCode) {
+        err.status = err.statusCode
+      }
+      throw err
+    }
+  }
+
+  async get<T>(id?: string): Promise<T | any> {
+    const db = await this.checkSetup()
+    if (!id) {
+      throw new Error("Unable to get doc without a valid _id.")
+    }
+    return this.updateOutput(() => db.get(id))
+  }
+
+  async remove(idOrDoc: string | Document, rev?: string) {
+    const db = await this.checkSetup()
+    let _id: string
+    let _rev: string
+
+    if (isDocument(idOrDoc)) {
+      _id = idOrDoc._id!
+      _rev = idOrDoc._rev!
+    } else {
+      _id = idOrDoc
+      _rev = rev!
+    }
+
+    if (!_id || !_rev) {
+      throw new Error("Unable to remove doc without a valid _id and _rev.")
+    }
+    return this.updateOutput(() => db.destroy(_id, _rev))
+  }
+
+  async post(document: AnyDocument, opts?: DatabasePutOpts) {
+    if (!document._id) {
+      document._id = newid()
+    }
+    return this.put(document, opts)
+  }
+
+  async put(document: AnyDocument, opts?: DatabasePutOpts) {
+    if (!document._id) {
+      throw new Error("Cannot store document without _id field.")
+    }
+    const db = await this.checkSetup()
+    if (!document.createdAt) {
+      document.createdAt = new Date().toISOString()
+    }
+    document.updatedAt = new Date().toISOString()
+    if (opts?.force && document._id) {
+      try {
+        const existing = await this.get(document._id)
+        if (existing) {
+          document._rev = existing._rev
+        }
+      } catch (err: any) {
+        if (err.status !== 404) {
+          throw err
+        }
+      }
+    }
+    return this.updateOutput(() => db.insert(document))
+  }
+
+  async bulkDocs(documents: AnyDocument[]) {
+    const db = await this.checkSetup()
+    return this.updateOutput(() => db.bulk({ docs: documents }))
+  }
+
+  async allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>> {
+    const db = await this.checkSetup()
+    return this.updateOutput(() => db.list(params))
+  }
+
+  async query<T>(
+    viewName: string,
+    params: DatabaseQueryOpts
+  ): Promise<AllDocsResponse<T>> {
+    const db = await this.checkSetup()
+    const [database, view] = viewName.split("/")
+    return this.updateOutput(() => db.view(database, view, params))
+  }
+
+  async destroy() {
+    try {
+      return await this.nano().db.destroy(this.name)
+    } catch (err: any) {
+      // didn't exist, don't worry
+      if (err.statusCode === 404) {
+        return
+      } else {
+        throw { ...err, status: err.statusCode }
+      }
+    }
+  }
+
+  async compact() {
+    const db = await this.checkSetup()
+    return this.updateOutput(() => db.compact())
+  }
+
+  // All below functions are in-frequently called, just utilise PouchDB
+  // for them as it implements them better than we can
+  async dump(stream: WriteStream, opts?: { filter?: any }) {
+    const pouch = getPouchDB(this.name)
+    // @ts-ignore
+    return pouch.dump(stream, opts)
+  }
+
+  async load(stream: ReadStream) {
+    const pouch = getPouchDB(this.name)
+    // @ts-ignore
+    return pouch.load(stream)
+  }
+
+  async createIndex(opts: DatabaseCreateIndexOpts) {
+    const pouch = getPouchDB(this.name)
+    return pouch.createIndex(opts)
+  }
+
+  async deleteIndex(opts: DatabaseDeleteIndexOpts) {
+    const pouch = getPouchDB(this.name)
+    return pouch.deleteIndex(opts)
+  }
+
+  async getIndexes() {
+    const pouch = getPouchDB(this.name)
+    return pouch.getIndexes()
+  }
+}

package/src/db/couch/connections.ts

@@ -0,0 +1,77 @@
+import env from "../../environment"
+
+export const getCouchInfo = (connection?: string) => {
+  const urlInfo = getUrlInfo(connection)
+  let username
+  let password
+  if (urlInfo.auth?.username) {
+    // set from url
+    username = urlInfo.auth.username
+  } else if (env.COUCH_DB_USERNAME) {
+    // set from env
+    username = env.COUCH_DB_USERNAME
+  } else if (!env.isTest()) {
+    throw new Error("CouchDB username not set")
+  }
+  if (urlInfo.auth?.password) {
+    // set from url
+    password = urlInfo.auth.password
+  } else if (env.COUCH_DB_PASSWORD) {
+    // set from env
+    password = env.COUCH_DB_PASSWORD
+  } else if (!env.isTest()) {
+    throw new Error("CouchDB password not set")
+  }
+  const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
+  return {
+    url: urlInfo.url!,
+    auth: {
+      username: username,
+      password: password,
+    },
+    cookie: `Basic ${authCookie}`,
+  }
+}
+
+export const getUrlInfo = (url = env.COUCH_DB_URL) => {
+  let cleanUrl, username, password, host
+  if (url) {
+    // Ensure the URL starts with a protocol
+    const protoRegex = /^https?:\/\//i
+    if (!protoRegex.test(url)) {
+      url = `http://${url}`
+    }
+
+    // Split into protocol and remainder
+    const split = url.split("://")
+    const protocol = split[0]
+    const rest = split.slice(1).join("://")
+
+    // Extract auth if specified
+    if (url.includes("@")) {
+      // Split into host and remainder
+      let parts = rest.split("@")
+      host = parts[parts.length - 1]
+      let auth = parts.slice(0, -1).join("@")
+
+      // Split auth into username and password
+      if (auth.includes(":")) {
+        const authParts = auth.split(":")
+        username = authParts[0]
+        password = authParts.slice(1).join(":")
+      } else {
+        username = auth
+      }
+    } else {
+      host = rest
+    }
+    cleanUrl = `${protocol}://${host}`
+  }
+  return {
+    url: cleanUrl,
+    auth: {
+      username,
+      password,
+    },
+  }
+}

package/src/db/couch/index.ts

@@ -0,0 +1,5 @@
+export * from "./connections"
+export * from "./DatabaseImpl"
+export * from "./utils"
+export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"
+export * from "../constants"

package/src/db/couch/pouchDB.ts

@@ -0,0 +1,97 @@
+import PouchDB from "pouchdb"
+import env from "../../environment"
+import { PouchOptions } from "@budibase/types"
+import { getCouchInfo } from "./connections"
+
+let Pouch: any
+let initialised = false
+
+/**
+ * Return a constructor for PouchDB.
+ * This should be rarely used outside of the main application config.
+ * Exposed for exceptional cases such as in-memory views.
+ */
+export const getPouch = (opts: PouchOptions = {}) => {
+  let { url, cookie } = getCouchInfo()
+  let POUCH_DB_DEFAULTS = {
+    prefix: url,
+    fetch: (url: string, opts: any) => {
+      // use a specific authorization cookie - be very explicit about how we authenticate
+      opts.headers.set("Authorization", cookie)
+      return PouchDB.fetch(url, opts)
+    },
+  }
+
+  if (opts.inMemory) {
+    const inMemory = require("pouchdb-adapter-memory")
+    PouchDB.plugin(inMemory)
+    POUCH_DB_DEFAULTS = {
+      // @ts-ignore
+      adapter: "memory",
+    }
+  }
+
+  if (opts.onDisk) {
+    POUCH_DB_DEFAULTS = {
+      // @ts-ignore
+      adapter: "leveldb",
+    }
+  }
+
+  if (opts.replication) {
+    const replicationStream = require("@budibase/pouchdb-replication-stream")
+    PouchDB.plugin(replicationStream.plugin)
+    // @ts-ignore
+    PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
+  }
+
+  if (opts.find) {
+    const find = require("pouchdb-find")
+    PouchDB.plugin(find)
+  }
+
+  return PouchDB.defaults(POUCH_DB_DEFAULTS)
+}
+
+export function init(opts?: PouchOptions) {
+  Pouch = getPouch(opts)
+  initialised = true
+}
+
+const checkInitialised = () => {
+  if (!initialised) {
+    throw new Error("init has not been called")
+  }
+}
+
+export function getPouchDB(dbName: string, opts?: any): PouchDB.Database {
+  checkInitialised()
+  const db = new Pouch(dbName, opts)
+  const dbPut = db.put
+  db.put = async (doc: any, options = {}) => {
+    if (!doc.createdAt) {
+      doc.createdAt = new Date().toISOString()
+    }
+    doc.updatedAt = new Date().toISOString()
+    return dbPut(doc, options)
+  }
+  db.exists = async () => {
+    const info = await db.info()
+    return !info.error
+  }
+  return db
+}
+
+// use this function if you have called getPouchDB - close
+// the databases you've opened once finished
+export async function closePouchDB(db: PouchDB.Database) {
+  if (!db || env.isTest()) {
+    return
+  }
+  try {
+    // specifically await so that if there is an error, it can be ignored
+    return await db.close()
+  } catch (err) {
+    // ignore error, already closed
+  }
+}

package/src/db/couch/utils.ts

@@ -0,0 +1,50 @@
+import { getCouchInfo } from "./connections"
+import fetch from "node-fetch"
+import { checkSlashesInUrl } from "../../helpers"
+
+export async function directCouchCall(
+  path: string,
+  method: string = "GET",
+  body?: any
+) {
+  let { url, cookie } = getCouchInfo()
+  const couchUrl = `${url}/${path}`
+  return await directCouchUrlCall({ url: couchUrl, cookie, method, body })
+}
+
+export async function directCouchUrlCall({
+  url,
+  cookie,
+  method,
+  body,
+}: {
+  url: string
+  cookie: string
+  method: string
+  body?: any
+}) {
+  const params: any = {
+    method: method,
+    headers: {
+      Authorization: cookie,
+    },
+  }
+  if (body && method !== "GET") {
+    params.body = JSON.stringify(body)
+    params.headers["Content-Type"] = "application/json"
+  }
+  return await fetch(checkSlashesInUrl(encodeURI(url)), params)
+}
+
+export async function directCouchQuery(
+  path: string,
+  method: string = "GET",
+  body?: any
+) {
+  const response = await directCouchCall(path, method, body)
+  if (response.status < 300) {
+    return await response.json()
+  } else {
+    throw "Cannot connect to CouchDB instance"
+  }
+}

package/src/db/db.ts

@@ -0,0 +1,43 @@
+import env from "../environment"
+import { directCouchQuery, DatabaseImpl } from "./couch"
+import { CouchFindOptions, Database } from "@budibase/types"
+
+const dbList = new Set()
+
+export function getDB(dbName?: string, opts?: any): Database {
+  return new DatabaseImpl(dbName, opts)
+}
+
+// we have to use a callback for this so that we can close
+// the DB when we're done, without this manual requests would
+// need to close the database when done with it to avoid memory leaks
+export async function doWithDB<T>(
+  dbName: string,
+  cb: (db: Database) => Promise<T>,
+  opts = {}
+) {
+  const db = getDB(dbName, opts)
+  // need this to be async so that we can correctly close DB after all
+  // async operations have been completed
+  return await cb(db)
+}
+
+export function allDbs() {
+  if (!env.isTest()) {
+    throw new Error("Cannot be used outside test environment.")
+  }
+  return [...dbList]
+}
+
+export async function directCouchAllDbs(queryString?: string) {
+  let couchPath = "/_all_dbs"
+  if (queryString) {
+    couchPath += `?${queryString}`
+  }
+  return await directCouchQuery(couchPath)
+}
+
+export async function directCouchFind(dbName: string, opts: CouchFindOptions) {
+  const json = await directCouchQuery(`${dbName}/_find`, "POST", opts)
+  return { rows: json.docs, bookmark: json.bookmark }
+}

package/src/db/errors.ts

@@ -0,0 +1,14 @@
+export function checkErrorCode(error: any, code: number) {
+  const stringCode = code.toString()
+  if (typeof error === "object") {
+    return error.status === code || error.message?.includes(stringCode)
+  } else if (typeof error === "number") {
+    return error === code
+  } else if (typeof error === "string") {
+    return error.includes(stringCode)
+  }
+}
+
+export function isDocumentConflictError(error: any) {
+  return checkErrorCode(error, 409)
+}

package/src/db/index.ts

@@ -0,0 +1,12 @@
+export * from "./couch"
+export * from "./db"
+export * from "./utils"
+export * from "./views"
+export * from "../docIds/conversions"
+export { default as Replication } from "./Replication"
+// exports to support old export structure
+export * from "../constants/db"
+export { getGlobalDBName, baseGlobalDBName } from "../context"
+export * from "./lucene"
+export * as searchIndexes from "./searchIndexes"
+export * from "./errors"