@budibase/server 1.0.46-alpha.7 → 1.0.48

package/src/integrations/googlesheets.ts

@@ -1,353 +0,0 @@
-import {
-  DatasourceFieldTypes,
-  Integration,
-  QueryJson,
-  QueryTypes,
-} from "../definitions/datasource"
-import { OAuth2Client } from "google-auth-library"
-import { DatasourcePlus } from "./base/datasourcePlus"
-import { Row, Table, TableSchema } from "../definitions/common"
-import { buildExternalTableId } from "./utils"
-import { DataSourceOperation, FieldTypes } from "../constants"
-import { GoogleSpreadsheet } from "google-spreadsheet"
-import { table } from "console"
-
-module GoogleSheetsModule {
-  const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-  const { getScopedConfig } = require("@budibase/backend-core/db")
-  const { Configs } = require("@budibase/backend-core/constants")
-
-  interface GoogleSheetsConfig {
-    spreadsheetId: string
-    auth: OAuthClientConfig
-  }
-
-  interface OAuthClientConfig {
-    appId: string
-    accessToken: string
-    refreshToken: string
-  }
-
-  const SCHEMA: Integration = {
-    plus: true,
-    auth: {
-      type: "google",
-    },
-    relationships: false,
-    docs: "https://developers.google.com/sheets/api/quickstart/nodejs",
-    description:
-      "Create and collaborate on online spreadsheets in real-time and from any device. ",
-    friendlyName: "Google Sheets",
-    datasource: {
-      spreadsheetId: {
-        type: DatasourceFieldTypes.STRING,
-        required: true,
-      },
-    },
-    query: {
-      create: {
-        type: QueryTypes.FIELDS,
-        fields: {
-          sheet: {
-            type: DatasourceFieldTypes.STRING,
-            required: true,
-          },
-          row: {
-            type: QueryTypes.JSON,
-            required: true,
-          },
-        },
-      },
-      read: {
-        type: QueryTypes.FIELDS,
-        fields: {
-          sheet: {
-            type: DatasourceFieldTypes.STRING,
-            required: true,
-          },
-        },
-      },
-      update: {
-        type: QueryTypes.FIELDS,
-        fields: {
-          sheet: {
-            type: DatasourceFieldTypes.STRING,
-            required: true,
-          },
-          rowIndex: {
-            type: DatasourceFieldTypes.STRING,
-            required: true,
-          },
-          row: {
-            type: QueryTypes.JSON,
-            required: true,
-          },
-        },
-      },
-      delete: {
-        type: QueryTypes.FIELDS,
-        fields: {
-          sheet: {
-            type: DatasourceFieldTypes.STRING,
-            required: true,
-          },
-          rowIndex: {
-            type: DatasourceFieldTypes.NUMBER,
-            required: true,
-          },
-        },
-      },
-    },
-  }
-
-  class GoogleSheetsIntegration implements DatasourcePlus {
-    private readonly config: GoogleSheetsConfig
-    private client: any
-    public tables: Record<string, Table> = {}
-    public schemaErrors: Record<string, string> = {}
-
-    constructor(config: GoogleSheetsConfig) {
-      this.config = config
-      const spreadsheetId = this.cleanSpreadsheetUrl(this.config.spreadsheetId)
-      this.client = new GoogleSpreadsheet(spreadsheetId)
-    }
-
-    /**
-     * Pull the spreadsheet ID out from a valid google sheets URL
-     * @param spreadsheetId - the URL or standard spreadsheetId of the google sheet
-     * @returns spreadsheet Id of the google sheet
-     */
-    cleanSpreadsheetUrl(spreadsheetId: string) {
-      if (!spreadsheetId) {
-        throw new Error(
-          "You must set a spreadsheet ID in your configuration to fetch tables."
-        )
-      }
-      const parts = spreadsheetId.split("/")
-      return parts.length > 5 ? parts[5] : spreadsheetId
-    }
-
-    async connect() {
-      try {
-        // Initialise oAuth client
-        const db = getGlobalDB()
-        const googleConfig = await getScopedConfig(db, {
-          type: Configs.GOOGLE,
-        })
-        const oauthClient = new OAuth2Client({
-          clientId: googleConfig.clientID,
-          clientSecret: googleConfig.clientSecret,
-        })
-        oauthClient.credentials.access_token = this.config.auth.accessToken
-        oauthClient.credentials.refresh_token = this.config.auth.refreshToken
-        this.client.useOAuth2Client(oauthClient)
-        await this.client.loadInfo()
-      } catch (err) {
-        console.error("Error connecting to google sheets", err)
-        throw err
-      }
-    }
-
-    async buildSchema(datasourceId: string) {
-      await this.connect()
-      const sheets = await this.client.sheetsByIndex
-      const tables: Record<string, Table> = {}
-      for (let sheet of sheets) {
-        // must fetch rows to determine schema
-        await sheet.getRows()
-        // build schema
-        const schema: TableSchema = {}
-
-        // build schema from headers
-        for (let header of sheet.headerValues) {
-          schema[header] = {
-            name: header,
-            type: FieldTypes.STRING,
-          }
-        }
-
-        // create tables
-        tables[sheet.title] = {
-          _id: buildExternalTableId(datasourceId, sheet.title),
-          name: sheet.title,
-          primary: ["rowNumber"],
-          schema,
-        }
-      }
-
-      this.tables = tables
-    }
-
-    async query(json: QueryJson) {
-      const sheet = json.endpoint.entityId
-
-      const handlers = {
-        [DataSourceOperation.CREATE]: () =>
-          this.create({ sheet, row: json.body }),
-        [DataSourceOperation.READ]: () => this.read({ sheet }),
-        [DataSourceOperation.UPDATE]: () =>
-          this.update({
-            // exclude the header row and zero index
-            rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
-            sheet,
-            row: json.body,
-          }),
-        [DataSourceOperation.DELETE]: () =>
-          this.delete({
-            // exclude the header row and zero index
-            rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
-            sheet,
-          }),
-        [DataSourceOperation.CREATE_TABLE]: () =>
-          this.createTable(json?.table?.name),
-        [DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table),
-        [DataSourceOperation.DELETE_TABLE]: () =>
-          this.deleteTable(json?.table?.name),
-      }
-
-      const internalQueryMethod = handlers[json.endpoint.operation]
-
-      return await internalQueryMethod()
-    }
-
-    buildRowObject(headers: string[], values: string[], rowNumber: number) {
-      const rowObject: { rowNumber: number; [key: string]: any } = { rowNumber }
-      for (let i = 0; i < headers.length; i++) {
-        rowObject._id = rowNumber
-        rowObject[headers[i]] = values[i]
-      }
-      return rowObject
-    }
-
-    async createTable(name?: string) {
-      try {
-        await this.connect()
-        const sheet = await this.client.addSheet({ title: name })
-        return sheet
-      } catch (err) {
-        console.error("Error creating new table in google sheets", err)
-        throw err
-      }
-    }
-
-    async updateTable(table?: any) {
-      try {
-        await this.connect()
-        const sheet = await this.client.sheetsByTitle[table.name]
-        await sheet.loadHeaderRow()
-
-        if (table._rename) {
-          const headers = []
-          for (let header of sheet.headerValues) {
-            if (header === table._rename.old) {
-              headers.push(table._rename.updated)
-            } else {
-              headers.push(header)
-            }
-          }
-          await sheet.setHeaderRow(headers)
-        } else {
-          let newField = Object.keys(table.schema).find(
-            key => !sheet.headerValues.includes(key)
-          )
-          await sheet.setHeaderRow([...sheet.headerValues, newField])
-        }
-      } catch (err) {
-        console.error("Error updating table in google sheets", err)
-        throw err
-      }
-    }
-
-    async deleteTable(sheet: any) {
-      try {
-        await this.connect()
-        const sheetToDelete = await this.client.sheetsByTitle[sheet]
-        return await sheetToDelete.delete()
-      } catch (err) {
-        console.error("Error deleting table in google sheets", err)
-        throw err
-      }
-    }
-
-    async create(query: { sheet: string; row: any }) {
-      try {
-        await this.connect()
-        const sheet = await this.client.sheetsByTitle[query.sheet]
-        const rowToInsert =
-          typeof query.row === "string" ? JSON.parse(query.row) : query.row
-        const row = await sheet.addRow(rowToInsert)
-        return [
-          this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber),
-        ]
-      } catch (err) {
-        console.error("Error writing to google sheets", err)
-        throw err
-      }
-    }
-
-    async read(query: { sheet: string }) {
-      try {
-        await this.connect()
-        const sheet = await this.client.sheetsByTitle[query.sheet]
-        const rows = await sheet.getRows()
-        const headerValues = sheet.headerValues
-        const response = []
-        for (let row of rows) {
-          response.push(
-            this.buildRowObject(headerValues, row._rawData, row._rowNumber)
-          )
-        }
-        return response
-      } catch (err) {
-        console.error("Error reading from google sheets", err)
-        throw err
-      }
-    }
-
-    async update(query: { sheet: string; rowIndex: number; row: any }) {
-      try {
-        await this.connect()
-        const sheet = await this.client.sheetsByTitle[query.sheet]
-        const rows = await sheet.getRows()
-        const row = rows[query.rowIndex]
-        if (row) {
-          const updateValues = query.row
-          for (let key in updateValues) {
-            row[key] = updateValues[key]
-          }
-          await row.save()
-          return [
-            this.buildRowObject(
-              sheet.headerValues,
-              row._rawData,
-              row._rowNumber
-            ),
-          ]
-        } else {
-          throw new Error("Row does not exist.")
-        }
-      } catch (err) {
-        console.error("Error reading from google sheets", err)
-        throw err
-      }
-    }
-
-    async delete(query: { sheet: string; rowIndex: number }) {
-      await this.connect()
-      const sheet = await this.client.sheetsByTitle[query.sheet]
-      const rows = await sheet.getRows()
-      const row = rows[query.rowIndex]
-      if (row) {
-        await row.delete()
-        return [{ deleted: query.rowIndex }]
-      } else {
-        throw new Error("Row does not exist.")
-      }
-    }
-  }
-
-  module.exports = {
-    schema: SCHEMA,
-    integration: GoogleSheetsIntegration,
-  }
-}

package/src/migrations/tests/usageQuotas/index.spec.js

@@ -1,27 +0,0 @@
-const env = require("../../../environment")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-
-const syncApps = jest.fn()
-const syncRows = jest.fn()
-
-jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
-jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
-
-const migrations = require("../../usageQuotas")
-
-describe("run", () => {
-  let config = new TestConfig(false)
-
-  beforeEach(async () => {
-    await config.init()
-    env._set("USE_QUOTAS", 1)
-  })
-
-  afterAll(config.end)  
-
-  it("runs the required migrations", async () => {
-    await migrations.run()
-    expect(syncApps).toHaveBeenCalledTimes(1)
-    expect(syncRows).toHaveBeenCalledTimes(1)
-  })
-})

package/src/migrations/tests/usageQuotas/syncApps.spec.js

@@ -1,37 +0,0 @@
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
-const syncApps = require("../../usageQuotas/syncApps")
-const env = require("../../../environment")
-
-describe("syncApps", () => {
-  let config = new TestConfig(false)
-
-  beforeEach(async () => {
-    await config.init()
-    env._set("USE_QUOTAS", 1)
-  })
-
-  afterAll(config.end)  
-
-  it("runs successfully", async () => {
-    // create the usage quota doc and mock usages
-    const db = getGlobalDB()
-    await getUsageQuotaDoc(db)
-    await update(Properties.APPS, 3)
-
-    let usageDoc = await getUsageQuotaDoc(db)
-    expect(usageDoc.usageQuota.apps).toEqual(3)
-
-    // create an extra app to test the migration
-    await config.createApp("quota-test")
-
-    // migrate
-    await syncApps.run()
-
-    // assert the migration worked 
-    usageDoc = await getUsageQuotaDoc(db)
-    expect(usageDoc.usageQuota.apps).toEqual(2)
-  })
-})
-

package/src/migrations/tests/usageQuotas/syncRows.spec.js

@@ -1,43 +0,0 @@
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
-const syncRows = require("../../usageQuotas/syncRows")
-const env = require("../../../environment")
-
-describe("syncRows", () => {
-  let config = new TestConfig(false)
-
-  beforeEach(async () => {
-    await config.init()
-    env._set("USE_QUOTAS", 1)
-  })
-
-  afterAll(config.end)  
-
-  it("runs successfully", async () => {
-    // create the usage quota doc and mock usages
-    const db = getGlobalDB()
-    await getUsageQuotaDoc(db)
-    await update(Properties.ROW, 300)
-
-    let usageDoc = await getUsageQuotaDoc(db)
-    expect(usageDoc.usageQuota.rows).toEqual(300)
-
-    // app 1
-    await config.createTable()
-    await config.createRow()
-    // app 2
-    await config.createApp("second-app")
-    await config.createTable()
-    await config.createRow()
-    await config.createRow()
-    
-    // migrate
-    await syncRows.run()
-
-    // assert the migration worked 
-    usageDoc = await getUsageQuotaDoc(db)
-    expect(usageDoc.usageQuota.rows).toEqual(3)
-  })
-})
-

package/src/migrations/usageQuotas/index.js

@@ -1,24 +0,0 @@
-const {
-  MIGRATIONS,
-  MIGRATION_DBS,
-  migrateIfRequired,
-} = require("@budibase/backend-core/migrations")
-const { useQuotas } = require("../../utilities/usageQuota")
-const syncApps = require("./syncApps")
-const syncRows = require("./syncRows")
-
-exports.run = async () => {
-  if (!useQuotas()) {
-    return
-  }
-
-  // Jan 2022
-  await migrateIfRequired(
-    MIGRATION_DBS.GLOBAL_DB,
-    MIGRATIONS.QUOTAS_1,
-    async () => {
-      await syncApps.run()
-      await syncRows.run()
-    }
-  )
-}

package/src/migrations/usageQuotas/syncApps.js

@@ -1,18 +0,0 @@
-const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
-const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../../db")
-const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
-
-exports.run = async () => {
-  const db = getGlobalDB()
-  // get app count
-  const devApps = await getAllApps(CouchDB, { dev: true })
-  const appCount = devApps ? devApps.length : 0
-
-  // sync app count
-  const tenantId = getTenantId()
-  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.apps = appCount
-  await db.put(usageDoc)
-}

package/src/migrations/usageQuotas/syncRows.js

@@ -1,21 +0,0 @@
-const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
-const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../../db")
-const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
-const { getUniqueRows } = require("../../utilities/usageQuota/rows")
-
-exports.run = async () => {
-  const db = getGlobalDB()
-  // get all rows in all apps
-  const allApps = await getAllApps(CouchDB, { all: true })
-  const appIds = allApps ? allApps.map(app => app.appId) : []
-  const rows = await getUniqueRows(appIds)
-  const rowCount = rows ? rows.length : 0
-
-  // sync row count
-  const tenantId = getTenantId()
-  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.rows = rowCount
-  await db.put(usageDoc)
-}

package/src/utilities/tests/usageQuota/usageQuota.spec.js

@@ -1,72 +0,0 @@
-const getTenantId = jest.fn()
-jest.mock("@budibase/backend-core/tenancy", () => ({
-  getTenantId
-}))
-const usageQuota = require("../../usageQuota")
-const env = require("../../../environment")
-
-class TestConfiguration {
-  constructor() {
-    this.enableQuotas()
-  }
-
-  enableQuotas = () => {
-    env.USE_QUOTAS = 1
-  }
-
-  disableQuotas = () => {
-    env.USE_QUOTAS = null
-  }
-
-  setTenantId = (tenantId) => {
-    getTenantId.mockReturnValue(tenantId)
-  } 
-
-  setExcludedTenants = (tenants) => {
-    env.EXCLUDE_QUOTAS_TENANTS = tenants
-  }
-
-  reset = () => {
-    this.disableQuotas()
-    this.setExcludedTenants(null)
-  }
-}
-
-describe("usageQuota", () => {
-  let config
-
-  beforeEach(() => {
-    config = new TestConfiguration()
-  })
-
-  afterEach(() => {
-    config.reset()
-  })
-
-  describe("useQuotas", () => {
-    it("works when no settings have been provided", () => {
-      config.reset()
-      expect(usageQuota.useQuotas()).toBe(false)
-    })
-    it("honours USE_QUOTAS setting", () => {
-      config.disableQuotas()
-      expect(usageQuota.useQuotas()).toBe(false)
-
-      config.enableQuotas()
-      expect(usageQuota.useQuotas()).toBe(true)
-    })
-    it("honours EXCLUDE_QUOTAS_TENANTS setting", () => {
-      config.setTenantId("test")
-
-      // tenantId is in the list
-      config.setExcludedTenants("test, test2, test2")
-      expect(usageQuota.useQuotas()).toBe(false)
-      config.setExcludedTenants("test,test2,test2")
-      expect(usageQuota.useQuotas()).toBe(false)
-
-      // tenantId is not in the list
-      config.setTenantId("other")
-      expect(usageQuota.useQuotas()).toBe(true)
-    })
-  })
-})

package/src/utilities/usageQuota/index.js

@@ -1,92 +0,0 @@
-const env = require("../../environment")
-const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
-const {
-  StaticDatabases,
-  generateNewUsageQuotaDoc,
-} = require("@budibase/backend-core/db")
-
-exports.useQuotas = () => {
-  // check if quotas are enabled
-  if (env.USE_QUOTAS) {
-    // check if there are any tenants without limits
-    if (env.EXCLUDE_QUOTAS_TENANTS) {
-      const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(
-        /\s/g,
-        ""
-      ).split(",")
-      const tenantId = getTenantId()
-      if (excludedTenants.includes(tenantId)) {
-        return false
-      }
-    }
-    return true
-  }
-  return false
-}
-
-exports.Properties = {
-  ROW: "rows",
-  UPLOAD: "storage", // doesn't work yet
-  VIEW: "views", // doesn't work yet
-  USER: "users", // doesn't work yet
-  AUTOMATION: "automationRuns", // doesn't work yet
-  APPS: "apps",
-  EMAILS: "emails", // doesn't work yet
-}
-
-exports.getUsageQuotaDoc = async db => {
-  let quota
-  try {
-    quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
-  } catch (err) {
-    // doc doesn't exist. Create it
-    quota = generateNewUsageQuotaDoc()
-    const response = await db.put(quota)
-    quota._rev = response.rev
-  }
-
-  return quota
-}
-
-/**
- * Given a specified tenantId this will add to the usage object for the specified property.
- * @param {string} property The property which is to be added to (within the nested usageQuota object).
- * @param {number} usage The amount (this can be negative) to adjust the number by.
- * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
- * also been reset after this call.
- */
-exports.update = async (property, usage, opts = { dryRun: false }) => {
-  if (!exports.useQuotas()) {
-    return
-  }
-
-  try {
-    const db = getGlobalDB()
-    const quota = await exports.getUsageQuotaDoc(db)
-
-    // increment the quota
-    quota.usageQuota[property] += usage
-
-    if (
-      quota.usageQuota[property] > quota.usageLimits[property] &&
-      usage > 0 // allow for decrementing usage when the quota is already exceeded
-    ) {
-      throw new Error(
-        `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
-      )
-    }
-
-    if (quota.usageQuota[property] < 0) {
-      // never go negative if the quota has previously been exceeded
-      quota.usageQuota[property] = 0
-    }
-
-    // update the usage quotas
-    if (!opts.dryRun) {
-      await db.put(quota)
-    }
-  } catch (err) {
-    console.error(`Error updating usage quotas for ${property}`, err)
-    throw err
-  }
-}