@budibase/server 1.0.47 → 1.0.49-alpha.2

package/src/integrations/base/sql.ts

@@ -72,16 +72,22 @@ class InternalBuilder {
 
   // right now we only do filters on the specific table being queried
   addFilters(
-    tableName: string,
     query: KnexQuery,
-    filters: SearchFilters | undefined
+    filters: SearchFilters | undefined,
+    opts: { relationship?: boolean; tableName?: string }
   ): KnexQuery {
     function iterate(
       structure: { [key: string]: any },
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        fn(`${tableName}.${key}`, value)
+        const isRelationshipField = key.includes(".")
+        if (!opts.relationship && !isRelationshipField) {
+          fn(`${opts.tableName}.${key}`, value)
+        }
+        if (opts.relationship && isRelationshipField) {
+          fn(key, value)
+        }
       }
     }
     if (!filters) {
@@ -272,7 +278,7 @@ class InternalBuilder {
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
-    query = this.addFilters(tableName, query, filters)
+    query = this.addFilters(query, filters, { tableName })
     // add sorting to pre-query
     query = this.addSorting(query, json)
     // @ts-ignore
@@ -285,20 +291,21 @@ class InternalBuilder {
       preQuery = this.addSorting(preQuery, json)
     }
     // handle joins
-    return this.addRelationships(
+    query = this.addRelationships(
       knex,
       preQuery,
       selectStatement,
       tableName,
       relationships
     )
+    return this.addFilters(query, filters, { relationship: true })
   }
 
   update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
     const parsedBody = parseBody(body)
-    query = this.addFilters(endpoint.entityId, query, filters)
+    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.update(parsedBody)
@@ -310,7 +317,7 @@ class InternalBuilder {
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
-    query = this.addFilters(endpoint.entityId, query, filters)
+    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.delete()

package/src/integrations/googlesheets.ts

@@ -0,0 +1,353 @@
+import {
+  DatasourceFieldTypes,
+  Integration,
+  QueryJson,
+  QueryTypes,
+} from "../definitions/datasource"
+import { OAuth2Client } from "google-auth-library"
+import { DatasourcePlus } from "./base/datasourcePlus"
+import { Row, Table, TableSchema } from "../definitions/common"
+import { buildExternalTableId } from "./utils"
+import { DataSourceOperation, FieldTypes } from "../constants"
+import { GoogleSpreadsheet } from "google-spreadsheet"
+import { table } from "console"
+
+module GoogleSheetsModule {
+  const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+  const { getScopedConfig } = require("@budibase/backend-core/db")
+  const { Configs } = require("@budibase/backend-core/constants")
+
+  interface GoogleSheetsConfig {
+    spreadsheetId: string
+    auth: OAuthClientConfig
+  }
+
+  interface OAuthClientConfig {
+    appId: string
+    accessToken: string
+    refreshToken: string
+  }
+
+  const SCHEMA: Integration = {
+    plus: true,
+    auth: {
+      type: "google",
+    },
+    relationships: false,
+    docs: "https://developers.google.com/sheets/api/quickstart/nodejs",
+    description:
+      "Create and collaborate on online spreadsheets in real-time and from any device. ",
+    friendlyName: "Google Sheets",
+    datasource: {
+      spreadsheetId: {
+        type: DatasourceFieldTypes.STRING,
+        required: true,
+      },
+    },
+    query: {
+      create: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          sheet: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          row: {
+            type: QueryTypes.JSON,
+            required: true,
+          },
+        },
+      },
+      read: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          sheet: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+        },
+      },
+      update: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          sheet: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          rowIndex: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          row: {
+            type: QueryTypes.JSON,
+            required: true,
+          },
+        },
+      },
+      delete: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          sheet: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          rowIndex: {
+            type: DatasourceFieldTypes.NUMBER,
+            required: true,
+          },
+        },
+      },
+    },
+  }
+
+  class GoogleSheetsIntegration implements DatasourcePlus {
+    private readonly config: GoogleSheetsConfig
+    private client: any
+    public tables: Record<string, Table> = {}
+    public schemaErrors: Record<string, string> = {}
+
+    constructor(config: GoogleSheetsConfig) {
+      this.config = config
+      const spreadsheetId = this.cleanSpreadsheetUrl(this.config.spreadsheetId)
+      this.client = new GoogleSpreadsheet(spreadsheetId)
+    }
+
+    /**
+     * Pull the spreadsheet ID out from a valid google sheets URL
+     * @param spreadsheetId - the URL or standard spreadsheetId of the google sheet
+     * @returns spreadsheet Id of the google sheet
+     */
+    cleanSpreadsheetUrl(spreadsheetId: string) {
+      if (!spreadsheetId) {
+        throw new Error(
+          "You must set a spreadsheet ID in your configuration to fetch tables."
+        )
+      }
+      const parts = spreadsheetId.split("/")
+      return parts.length > 5 ? parts[5] : spreadsheetId
+    }
+
+    async connect() {
+      try {
+        // Initialise oAuth client
+        const db = getGlobalDB()
+        const googleConfig = await getScopedConfig(db, {
+          type: Configs.GOOGLE,
+        })
+        const oauthClient = new OAuth2Client({
+          clientId: googleConfig.clientID,
+          clientSecret: googleConfig.clientSecret,
+        })
+        oauthClient.credentials.access_token = this.config.auth.accessToken
+        oauthClient.credentials.refresh_token = this.config.auth.refreshToken
+        this.client.useOAuth2Client(oauthClient)
+        await this.client.loadInfo()
+      } catch (err) {
+        console.error("Error connecting to google sheets", err)
+        throw err
+      }
+    }
+
+    async buildSchema(datasourceId: string) {
+      await this.connect()
+      const sheets = await this.client.sheetsByIndex
+      const tables: Record<string, Table> = {}
+      for (let sheet of sheets) {
+        // must fetch rows to determine schema
+        await sheet.getRows()
+        // build schema
+        const schema: TableSchema = {}
+
+        // build schema from headers
+        for (let header of sheet.headerValues) {
+          schema[header] = {
+            name: header,
+            type: FieldTypes.STRING,
+          }
+        }
+
+        // create tables
+        tables[sheet.title] = {
+          _id: buildExternalTableId(datasourceId, sheet.title),
+          name: sheet.title,
+          primary: ["rowNumber"],
+          schema,
+        }
+      }
+
+      this.tables = tables
+    }
+
+    async query(json: QueryJson) {
+      const sheet = json.endpoint.entityId
+
+      const handlers = {
+        [DataSourceOperation.CREATE]: () =>
+          this.create({ sheet, row: json.body }),
+        [DataSourceOperation.READ]: () => this.read({ sheet }),
+        [DataSourceOperation.UPDATE]: () =>
+          this.update({
+            // exclude the header row and zero index
+            rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
+            sheet,
+            row: json.body,
+          }),
+        [DataSourceOperation.DELETE]: () =>
+          this.delete({
+            // exclude the header row and zero index
+            rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
+            sheet,
+          }),
+        [DataSourceOperation.CREATE_TABLE]: () =>
+          this.createTable(json?.table?.name),
+        [DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table),
+        [DataSourceOperation.DELETE_TABLE]: () =>
+          this.deleteTable(json?.table?.name),
+      }
+
+      const internalQueryMethod = handlers[json.endpoint.operation]
+
+      return await internalQueryMethod()
+    }
+
+    buildRowObject(headers: string[], values: string[], rowNumber: number) {
+      const rowObject: { rowNumber: number; [key: string]: any } = { rowNumber }
+      for (let i = 0; i < headers.length; i++) {
+        rowObject._id = rowNumber
+        rowObject[headers[i]] = values[i]
+      }
+      return rowObject
+    }
+
+    async createTable(name?: string) {
+      try {
+        await this.connect()
+        const sheet = await this.client.addSheet({ title: name })
+        return sheet
+      } catch (err) {
+        console.error("Error creating new table in google sheets", err)
+        throw err
+      }
+    }
+
+    async updateTable(table?: any) {
+      try {
+        await this.connect()
+        const sheet = await this.client.sheetsByTitle[table.name]
+        await sheet.loadHeaderRow()
+
+        if (table._rename) {
+          const headers = []
+          for (let header of sheet.headerValues) {
+            if (header === table._rename.old) {
+              headers.push(table._rename.updated)
+            } else {
+              headers.push(header)
+            }
+          }
+          await sheet.setHeaderRow(headers)
+        } else {
+          let newField = Object.keys(table.schema).find(
+            key => !sheet.headerValues.includes(key)
+          )
+          await sheet.setHeaderRow([...sheet.headerValues, newField])
+        }
+      } catch (err) {
+        console.error("Error updating table in google sheets", err)
+        throw err
+      }
+    }
+
+    async deleteTable(sheet: any) {
+      try {
+        await this.connect()
+        const sheetToDelete = await this.client.sheetsByTitle[sheet]
+        return await sheetToDelete.delete()
+      } catch (err) {
+        console.error("Error deleting table in google sheets", err)
+        throw err
+      }
+    }
+
+    async create(query: { sheet: string; row: any }) {
+      try {
+        await this.connect()
+        const sheet = await this.client.sheetsByTitle[query.sheet]
+        const rowToInsert =
+          typeof query.row === "string" ? JSON.parse(query.row) : query.row
+        const row = await sheet.addRow(rowToInsert)
+        return [
+          this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber),
+        ]
+      } catch (err) {
+        console.error("Error writing to google sheets", err)
+        throw err
+      }
+    }
+
+    async read(query: { sheet: string }) {
+      try {
+        await this.connect()
+        const sheet = await this.client.sheetsByTitle[query.sheet]
+        const rows = await sheet.getRows()
+        const headerValues = sheet.headerValues
+        const response = []
+        for (let row of rows) {
+          response.push(
+            this.buildRowObject(headerValues, row._rawData, row._rowNumber)
+          )
+        }
+        return response
+      } catch (err) {
+        console.error("Error reading from google sheets", err)
+        throw err
+      }
+    }
+
+    async update(query: { sheet: string; rowIndex: number; row: any }) {
+      try {
+        await this.connect()
+        const sheet = await this.client.sheetsByTitle[query.sheet]
+        const rows = await sheet.getRows()
+        const row = rows[query.rowIndex]
+        if (row) {
+          const updateValues = query.row
+          for (let key in updateValues) {
+            row[key] = updateValues[key]
+          }
+          await row.save()
+          return [
+            this.buildRowObject(
+              sheet.headerValues,
+              row._rawData,
+              row._rowNumber
+            ),
+          ]
+        } else {
+          throw new Error("Row does not exist.")
+        }
+      } catch (err) {
+        console.error("Error reading from google sheets", err)
+        throw err
+      }
+    }
+
+    async delete(query: { sheet: string; rowIndex: number }) {
+      await this.connect()
+      const sheet = await this.client.sheetsByTitle[query.sheet]
+      const rows = await sheet.getRows()
+      const row = rows[query.rowIndex]
+      if (row) {
+        await row.delete()
+        return [{ deleted: query.rowIndex }]
+      } else {
+        throw new Error("Row does not exist.")
+      }
+    }
+  }
+
+  module.exports = {
+    schema: SCHEMA,
+    integration: GoogleSheetsIntegration,
+  }
+}

package/src/integrations/index.ts

@@ -9,6 +9,7 @@ const airtable = require("./airtable")
 const mysql = require("./mysql")
 const arangodb = require("./arangodb")
 const rest = require("./rest")
+const googlesheets = require("./googlesheets")
 const { SourceNames } = require("../definitions/datasource")
 
 const DEFINITIONS = {
@@ -23,6 +24,7 @@ const DEFINITIONS = {
   [SourceNames.MYSQL]: mysql.schema,
   [SourceNames.ARANGODB]: arangodb.schema,
   [SourceNames.REST]: rest.schema,
+  [SourceNames.GOOGLE_SHEETS]: googlesheets.schema,
 }
 
 const INTEGRATIONS = {
@@ -37,6 +39,7 @@ const INTEGRATIONS = {
   [SourceNames.MYSQL]: mysql.integration,
   [SourceNames.ARANGODB]: arangodb.integration,
   [SourceNames.REST]: rest.integration,
+  [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/integrations/tests/sql.spec.js

@@ -119,6 +119,22 @@ describe("SQL query builder", () => {
     })
   })
 
+  it("should allow filtering on a related field", () => {
+    const query = sql._query(generateReadJson({
+      filters: {
+        equal: {
+          age: 10,
+          "task.name": "task 1",
+        },
+      },
+    }))
+    // order of bindings changes because relationship filters occur outside inner query
+    expect(query).toEqual({
+      bindings: [10, limit, "task 1"],
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" = $1 limit $2) as "${TABLE_NAME}" where "task"."name" = $3`
+    })
+  })
+
   it("should test an create statement", () => {
     const query = sql._query(generateCreateJson(TABLE_NAME, {
       name: "Michael",

package/src/integrations/utils.ts

@@ -1,6 +1,5 @@
-import { SqlQuery } from "../definitions/datasource"
+import { SourceNames, SqlQuery } from "../definitions/datasource"
 import { Datasource, Table } from "../definitions/common"
-import { SourceNames } from "../definitions/datasource"
 import { DocumentTypes, SEPARATOR } from "../db/utils"
 import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
 
@@ -127,7 +126,12 @@ export function isSQL(datasource: Datasource): boolean {
   if (!datasource || !datasource.source) {
     return false
   }
-  const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL]
+  const SQL = [
+    SourceNames.POSTGRES,
+    SourceNames.SQL_SERVER,
+    SourceNames.MYSQL,
+    SourceNames.ORACLE,
+  ]
   return SQL.indexOf(datasource.source) !== -1
 }
 
@@ -139,11 +143,60 @@ export function isIsoDateString(str: string) {
   return d.toISOString() === str
 }
 
-// add the existing relationships from the entities if they exist, to prevent them from being overridden
+/**
+ * This function will determine whether a column is a relationship and whether it
+ * is currently valid. The reason for the validity check is that tables can be deleted
+ * outside of Budibase control and if this is the case it will break Budibase relationships.
+ * The tableIds is a list passed down from the main finalise tables function, which is
+ * based on the tables that have just been fetched. This will only really be used on subsequent
+ * fetches to the first one - if the user is periodically refreshing Budibase knowledge of tables.
+ * @param column The column to check, to see if it is a valid relationship.
+ * @param tableIds The IDs of the tables which currently exist.
+ */
+function shouldCopyRelationship(
+  column: { type: string; tableId?: string },
+  tableIds: [string]
+) {
+  return (
+    column.type === FieldTypes.LINK &&
+    column.tableId &&
+    tableIds.includes(column.tableId)
+  )
+}
+
+/**
+ * Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
+ * types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
+ * sure that these get copied over when tables are fetched. Also checks whether they are still valid, if a
+ * column has changed type in the external database then copying it over may not be possible.
+ * @param column The column to check for options or boolean type.
+ * @param fetchedColumn The fetched column to check for the type in the external database.
+ */
+function shouldCopySpecialColumn(
+  column: { type: string },
+  fetchedColumn: { type: string } | undefined
+) {
+  return (
+    column.type === FieldTypes.OPTIONS ||
+    column.type === FieldTypes.ARRAY ||
+    ((!fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER) &&
+      column.type === FieldTypes.BOOLEAN)
+  )
+}
+
+/**
+ * Looks for columns which need to be copied over into the new table definitions, like relationships
+ * and options types.
+ * @param tableName The name of the table which is being checked.
+ * @param table The specific table which is being checked.
+ * @param entities All the tables that existed before - the old table definitions.
+ * @param tableIds The IDs of the tables which exist now, to check if anything has been removed.
+ */
 function copyExistingPropsOver(
   tableName: string,
   table: Table,
-  entities: { [key: string]: any }
+  entities: { [key: string]: any },
+  tableIds: [string]
 ) {
   if (entities && entities[tableName]) {
     if (entities[tableName].primaryDisplay) {
@@ -154,11 +207,10 @@ function copyExistingPropsOver(
       if (!existingTableSchema.hasOwnProperty(key)) {
         continue
       }
+      const column = existingTableSchema[key]
       if (
-        existingTableSchema[key].type === FieldTypes.LINK ||
-        existingTableSchema[key].type === FieldTypes.OPTIONS ||
-        ((!table.schema[key] || table.schema[key].type === FieldTypes.NUMBER) &&
-          existingTableSchema[key].type === FieldTypes.BOOLEAN)
+        shouldCopyRelationship(column, tableIds) ||
+        shouldCopySpecialColumn(column, table.schema[key])
       ) {
         table.schema[key] = existingTableSchema[key]
       }
@@ -167,6 +219,13 @@ function copyExistingPropsOver(
   return table
 }
 
+/**
+ * Look through the final table definitions to see if anything needs to be
+ * copied over from the old and if any errors have occurred mark them so
+ * that the user can be made aware.
+ * @param tables The list of tables that have been retrieved from the external database.
+ * @param entities The old list of tables, if there was any to look for definitions in.
+ */
 export function finaliseExternalTables(
   tables: { [key: string]: any },
   entities: { [key: string]: any }
@@ -174,6 +233,8 @@ export function finaliseExternalTables(
   const invalidColumns = Object.values(InvalidColumns)
   let finalTables: { [key: string]: any } = {}
   const errors: { [key: string]: string } = {}
+  // @ts-ignore
+  const tableIds: [string] = Object.values(tables).map(table => table._id)
   for (let [name, table] of Object.entries(tables)) {
     const schemaFields = Object.keys(table.schema)
     // make sure every table has a key
@@ -185,7 +246,7 @@ export function finaliseExternalTables(
       continue
     }
     // make sure all previous props have been added back
-    finalTables[name] = copyExistingPropsOver(name, table, entities)
+    finalTables[name] = copyExistingPropsOver(name, table, entities, tableIds)
   }
   // sort the tables by name
   finalTables = Object.entries(finalTables)