@budibase/server 2.4.40 → 2.4.41

package/src/integrations/googlesheets.ts

@@ -1,22 +1,25 @@
 import {
   DatasourceFieldType,
   DatasourcePlus,
+  FieldType,
   Integration,
+  Operation,
   PaginationJson,
   QueryJson,
   QueryType,
+  Row,
   SearchFilters,
   SortJson,
   Table,
-  TableSchema,
+  TableRequest,
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
-import { buildExternalTableId } from "./utils"
-import { DataSourceOperation, FieldTypes } from "../constants"
+import { buildExternalTableId, finaliseExternalTables } from "./utils"
 import { GoogleSpreadsheet } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { configs, HTTPError } from "@budibase/backend-core"
 import { dataFilters } from "@budibase/shared-core"
+import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
 
 interface GoogleSheetsConfig {
   spreadsheetId: string
@@ -39,6 +42,17 @@ interface AuthTokenResponse {
   access_token: string
 }
 
+const ALLOWED_TYPES = [
+  FieldType.STRING,
+  FieldType.FORMULA,
+  FieldType.NUMBER,
+  FieldType.LONGFORM,
+  FieldType.DATETIME,
+  FieldType.OPTIONS,
+  FieldType.BOOLEAN,
+  FieldType.BARCODEQR,
+]
+
 const SCHEMA: Integration = {
   plus: true,
   auth: {
@@ -199,73 +213,90 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
       this.client.useOAuth2Client(oauthClient)
       await this.client.loadInfo()
-    } catch (err) {
+    } catch (err: any) {
+      // this happens for xlsx imports
+      if (err.message?.includes("operation is not supported")) {
+        err.message =
+          "This operation is not supported - XLSX sheets must be converted."
+      }
       console.error("Error connecting to google sheets", err)
       throw err
     }
   }
 
-  async buildSchema(datasourceId: string) {
+  getTableSchema(title: string, headerValues: string[], id?: string) {
+    // base table
+    const table: Table = {
+      name: title,
+      primary: [GOOGLE_SHEETS_PRIMARY_KEY],
+      schema: {},
+    }
+    if (id) {
+      table._id = id
+    }
+    // build schema from headers
+    for (let header of headerValues) {
+      table.schema[header] = {
+        name: header,
+        type: FieldType.STRING,
+      }
+    }
+    return table
+  }
+
+  async buildSchema(datasourceId: string, entities: Record<string, Table>) {
     await this.connect()
     const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
     for (let sheet of sheets) {
       // must fetch rows to determine schema
       await sheet.getRows()
-      // build schema
-      const schema: TableSchema = {}
-
-      // build schema from headers
-      for (let header of sheet.headerValues) {
-        schema[header] = {
-          name: header,
-          type: FieldTypes.STRING,
-        }
-      }
 
-      // create tables
-      tables[sheet.title] = {
-        _id: buildExternalTableId(datasourceId, sheet.title),
-        name: sheet.title,
-        primary: ["rowNumber"],
-        schema,
-      }
+      const id = buildExternalTableId(datasourceId, sheet.title)
+      tables[sheet.title] = this.getTableSchema(
+        sheet.title,
+        sheet.headerValues,
+        id
+      )
     }
-
-    this.tables = tables
+    const final = finaliseExternalTables(tables, entities)
+    this.tables = final.tables
+    this.schemaErrors = final.errors
   }
 
   async query(json: QueryJson) {
     const sheet = json.endpoint.entityId
-
-    const handlers = {
-      [DataSourceOperation.CREATE]: () =>
-        this.create({ sheet, row: json.body }),
-      [DataSourceOperation.READ]: () => this.read({ ...json, sheet }),
-      [DataSourceOperation.UPDATE]: () =>
-        this.update({
+    switch (json.endpoint.operation) {
+      case Operation.CREATE:
+        return this.create({ sheet, row: json.body as Row })
+      case Operation.BULK_CREATE:
+        return this.createBulk({ sheet, rows: json.body as Row[] })
+      case Operation.READ:
+        return this.read({ ...json, sheet })
+      case Operation.UPDATE:
+        return this.update({
           // exclude the header row and zero index
           rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
           sheet,
           row: json.body,
-        }),
-      [DataSourceOperation.DELETE]: () =>
-        this.delete({
+        })
+      case Operation.DELETE:
+        return this.delete({
           // exclude the header row and zero index
           rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
           sheet,
-        }),
-      [DataSourceOperation.CREATE_TABLE]: () =>
-        this.createTable(json?.table?.name),
-      [DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table),
-      [DataSourceOperation.DELETE_TABLE]: () =>
-        this.deleteTable(json?.table?.name),
+        })
+      case Operation.CREATE_TABLE:
+        return this.createTable(json?.table?.name)
+      case Operation.UPDATE_TABLE:
+        return this.updateTable(json.table!)
+      case Operation.DELETE_TABLE:
+        return this.deleteTable(json?.table?.name)
+      default:
+        throw new Error(
+          `GSheets integration does not support "${json.endpoint.operation}".`
+        )
     }
-
-    // @ts-ignore
-    const internalQueryMethod = handlers[json.endpoint.operation]
-
-    return await internalQueryMethod()
   }
 
   buildRowObject(headers: string[], values: string[], rowNumber: number) {
@@ -278,47 +309,70 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }
 
   async createTable(name?: string) {
+    if (!name) {
+      throw new Error("Must provide name for new sheet.")
+    }
     try {
       await this.connect()
-      return await this.client.addSheet({ title: name, headerValues: ["test"] })
+      return await this.client.addSheet({ title: name, headerValues: [name] })
     } catch (err) {
       console.error("Error creating new table in google sheets", err)
       throw err
     }
   }
 
-  async updateTable(table?: any) {
-    try {
-      await this.connect()
-      const sheet = this.client.sheetsByTitle[table.name]
-      await sheet.loadHeaderRow()
-
-      if (table._rename) {
-        const headers = []
-        for (let header of sheet.headerValues) {
-          if (header === table._rename.old) {
-            headers.push(table._rename.updated)
-          } else {
-            headers.push(header)
-          }
+  async updateTable(table: TableRequest) {
+    await this.connect()
+    const sheet = this.client.sheetsByTitle[table.name]
+    await sheet.loadHeaderRow()
+
+    if (table._rename) {
+      const headers = []
+      for (let header of sheet.headerValues) {
+        if (header === table._rename.old) {
+          headers.push(table._rename.updated)
+        } else {
+          headers.push(header)
         }
+      }
+      try {
         await sheet.setHeaderRow(headers)
-      } else {
-        const updatedHeaderValues = [...sheet.headerValues]
-
-        const newField = Object.keys(table.schema).find(
-          key => !sheet.headerValues.includes(key)
-        )
+      } catch (err) {
+        console.error("Error updating column name in google sheets", err)
+        throw err
+      }
+    } else {
+      const updatedHeaderValues = [...sheet.headerValues]
+
+      // add new column - doesn't currently exist
+      for (let [key, column] of Object.entries(table.schema)) {
+        if (!ALLOWED_TYPES.includes(column.type)) {
+          throw new Error(
+            `Column type: ${column.type} not allowed for GSheets integration.`
+          )
+        }
+        if (
+          !sheet.headerValues.includes(key) &&
+          column.type !== FieldType.FORMULA
+        ) {
+          updatedHeaderValues.push(key)
+        }
+      }
 
-        if (newField) {
-          updatedHeaderValues.push(newField)
+      // clear out deleted columns
+      for (let key of sheet.headerValues) {
+        if (!Object.keys(table.schema).includes(key)) {
+          const idx = updatedHeaderValues.indexOf(key)
+          updatedHeaderValues.splice(idx, 1)
         }
+      }
 
+      try {
         await sheet.setHeaderRow(updatedHeaderValues)
+      } catch (err) {
+        console.error("Error updating table in google sheets", err)
+        throw err
       }
-    } catch (err) {
-      console.error("Error updating table in google sheets", err)
-      throw err
     }
   }
 
@@ -349,6 +403,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     }
   }
 
+  async createBulk(query: { sheet: string; rows: any[] }) {
+    try {
+      await this.connect()
+      const sheet = this.client.sheetsByTitle[query.sheet]
+      let rowsToInsert = []
+      for (let row of query.rows) {
+        rowsToInsert.push(typeof row === "string" ? JSON.parse(row) : row)
+      }
+      const rows = await sheet.addRows(rowsToInsert)
+      return rows.map(row =>
+        this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber)
+      )
+    } catch (err) {
+      console.error("Error bulk writing to google sheets", err)
+      throw err
+    }
+  }
+
   async read(query: {
     sheet: string
     filters?: SearchFilters

package/src/integrations/utils.ts

@@ -4,6 +4,7 @@ import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
+const ENCODED_SPACE = encodeURIComponent(" ")
 
 const SQL_NUMBER_TYPE_MAP = {
   integer: FieldTypes.NUMBER,
@@ -79,6 +80,10 @@ export function isExternalTable(tableId: string) {
 }
 
 export function buildExternalTableId(datasourceId: string, tableName: string) {
+  // encode spaces
+  if (tableName.includes(" ")) {
+    tableName = encodeURIComponent(tableName)
+  }
   return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
 }
 
@@ -90,6 +95,10 @@ export function breakExternalTableId(tableId: string | undefined) {
   let datasourceId = parts.shift()
   // if they need joined
   let tableName = parts.join(DOUBLE_SEPARATOR)
+  // if contains encoded spaces, decode it
+  if (tableName.includes(ENCODED_SPACE)) {
+    tableName = decodeURIComponent(tableName)
+  }
   return { datasourceId, tableName }
 }
 
@@ -200,9 +209,9 @@ export function isIsoDateString(str: string) {
  * @param column The column to check, to see if it is a valid relationship.
  * @param tableIds The IDs of the tables which currently exist.
  */
-function shouldCopyRelationship(
+export function shouldCopyRelationship(
   column: { type: string; tableId?: string },
-  tableIds: [string]
+  tableIds: string[]
 ) {
   return (
     column.type === FieldTypes.LINK &&
@@ -219,7 +228,7 @@ function shouldCopyRelationship(
  * @param column The column to check for options or boolean type.
  * @param fetchedColumn The fetched column to check for the type in the external database.
  */
-function shouldCopySpecialColumn(
+export function shouldCopySpecialColumn(
   column: { type: string },
   fetchedColumn: { type: string } | undefined
 ) {
@@ -257,9 +266,12 @@ function copyExistingPropsOver(
   tableIds: [string]
 ) {
   if (entities && entities[tableName]) {
-    if (entities[tableName].primaryDisplay) {
+    if (entities[tableName]?.primaryDisplay) {
       table.primaryDisplay = entities[tableName].primaryDisplay
     }
+    if (entities[tableName]?.created) {
+      table.created = entities[tableName]?.created
+    }
     const existingTableSchema = entities[tableName].schema
     for (let key in existingTableSchema) {
       if (!existingTableSchema.hasOwnProperty(key)) {