@budibase/server 1.0.46-alpha.5 → 1.0.46-alpha.6

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.46-alpha.5",
+  "version": "1.0.46-alpha.6",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.46-alpha.5",
-    "@budibase/client": "^1.0.46-alpha.5",
-    "@budibase/string-templates": "^1.0.46-alpha.5",
+    "@budibase/backend-core": "^1.0.46-alpha.6",
+    "@budibase/client": "^1.0.46-alpha.6",
+    "@budibase/string-templates": "^1.0.46-alpha.6",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -92,6 +92,8 @@
     "fix-path": "3.0.0",
     "form-data": "^4.0.0",
     "fs-extra": "8.1.0",
+    "google-auth-library": "^7.11.0",
+    "google-spreadsheet": "^3.2.0",
     "jimp": "0.16.1",
     "joi": "17.2.1",
     "js-yaml": "^4.1.0",
@@ -139,6 +141,7 @@
     "@jest/test-sequencer": "^24.8.0",
     "@types/apidoc": "^0.50.0",
     "@types/bull": "^3.15.1",
+    "@types/google-spreadsheet": "^3.1.5",
     "@types/jest": "^26.0.23",
     "@types/koa": "^2.13.3",
     "@types/koa-router": "^7.4.2",
@@ -165,5 +168,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "cbfb0f8a13790f223580d3fee84e334eeda5b56e"
+  "gitHead": "a3e5c5af7f047b46ba1d7875ad489f357632b267"
 }

package/src/api/controllers/datasource.js

@@ -38,6 +38,13 @@ exports.fetch = async function (ctx) {
     )
   ).rows.map(row => row.doc)
 
+  for (let datasource of datasources) {
+    if (datasource.config && datasource.config.auth) {
+      // strip secrets from response so they don't show in the network request
+      delete datasource.config.auth
+    }
+  }
+
   ctx.body = [bbInternalDb, ...datasources]
 }
 
@@ -94,8 +101,13 @@ exports.update = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const datasourceId = ctx.params.datasourceId
   let datasource = await db.get(datasourceId)
+  const auth = datasource.config.auth
   await invalidateVariables(datasource, ctx.request.body)
   datasource = { ...datasource, ...ctx.request.body }
+  if (auth && !ctx.request.body.auth) {
+    // don't strip auth config from DB
+    datasource.config.auth = auth
+  }
 
   const response = await db.put(datasource)
   datasource._rev = response.rev

package/src/api/controllers/query/index.js

@@ -141,6 +141,16 @@ async function execute(ctx, opts = { rowsOnly: false }) {
   const query = await db.get(ctx.params.queryId)
   const datasource = await db.get(query.datasourceId)
 
+  const enrichedParameters = ctx.request.body.parameters || {}
+  // make sure parameters are fully enriched with defaults
+  if (query && query.parameters) {
+    for (let parameter of query.parameters) {
+      if (!enrichedParameters[parameter.name]) {
+        enrichedParameters[parameter.name] = parameter.default
+      }
+    }
+  }
+
   // call the relevant CRUD method on the integration class
   try {
     const { rows, pagination, extra } = await Runner.run({
@@ -149,7 +159,7 @@ async function execute(ctx, opts = { rowsOnly: false }) {
       queryVerb: query.queryVerb,
       fields: query.fields,
       pagination: ctx.request.body.pagination,
-      parameters: ctx.request.body.parameters,
+      parameters: enrichedParameters,
       transformer: query.transformer,
       queryId: ctx.params.queryId,
     })
@@ -178,8 +188,9 @@ const removeDynamicVariables = async (db, queryId) => {
 
   if (dynamicVariables) {
     // delete dynamic variables from the datasource
-    const newVariables = dynamicVariables.filter(dv => dv.queryId !== queryId)
-    datasource.config.dynamicVariables = newVariables
+    datasource.config.dynamicVariables = dynamicVariables.filter(
+      dv => dv.queryId !== queryId
+    )
     await db.put(datasource)
 
     // invalidate the deleted variables

package/src/api/controllers/row/utils.js

@@ -52,10 +52,7 @@ exports.validate = async ({ appId, tableId, row, table }) => {
     const constraints = cloneDeep(table.schema[fieldName].constraints)
     const type = table.schema[fieldName].type
     // special case for options, need to always allow unselected (null)
-    if (
-      (type === FieldTypes.OPTIONS || type === FieldTypes.ARRAY) &&
-      constraints.inclusion
-    ) {
+    if (type === FieldTypes.OPTIONS && constraints.inclusion) {
       constraints.inclusion.push(null)
     }
     let res

package/src/constants/index.js

@@ -75,6 +75,10 @@ exports.DataSourceOperation = {
   DELETE_TABLE: "DELETE_TABLE",
 }
 
+exports.DatasourceAuthTypes = {
+  GOOGLE: "google",
+}
+
 exports.SortDirection = {
   ASCENDING: "ASCENDING",
   DESCENDING: "DESCENDING",

package/src/definitions/datasource.ts

@@ -47,6 +47,7 @@ export enum SourceNames {
   ARANGODB = "ARANGODB",
   REST = "REST",
   ORACLE = "ORACLE",
+  GOOGLE_SHEETS = "GOOGLE_SHEETS",
 }
 
 export enum IncludeRelationships {
@@ -86,6 +87,8 @@ export interface ExtraQueryConfig {
 export interface Integration {
   docs: string
   plus?: boolean
+  auth?: { type: string }
+  relationships?: boolean
   description: string
   friendlyName: string
   datasource: {}

package/src/integrations/googlesheets.ts

@@ -0,0 +1,353 @@
+import {
+  DatasourceFieldTypes,
+  Integration,
+  QueryJson,
+  QueryTypes,
+} from "../definitions/datasource"
+import { OAuth2Client } from "google-auth-library"
+import { DatasourcePlus } from "./base/datasourcePlus"
+import { Row, Table, TableSchema } from "../definitions/common"
+import { buildExternalTableId } from "./utils"
+import { DataSourceOperation, FieldTypes } from "../constants"
+import { GoogleSpreadsheet } from "google-spreadsheet"
+import { table } from "console"
+
+module GoogleSheetsModule {
+  const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+  const { getScopedConfig } = require("@budibase/backend-core/db")
+  const { Configs } = require("@budibase/backend-core/constants")
+
+  interface GoogleSheetsConfig {
+    spreadsheetId: string
+    auth: OAuthClientConfig
+  }
+
+  interface OAuthClientConfig {
+    appId: string
+    accessToken: string
+    refreshToken: string
+  }
+
+  const SCHEMA: Integration = {
+    plus: true,
+    auth: {
+      type: "google",
+    },
+    relationships: false,
+    docs: "https://developers.google.com/sheets/api/quickstart/nodejs",
+    description:
+      "Create and collaborate on online spreadsheets in real-time and from any device. ",
+    friendlyName: "Google Sheets",
+    datasource: {
+      spreadsheetId: {
+        type: DatasourceFieldTypes.STRING,
+        required: true,
+      },
+    },
+    query: {
+      create: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          sheet: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          row: {
+            type: QueryTypes.JSON,
+            required: true,
+          },
+        },
+      },
+      read: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          sheet: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+        },
+      },
+      update: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          sheet: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          rowIndex: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          row: {
+            type: QueryTypes.JSON,
+            required: true,
+          },
+        },
+      },
+      delete: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          sheet: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          rowIndex: {
+            type: DatasourceFieldTypes.NUMBER,
+            required: true,
+          },
+        },
+      },
+    },
+  }
+
+  class GoogleSheetsIntegration implements DatasourcePlus {
+    private readonly config: GoogleSheetsConfig
+    private client: any
+    public tables: Record<string, Table> = {}
+    public schemaErrors: Record<string, string> = {}
+
+    constructor(config: GoogleSheetsConfig) {
+      this.config = config
+      const spreadsheetId = this.cleanSpreadsheetUrl(this.config.spreadsheetId)
+      this.client = new GoogleSpreadsheet(spreadsheetId)
+    }
+
+    /**
+     * Pull the spreadsheet ID out from a valid google sheets URL
+     * @param spreadsheetId - the URL or standard spreadsheetId of the google sheet
+     * @returns spreadsheet Id of the google sheet
+     */
+    cleanSpreadsheetUrl(spreadsheetId: string) {
+      if (!spreadsheetId) {
+        throw new Error(
+          "You must set a spreadsheet ID in your configuration to fetch tables."
+        )
+      }
+      const parts = spreadsheetId.split("/")
+      return parts.length > 5 ? parts[5] : spreadsheetId
+    }
+
+    async connect() {
+      try {
+        // Initialise oAuth client
+        const db = getGlobalDB()
+        const googleConfig = await getScopedConfig(db, {
+          type: Configs.GOOGLE,
+        })
+        const oauthClient = new OAuth2Client({
+          clientId: googleConfig.clientID,
+          clientSecret: googleConfig.clientSecret,
+        })
+        oauthClient.credentials.access_token = this.config.auth.accessToken
+        oauthClient.credentials.refresh_token = this.config.auth.refreshToken
+        this.client.useOAuth2Client(oauthClient)
+        await this.client.loadInfo()
+      } catch (err) {
+        console.error("Error connecting to google sheets", err)
+        throw err
+      }
+    }
+
+    async buildSchema(datasourceId: string) {
+      await this.connect()
+      const sheets = await this.client.sheetsByIndex
+      const tables: Record<string, Table> = {}
+      for (let sheet of sheets) {
+        // must fetch rows to determine schema
+        await sheet.getRows()
+        // build schema
+        const schema: TableSchema = {}
+
+        // build schema from headers
+        for (let header of sheet.headerValues) {
+          schema[header] = {
+            name: header,
+            type: FieldTypes.STRING,
+          }
+        }
+
+        // create tables
+        tables[sheet.title] = {
+          _id: buildExternalTableId(datasourceId, sheet.title),
+          name: sheet.title,
+          primary: ["rowNumber"],
+          schema,
+        }
+      }
+
+      this.tables = tables
+    }
+
+    async query(json: QueryJson) {
+      const sheet = json.endpoint.entityId
+
+      const handlers = {
+        [DataSourceOperation.CREATE]: () =>
+          this.create({ sheet, row: json.body }),
+        [DataSourceOperation.READ]: () => this.read({ sheet }),
+        [DataSourceOperation.UPDATE]: () =>
+          this.update({
+            // exclude the header row and zero index
+            rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
+            sheet,
+            row: json.body,
+          }),
+        [DataSourceOperation.DELETE]: () =>
+          this.delete({
+            // exclude the header row and zero index
+            rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
+            sheet,
+          }),
+        [DataSourceOperation.CREATE_TABLE]: () =>
+          this.createTable(json?.table?.name),
+        [DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table),
+        [DataSourceOperation.DELETE_TABLE]: () =>
+          this.deleteTable(json?.table?.name),
+      }
+
+      const internalQueryMethod = handlers[json.endpoint.operation]
+
+      return await internalQueryMethod()
+    }
+
+    buildRowObject(headers: string[], values: string[], rowNumber: number) {
+      const rowObject: { rowNumber: number; [key: string]: any } = { rowNumber }
+      for (let i = 0; i < headers.length; i++) {
+        rowObject._id = rowNumber
+        rowObject[headers[i]] = values[i]
+      }
+      return rowObject
+    }
+
+    async createTable(name?: string) {
+      try {
+        await this.connect()
+        const sheet = await this.client.addSheet({ title: name })
+        return sheet
+      } catch (err) {
+        console.error("Error creating new table in google sheets", err)
+        throw err
+      }
+    }
+
+    async updateTable(table?: any) {
+      try {
+        await this.connect()
+        const sheet = await this.client.sheetsByTitle[table.name]
+        await sheet.loadHeaderRow()
+
+        if (table._rename) {
+          const headers = []
+          for (let header of sheet.headerValues) {
+            if (header === table._rename.old) {
+              headers.push(table._rename.updated)
+            } else {
+              headers.push(header)
+            }
+          }
+          await sheet.setHeaderRow(headers)
+        } else {
+          let newField = Object.keys(table.schema).find(
+            key => !sheet.headerValues.includes(key)
+          )
+          await sheet.setHeaderRow([...sheet.headerValues, newField])
+        }
+      } catch (err) {
+        console.error("Error updating table in google sheets", err)
+        throw err
+      }
+    }
+
+    async deleteTable(sheet: any) {
+      try {
+        await this.connect()
+        const sheetToDelete = await this.client.sheetsByTitle[sheet]
+        return await sheetToDelete.delete()
+      } catch (err) {
+        console.error("Error deleting table in google sheets", err)
+        throw err
+      }
+    }
+
+    async create(query: { sheet: string; row: any }) {
+      try {
+        await this.connect()
+        const sheet = await this.client.sheetsByTitle[query.sheet]
+        const rowToInsert =
+          typeof query.row === "string" ? JSON.parse(query.row) : query.row
+        const row = await sheet.addRow(rowToInsert)
+        return [
+          this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber),
+        ]
+      } catch (err) {
+        console.error("Error writing to google sheets", err)
+        throw err
+      }
+    }
+
+    async read(query: { sheet: string }) {
+      try {
+        await this.connect()
+        const sheet = await this.client.sheetsByTitle[query.sheet]
+        const rows = await sheet.getRows()
+        const headerValues = sheet.headerValues
+        const response = []
+        for (let row of rows) {
+          response.push(
+            this.buildRowObject(headerValues, row._rawData, row._rowNumber)
+          )
+        }
+        return response
+      } catch (err) {
+        console.error("Error reading from google sheets", err)
+        throw err
+      }
+    }
+
+    async update(query: { sheet: string; rowIndex: number; row: any }) {
+      try {
+        await this.connect()
+        const sheet = await this.client.sheetsByTitle[query.sheet]
+        const rows = await sheet.getRows()
+        const row = rows[query.rowIndex]
+        if (row) {
+          const updateValues = query.row
+          for (let key in updateValues) {
+            row[key] = updateValues[key]
+          }
+          await row.save()
+          return [
+            this.buildRowObject(
+              sheet.headerValues,
+              row._rawData,
+              row._rowNumber
+            ),
+          ]
+        } else {
+          throw new Error("Row does not exist.")
+        }
+      } catch (err) {
+        console.error("Error reading from google sheets", err)
+        throw err
+      }
+    }
+
+    async delete(query: { sheet: string; rowIndex: number }) {
+      await this.connect()
+      const sheet = await this.client.sheetsByTitle[query.sheet]
+      const rows = await sheet.getRows()
+      const row = rows[query.rowIndex]
+      if (row) {
+        await row.delete()
+        return [{ deleted: query.rowIndex }]
+      } else {
+        throw new Error("Row does not exist.")
+      }
+    }
+  }
+
+  module.exports = {
+    schema: SCHEMA,
+    integration: GoogleSheetsIntegration,
+  }
+}

package/src/integrations/index.ts

@@ -9,6 +9,7 @@ const airtable = require("./airtable")
 const mysql = require("./mysql")
 const arangodb = require("./arangodb")
 const rest = require("./rest")
+const googlesheets = require("./googlesheets")
 const { SourceNames } = require("../definitions/datasource")
 
 const DEFINITIONS = {
@@ -23,6 +24,7 @@ const DEFINITIONS = {
   [SourceNames.MYSQL]: mysql.schema,
   [SourceNames.ARANGODB]: arangodb.schema,
   [SourceNames.REST]: rest.schema,
+  [SourceNames.GOOGLE_SHEETS]: googlesheets.schema,
 }
 
 const INTEGRATIONS = {
@@ -37,6 +39,7 @@ const INTEGRATIONS = {
   [SourceNames.MYSQL]: mysql.integration,
   [SourceNames.ARANGODB]: arangodb.integration,
   [SourceNames.REST]: rest.integration,
+  [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/integrations/rest.ts

@@ -43,8 +43,8 @@ const coreFields = {
     enum: Object.values(BodyTypes),
   },
   pagination: {
-    type: DatasourceFieldTypes.OBJECT
-  }
+    type: DatasourceFieldTypes.OBJECT,
+  },
 }
 
 module RestModule {
@@ -178,12 +178,17 @@ module RestModule {
           headers,
         },
         pagination: {
-          cursor: nextCursor
-        }
+          cursor: nextCursor,
+        },
       }
     }
 
-    getUrl(path: string, queryString: string, pagination: PaginationConfig | null, paginationValues: PaginationValues | null): string {
+    getUrl(
+      path: string,
+      queryString: string,
+      pagination: PaginationConfig | null,
+      paginationValues: PaginationValues | null
+    ): string {
       // Add pagination params to query string if required
       if (pagination?.location === "query" && paginationValues) {
         const { pageParam, sizeParam } = pagination
@@ -217,14 +222,22 @@ module RestModule {
       return complete
     }
 
-    addBody(bodyType: string, body: string | any, input: any, pagination: PaginationConfig | null, paginationValues: PaginationValues | null) {
+    addBody(
+      bodyType: string,
+      body: string | any,
+      input: any,
+      pagination: PaginationConfig | null,
+      paginationValues: PaginationValues | null
+    ) {
       if (!input.headers) {
         input.headers = {}
       }
       if (bodyType === BodyTypes.NONE) {
         return input
       }
-      let error, object: any = {}, string = ""
+      let error,
+        object: any = {},
+        string = ""
       try {
         if (body) {
           string = typeof body !== "string" ? JSON.stringify(body) : body
@@ -333,7 +346,7 @@ module RestModule {
         requestBody,
         authConfigId,
         pagination,
-        paginationValues
+        paginationValues,
       } = query
       const authHeaders = this.getAuthHeaders(authConfigId)
 
@@ -352,7 +365,13 @@ module RestModule {
       }
 
       let input: any = { method, headers: this.headers }
-      input = this.addBody(bodyType, requestBody, input, pagination, paginationValues)
+      input = this.addBody(
+        bodyType,
+        requestBody,
+        input,
+        pagination,
+        paginationValues
+      )
 
       this.startTimeMs = performance.now()
       const url = this.getUrl(path, queryString, pagination, paginationValues)

package/src/integrations/s3.ts

@@ -38,7 +38,7 @@ module S3Module {
       signatureVersion: {
         type: "string",
         required: false,
-        default: "v4"
+        default: "v4",
       },
     },
     query: {