@budibase/server 2.7.7-alpha.3 → 2.7.7-alpha.4

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.7.7-alpha.3",
+  "version": "2.7.7-alpha.4",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -46,12 +46,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.7.7-alpha.3",
-    "@budibase/client": "2.7.7-alpha.3",
-    "@budibase/pro": "2.7.7-alpha.3",
-    "@budibase/shared-core": "2.7.7-alpha.3",
-    "@budibase/string-templates": "2.7.7-alpha.3",
-    "@budibase/types": "2.7.7-alpha.3",
+    "@budibase/backend-core": "2.7.7-alpha.4",
+    "@budibase/client": "2.7.7-alpha.4",
+    "@budibase/pro": "2.7.7-alpha.4",
+    "@budibase/shared-core": "2.7.7-alpha.4",
+    "@budibase/string-templates": "2.7.7-alpha.4",
+    "@budibase/types": "2.7.7-alpha.4",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -195,5 +195,5 @@
       }
     }
   },
-  "gitHead": "4066481ab4d8e674be19bed0517878f7f9c276f9"
+  "gitHead": "a8cf83aeb0c37afa6e3df230a5235c0ae18d98f6"
 }

package/src/api/controllers/datasource.ts

@@ -103,6 +103,22 @@ async function buildSchemaHelper(datasource: Datasource) {
   return { tables: connector.tables, error }
 }
 
+async function buildFilteredSchema(datasource: Datasource, filter?: string[]) {
+  let { tables, error } = await buildSchemaHelper(datasource)
+  let finalTables = tables
+  if (filter) {
+    finalTables = {}
+    for (let key in tables) {
+      if (
+        filter.some((filter: any) => filter.toLowerCase() === key.toLowerCase())
+      ) {
+        finalTables[key] = tables[key]
+      }
+    }
+  }
+  return { tables: finalTables, error }
+}
+
 export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
@@ -174,43 +190,28 @@ export async function information(
   }
   const tableNames = await connector.getTableNames()
   ctx.body = {
-    tableNames,
+    tableNames: tableNames.sort(),
   }
 }
 
 export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
-  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
   const tablesFilter = ctx.request.body.tablesFilter
+  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
 
-  let { tables, error } = await buildSchemaHelper(datasource)
-  if (tablesFilter) {
-    if (!datasource.entities) {
-      datasource.entities = {}
-    }
-    for (let key in tables) {
-      if (
-        tablesFilter.some(
-          (filter: any) => filter.toLowerCase() === key.toLowerCase()
-        )
-      ) {
-        datasource.entities[key] = tables[key]
-      }
-    }
-  } else {
-    datasource.entities = tables
-  }
+  const { tables, error } = await buildFilteredSchema(datasource, tablesFilter)
+  datasource.entities = tables
 
   setDefaultDisplayColumns(datasource)
   const dbResp = await db.put(datasource)
   datasource._rev = dbResp.rev
   const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource)
 
-  const response: any = { datasource: cleanedDatasource }
+  const res: any = { datasource: cleanedDatasource }
   if (error) {
-    response.error = error
+    res.error = error
   }
-  ctx.body = response
+  ctx.body = res
 }
 
 /**
@@ -320,6 +321,7 @@ export async function save(
   const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
+  const tablesFilter = ctx.request.body.tablesFilter
 
   const datasource = {
     _id: generateDatasourceID({ plus }),
@@ -329,7 +331,10 @@ export async function save(
 
   let schemaError = null
   if (fetchSchema) {
-    const { tables, error } = await buildSchemaHelper(datasource)
+    const { tables, error } = await buildFilteredSchema(
+      datasource,
+      tablesFilter
+    )
     schemaError = error
     datasource.entities = tables
     setDefaultDisplayColumns(datasource)

package/src/integrations/googlesheets.ts

@@ -21,7 +21,7 @@ import { buildExternalTableId, finaliseExternalTables } from "./utils"
 import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { cache, configs, context, HTTPError } from "@budibase/backend-core"
-import { dataFilters } from "@budibase/shared-core"
+import { dataFilters, utils } from "@budibase/shared-core"
 import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
 import sdk from "../sdk"
 
@@ -150,7 +150,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async testConnection(): Promise<ConnectionInfo> {
     try {
-      await setupCreationAuth(this.config)
       await this.connect()
       return { connected: true }
     } catch (e: any) {
@@ -211,6 +210,8 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async connect() {
     try {
+      await setupCreationAuth(this.config)
+
       // Initialise oAuth client
       let googleConfig = await configs.getGoogleDatasourceConfig()
       if (!googleConfig) {
@@ -273,24 +274,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }
 
   async buildSchema(datasourceId: string, entities: Record<string, Table>) {
-    // not fully configured yet
-    if (!this.config.auth) {
-      return
-    }
     await this.connect()
     const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
-    for (let sheet of sheets) {
-      // must fetch rows to determine schema
-      await sheet.getRows()
-
-      const id = buildExternalTableId(datasourceId, sheet.title)
-      tables[sheet.title] = this.getTableSchema(
-        sheet.title,
-        sheet.headerValues,
-        id
-      )
-    }
+    await utils.parallelForeach(
+      sheets,
+      async sheet => {
+        // must fetch rows to determine schema
+        await sheet.getRows({ limit: 0, offset: 0 })
+
+        const id = buildExternalTableId(datasourceId, sheet.title)
+        tables[sheet.title] = this.getTableSchema(
+          sheet.title,
+          sheet.headerValues,
+          id
+        )
+      },
+      10
+    )
     const final = finaliseExternalTables(tables, entities)
     this.tables = final.tables
     this.schemaErrors = final.errors

package/src/integrations/postgres.ts

@@ -322,7 +322,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
       await this.openConnection()
       const columnsResponse: { rows: PostgresColumn[] } =
         await this.client.query(this.COLUMNS_SQL)
-      return columnsResponse.rows.map(row => row.table_name)
+      const names = columnsResponse.rows.map(row => row.table_name)
+      return [...new Set(names)]
     } finally {
       await this.closeConnection()
     }

package/src/sdk/app/datasources/datasources.ts

@@ -164,5 +164,6 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
       delete update.config[key]
     }
   }
+
   return update
 }