@budibase/server 1.0.211 → 1.0.212-alpha.2

package/dist/utilities/fileSystem/index.js

@@ -17,7 +17,7 @@ const { ObjectStoreBuckets } = require("../../constants");
 const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
 const env = require("../../environment");
-const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, } = require("../../db/utils");
+const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, } = require("../../db/utils");
 const MemoryStream = require("memorystream");
 const { getAppId } = require("@budibase/backend-core/context");
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..");
@@ -98,6 +98,17 @@ exports.apiFileReturn = contents => {
     fs.writeFileSync(path, contents);
     return fs.createReadStream(path);
 };
+exports.defineFilter = excludeRows => {
+    if (excludeRows) {
+        return doc => !(doc._id.includes(USER_METDATA_PREFIX) ||
+            doc._id.includes(LINK_USER_METADATA_PREFIX) ||
+            doc._id.includes(TABLE_ROW_PREFIX));
+    }
+    else if (!excludeRows) {
+        return doc => !(doc._id.includes(USER_METDATA_PREFIX) ||
+            doc._id.includes(LINK_USER_METADATA_PREFIX));
+    }
+};
 /**
  * Local utility to back up the database state for an app, excluding global user
  * data or user relationships.
@@ -105,9 +116,8 @@ exports.apiFileReturn = contents => {
  * @param {object} config Config to send to export DB
  * @returns {*} either a string or a stream of the backup
  */
-const backupAppData = (appId, config) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: doc => !(doc._id.includes(USER_METDATA_PREFIX) ||
-            doc._id.includes(LINK_USER_METADATA_PREFIX)) }));
+const backupAppData = (appId, config, includeRows) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: exports.defineFilter(includeRows) }));
 });
 /**
  * Takes a copy of the database state for an app to the object store.
@@ -123,8 +133,8 @@ exports.performBackup = (appId, backupName) => __awaiter(void 0, void 0, void 0,
  * @param {string} appId The ID of the app which is to be backed up.
  * @returns {*} a readable stream of the backup which is written in real time
  */
-exports.streamBackup = (appId) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield backupAppData(appId, { stream: true });
+exports.streamBackup = (appId, includeRows) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield backupAppData(appId, { stream: true }, includeRows);
 });
 /**
  * Exports a DB to either file or a variable (memory).

package/dist/utilities/fileSystem/utilities.js

@@ -1,5 +1,5 @@
 "use strict";
-const { ObjectStore, makeSureBucketExists, upload, deleteFiles, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, } = require("@budibase/backend-core/objectStore");
+const { ObjectStore, makeSureBucketExists, upload, deleteFiles, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, downloadTarballDirect, } = require("@budibase/backend-core/objectStore");
 /***********************************
  *              NOTE               *
  * This file purely exists so that *
@@ -17,4 +17,5 @@ exports.retrieveToTmp = retrieveToTmp;
 exports.deleteFolder = deleteFolder;
 exports.uploadDirectory = uploadDirectory;
 exports.downloadTarball = downloadTarball;
+exports.downloadTarballDirect = downloadTarballDirect;
 exports.deleteFiles = deleteFiles;

package/dist/utilities/index.js

@@ -57,8 +57,12 @@ exports.objectStoreUrl = () => {
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */
-exports.clientLibraryPath = (appId, version) => {
+exports.clientLibraryPath = (appId, version, ctx) => {
     if (env.isProd()) {
+        // TODO: remove - for beta testing UI
+        if (ctx && ctx.cookies.get("beta:design_ui")) {
+            return "https://cdn.budi.live/beta:design_ui/budibase-client.js";
+        }
         let url = `${exports.objectStoreUrl()}/${sanitizeKey(appId)}/budibase-client.js`;
         // append app version to bust the cache
         if (version) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.211",
+  "version": "1.0.212-alpha.2",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,10 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^1.0.211",
-    "@budibase/client": "^1.0.211",
-    "@budibase/pro": "1.0.210",
-    "@budibase/string-templates": "^1.0.211",
+    "@budibase/backend-core": "^1.0.212-alpha.2",
+    "@budibase/client": "^1.0.212-alpha.2",
+    "@budibase/pro": "1.0.212-alpha.1",
+    "@budibase/string-templates": "^1.0.212-alpha.2",
+    "@budibase/types": "^1.0.212-alpha.2",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -135,6 +136,7 @@
     "pouchdb-replication-stream": "1.2.9",
     "redis": "4",
     "server-destroy": "1.0.1",
+    "snowflake-promise": "^4.5.0",
     "svelte": "3.44.1",
     "swagger-parser": "10.0.3",
     "to-json-schema": "0.2.5",
@@ -150,7 +152,6 @@
     "@babel/core": "7.17.4",
     "@babel/preset-env": "7.16.11",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/types": "^1.0.211",
     "@jest/test-sequencer": "24.9.0",
     "@types/apidoc": "0.50.0",
     "@types/bson": "4.2.0",
@@ -194,5 +195,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "e5c895c869a4cf8fba86e759a6812b4c3bf7c953"
+  "gitHead": "fd7ab042b5b5525160be8849102b661d8be4cf90"
 }

package/scripts/integrations/oracle/instantclient/linux/arm64/install.sh

@@ -0,0 +1,23 @@
+#!/bin/bash 
+
+# Must be root to continue
+if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
+
+# Allow for re-runs
+rm -rf /opt/oracle
+
+echo "Installing oracle instant client"
+
+# copy and unzip package
+mkdir -p /opt/oracle
+cp scripts/integrations/oracle/instantclient/linux/arm64/basiclite-19.10.zip /opt/oracle
+cd /opt/oracle
+unzip -qq basiclite-19.10.zip -d .
+rm *.zip
+mv instantclient* instantclient
+
+# update runtime link path
+sh -c "echo /opt/oracle/instantclient > /etc/ld.so.conf.d/oracle-instantclient.conf"
+ldconfig /etc/ld.so.conf.d
+
+echo "Installation complete"

package/scripts/integrations/oracle/instantclient/linux/install.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )"
+if [[ $TARGETARCH == arm* ]] ;
+then
+  echo "Installing ARM Oracle instant client..."
+  $SCRIPT_DIR/arm64/install.sh
+else
+  echo "Installing x86-64 Oracle instant client..."
+  $SCRIPT_DIR/x86-64/install.sh
+fi

package/src/api/controllers/application.ts

@@ -227,7 +227,11 @@ export const fetchAppPackage = async (ctx: any) => {
     application,
     screens,
     layouts,
-    clientLibPath: clientLibraryPath(ctx.params.appId, application.version),
+    clientLibPath: clientLibraryPath(
+      ctx.params.appId,
+      application.version,
+      ctx
+    ),
   }
 }
 

package/src/api/controllers/backup.js

@@ -3,11 +3,12 @@ const { events, context } = require("@budibase/backend-core")
 const { DocumentTypes } = require("../../db/utils")
 
 exports.exportAppDump = async function (ctx) {
-  const { appId } = ctx.query
+  let { appId, excludeRows } = ctx.query
   const appName = decodeURI(ctx.query.appname)
+  excludeRows = excludeRows === "true"
   const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
   ctx.attachment(backupIdentifier)
-  ctx.body = await streamBackup(appId)
+  ctx.body = await streamBackup(appId, excludeRows)
 
   await context.doInAppContext(appId, async () => {
     const appDb = context.getAppDB()

package/src/api/controllers/dev.js

@@ -3,8 +3,12 @@ const env = require("../../environment")
 const { checkSlashesInUrl } = require("../../utilities")
 const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
-const { Replication, getProdAppID } = require("@budibase/backend-core/db")
-const { DocumentTypes } = require("../../db/utils")
+const {
+  Replication,
+  getProdAppID,
+  dangerousGetDB,
+} = require("@budibase/backend-core/db")
+const { DocumentTypes, getRowParams } = require("../../db/utils")
 const { app: appCache } = require("@budibase/backend-core/cache")
 const { getProdAppDB, getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
@@ -133,3 +137,32 @@ exports.getBudibaseVersion = async ctx => {
   }
   await events.installation.versionChecked(version)
 }
+
+// TODO: remove as part of beta program
+exports.checkBetaAccess = async ctx => {
+  const userToCheck = ctx.query.email
+  const BETA_USERS_DB = "app_bb_f9b77d06b9db4e3ca185476ab87a2364"
+  const BETA_USERS_TABLE = "ta_8c2c6df1c03f49cfb6340e85e066dd15"
+
+  try {
+    const db = dangerousGetDB(BETA_USERS_DB)
+    const betaUsers = (
+      await db.allDocs(
+        getRowParams(BETA_USERS_TABLE, null, {
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+
+    let access = false
+    for (let betaUser of betaUsers) {
+      if (betaUser["Email address"].trim() === userToCheck) {
+        access = true
+        break
+      }
+    }
+    ctx.body = { access }
+  } catch (err) {
+    ctx.body = { access: false }
+  }
+}

package/src/api/controllers/row/external.js

@@ -155,34 +155,44 @@ exports.validate = async () => {
 }
 
 exports.exportRows = async ctx => {
-  const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
+  const { datasourceId } = breakExternalTableId(ctx.params.tableId)
   const db = getAppDB()
-  let format = ctx.query.format
+  const format = ctx.query.format
+  const { columns } = ctx.request.body
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
-  const tables = datasource.entities
-  const table = tables[tableName]
   ctx.request.body = {
     query: {
       oneOf: {
-        [table.primaryDisplay]: ctx.request.body.rows.map(
-          id => breakRowIdField(id)[0]
-        ),
+        _id: ctx.request.body.rows.map(row => JSON.parse(decodeURI(row))[0]),
       },
     },
   }
 
   let result = await exports.search(ctx)
+  let rows = []
+
+  // Filter data to only specified columns if required
+  if (columns && columns.length) {
+    for (let i = 0; i < result.rows.length; i++) {
+      rows[i] = {}
+      for (let column of columns) {
+        rows[i][column] = result.rows[i][column]
+      }
+    }
+  } else {
+    rows = result.rows
+  }
 
-  let headers = Object.keys(result.rows[0])
+  let headers = Object.keys(rows[0])
   const exporter = exporters[format]
   const filename = `export.${format}`
 
   // send down the file
   ctx.attachment(filename)
-  return apiFileReturn(exporter(headers, result.rows))
+  return apiFileReturn(exporter(headers, rows))
 }
 
 exports.fetchEnrichedRow = async ctx => {

package/src/api/controllers/row/utils.js

@@ -64,6 +64,9 @@ exports.validate = async ({ tableId, row, table }) => {
     // Validate.js doesn't seem to handle array
     if (type === FieldTypes.ARRAY && row[fieldName]) {
       if (row[fieldName].length) {
+        if (!Array.isArray(row[fieldName])) {
+          row[fieldName] = row[fieldName].split(",")
+        }
         row[fieldName].map(val => {
           if (
             !constraints.inclusion.includes(val) &&

package/src/api/controllers/static/index.js

@@ -16,9 +16,15 @@ const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes, isDevAppID } = require("../../../db/utils")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
+const { setCookie, clearCookie } = require("@budibase/backend-core/utils")
 const AWS = require("aws-sdk")
 const { events } = require("@budibase/backend-core")
 
+const fs = require("fs")
+const {
+  downloadTarballDirect,
+} = require("../../../utilities/fileSystem/utilities")
+
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
     bucket,
@@ -38,8 +44,41 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
   }
 }
 
+exports.toggleBetaUiFeature = async function (ctx) {
+  const cookieName = `beta:${ctx.params.feature}`
+
+  if (ctx.cookies.get(cookieName)) {
+    clearCookie(ctx, cookieName)
+    ctx.body = {
+      message: `${ctx.params.feature} disabled`,
+    }
+    return
+  }
+
+  let builderPath = resolve(TOP_LEVEL_PATH, "new_design_ui")
+
+  // // download it from S3
+  if (!fs.existsSync(builderPath)) {
+    fs.mkdirSync(builderPath)
+  }
+  await downloadTarballDirect(
+    "https://cdn.budi.live/beta:design_ui/new_ui.tar.gz",
+    builderPath
+  )
+  setCookie(ctx, {}, cookieName)
+
+  ctx.body = {
+    message: `${ctx.params.feature} enabled`,
+  }
+}
+
 exports.serveBuilder = async function (ctx) {
-  let builderPath = resolve(TOP_LEVEL_PATH, "builder")
+  // Temporary: New Design UI
+  const designUiCookie = ctx.cookies.get("beta:design_ui")
+  // TODO: get this from the tmp Dir that we downloaded from MinIO
+  const uiPath = designUiCookie ? "new_design_ui" : "builder"
+
+  let builderPath = resolve(TOP_LEVEL_PATH, uiPath)
   await send(ctx, ctx.file, { root: builderPath })
   if (!ctx.file.includes("assets/")) {
     await events.serve.servedBuilder()
@@ -78,7 +117,7 @@ exports.serveApp = async function (ctx) {
       title: appInfo.name,
       production: env.isProd(),
       appId,
-      clientLibPath: clientLibraryPath(appId, appInfo.version),
+      clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
     })
 
     const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)

package/src/api/routes/dev.js

@@ -22,5 +22,6 @@ router
   .get("/api/dev/version", authorized(BUILDER), controller.getBudibaseVersion)
   .delete("/api/dev/:appId/lock", authorized(BUILDER), controller.clearLock)
   .post("/api/dev/:appId/revert", authorized(BUILDER), controller.revert)
+  .get("/api/beta/access", controller.checkBetaAccess)
 
 module.exports = router

package/src/api/routes/static.ts

@@ -38,6 +38,7 @@ router
   // TODO: for now this builder endpoint is not authorized/secured, will need to be
   .get("/builder/:file*", controller.serveBuilder)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
+  .post("/api/beta/:feature", controller.toggleBetaUiFeature)
   .post(
     "/api/attachments/:tableId/upload",
     paramResource("tableId"),

package/src/automations/automationUtils.js

@@ -1,5 +1,10 @@
 const { getTable } = require("../api/controllers/table/utils")
-const { findHBSBlocks } = require("@budibase/string-templates")
+const {
+  findHBSBlocks,
+  decodeJSBinding,
+  isJSBinding,
+  encodeJSBinding,
+} = require("@budibase/string-templates")
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -77,11 +82,21 @@ exports.getError = err => {
 }
 
 exports.substituteLoopStep = (hbsString, substitute) => {
-  let blocks = findHBSBlocks(hbsString)
+  let blocks = []
+  let checkForJS = isJSBinding(hbsString)
+  if (checkForJS) {
+    hbsString = decodeJSBinding(hbsString)
+    blocks.push(hbsString)
+  } else {
+    blocks = findHBSBlocks(hbsString)
+  }
   for (let block of blocks) {
-    let oldBlock = block
     block = block.replace(/loop/, substitute)
-    hbsString = hbsString.replace(new RegExp(oldBlock, "g"), block)
+    if (checkForJS) {
+      hbsString = encodeJSBinding(block)
+    } else {
+      hbsString = block
+    }
   }
 
   return hbsString

package/src/automations/triggers.js

@@ -97,7 +97,7 @@ exports.externalTrigger = async function (
     // values are likely to be submitted as strings, so we shall convert to correct type
     const coercedFields = {}
     const fields = automation.definition.trigger.inputs.fields
-    for (let key of Object.keys(fields)) {
+    for (let key of Object.keys(fields || {})) {
       coercedFields[key] = coerce(params.fields[key], fields[key])
     }
     params.fields = coercedFields

package/src/db/utils.js

@@ -74,6 +74,7 @@ exports.isDevAppID = isDevAppID
 exports.isProdAppID = isProdAppID
 exports.USER_METDATA_PREFIX = `${DocumentTypes.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
 exports.LINK_USER_METADATA_PREFIX = `${DocumentTypes.LINK}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
+exports.TABLE_ROW_PREFIX = `${DocumentTypes.ROW}${SEPARATOR}${DocumentTypes.TABLE}`
 exports.ViewNames = ViewNames
 exports.InternalTables = InternalTables
 exports.DocumentTypes = DocumentTypes

package/src/definitions/common.ts

@@ -12,6 +12,7 @@ export interface Application extends Base {
 export interface FieldSchema {
   // TODO: replace with field types enum when done
   type: string
+  externalType?: string
   fieldName?: string
   name: string
   tableId?: string

package/src/definitions/datasource.ts

@@ -50,6 +50,7 @@ export enum SourceNames {
   GOOGLE_SHEETS = "GOOGLE_SHEETS",
   FIRESTORE = "FIRESTORE",
   REDIS = "REDIS",
+  SNOWFLAKE = "SNOWFLAKE",
 }
 
 export enum IncludeRelationships {
@@ -93,6 +94,7 @@ export interface Integration {
   relationships?: boolean
   description: string
   friendlyName: string
+  type?: string
   datasource: {}
   query: {
     [key: string]: QueryDefinition

package/src/environment.js

@@ -67,7 +67,8 @@ module.exports = {
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,
   LOG_LEVEL: process.env.LOG_LEVEL,
-  AUTOMATION_MAX_ITERATIONS: process.env.AUTOMATION_MAX_ITERATIONS,
+  AUTOMATION_MAX_ITERATIONS:
+    parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
   QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT),

package/src/integrations/airtable.ts

@@ -18,6 +18,7 @@ module AirtableModule {
     description:
       "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.",
     friendlyName: "Airtable",
+    type: "Spreadsheet",
     datasource: {
       apiKey: {
         type: DatasourceFieldTypes.PASSWORD,

package/src/integrations/arangodb.ts

@@ -19,6 +19,7 @@ module ArangoModule {
   const SCHEMA: Integration = {
     docs: "https://github.com/arangodb/arangojs",
     friendlyName: "ArangoDB",
+    type: "Non-relational",
     description:
       "ArangoDB is a scalable open-source multi-model database natively supporting graph, document and search. All supported data models & access patterns can be combined in queries allowing for maximal flexibility. ",
     datasource: {

package/src/integrations/base/sql.ts

@@ -89,6 +89,28 @@ function parseFilters(filters: SearchFilters | undefined): SearchFilters {
   return filters
 }
 
+function generateSelectStatement(
+  json: QueryJson,
+  knex: Knex
+): (string | Knex.Raw)[] {
+  const { resource, meta } = json
+  const schema = meta?.table?.schema
+  return resource.fields.map(field => {
+    const fieldNames = field.split(/\./g)
+    const tableName = fieldNames[0]
+    const columnName = fieldNames[1]
+    if (columnName && knex.client.config.client === SqlClients.POSTGRES) {
+      const externalType = schema?.[columnName].externalType
+      if (externalType?.includes("money")) {
+        return knex.raw(
+          `"${tableName}"."${columnName}"::money::numeric as "${field}"`
+        )
+      }
+    }
+    return `${field} as ${field}`
+  })
+}
+
 class InternalBuilder {
   private readonly client: string
 
@@ -216,9 +238,7 @@ class InternalBuilder {
   }
 
   addRelationships(
-    knex: Knex,
     query: KnexQuery,
-    fields: string | string[],
     fromTable: string,
     relationships: RelationshipsJson[] | undefined
   ): KnexQuery {
@@ -323,12 +343,12 @@ class InternalBuilder {
     if (!resource) {
       resource = { fields: [] }
     }
-    let selectStatement: string | string[] = "*"
+    let selectStatement: string | (string | Knex.Raw)[] = "*"
     // handle select
     if (resource.fields && resource.fields.length > 0) {
       // select the resources as the format "table.columnName" - this is what is provided
       // by the resource builder further up
-      selectStatement = resource.fields.map(field => `${field} as ${field}`)
+      selectStatement = generateSelectStatement(json, knex)
     }
     let foundLimit = limit || BASE_LIMIT
     // handle pagination
@@ -363,13 +383,7 @@ class InternalBuilder {
       preQuery = this.addSorting(preQuery, json)
     }
     // handle joins
-    query = this.addRelationships(
-      knex,
-      preQuery,
-      selectStatement,
-      tableName,
-      relationships
-    )
+    query = this.addRelationships(preQuery, tableName, relationships)
     return this.addFilters(query, filters, { relationship: true })
   }
 

package/src/integrations/couchdb.ts

@@ -16,6 +16,7 @@ module CouchDBModule {
   const SCHEMA: Integration = {
     docs: "https://docs.couchdb.org/en/stable/",
     friendlyName: "CouchDB",
+    type: "Non-relational",
     description:
       "Apache CouchDB is an open-source document-oriented NoSQL database, implemented in Erlang.",
     datasource: {

package/src/integrations/dynamodb.ts

@@ -21,6 +21,7 @@ module DynamoModule {
     description:
       "Amazon DynamoDB is a key-value and document database that delivers single-digit millisecond performance at any scale.",
     friendlyName: "DynamoDB",
+    type: "Non-relational",
     datasource: {
       region: {
         type: DatasourceFieldTypes.STRING,

package/src/integrations/elasticsearch.ts

@@ -17,6 +17,7 @@ module ElasticsearchModule {
     description:
       "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.",
     friendlyName: "ElasticSearch",
+    type: "Non-relational",
     datasource: {
       url: {
         type: DatasourceFieldTypes.STRING,

package/src/integrations/firebase.ts

@@ -16,6 +16,7 @@ module Firebase {
   const SCHEMA: Integration = {
     docs: "https://firebase.google.com/docs/firestore/quickstart",
     friendlyName: "Firestore",
+    type: "Non-relational",
     description:
       "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.",
     datasource: {

package/src/integrations/googlesheets.ts

@@ -49,6 +49,7 @@ module GoogleSheetsModule {
     description:
       "Create and collaborate on online spreadsheets in real-time and from any device. ",
     friendlyName: "Google Sheets",
+    type: "Spreadsheet",
     datasource: {
       spreadsheetId: {
         display: "Google Sheet URL",

package/src/integrations/index.ts

@@ -12,6 +12,7 @@ const rest = require("./rest")
 const googlesheets = require("./googlesheets")
 const firebase = require("./firebase")
 const redis = require("./redis")
+const snowflake = require("./snowflake")
 const { SourceNames } = require("../definitions/datasource")
 const environment = require("../environment")
 
@@ -29,6 +30,7 @@ const DEFINITIONS = {
   [SourceNames.REST]: rest.schema,
   [SourceNames.FIRESTORE]: firebase.schema,
   [SourceNames.REDIS]: redis.schema,
+  [SourceNames.SNOWFLAKE]: snowflake.schema,
 }
 
 const INTEGRATIONS = {
@@ -46,6 +48,8 @@ const INTEGRATIONS = {
   [SourceNames.FIRESTORE]: firebase.integration,
   [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
   [SourceNames.REDIS]: redis.integration,
+  [SourceNames.FIREBASE]: firebase.integration,
+  [SourceNames.SNOWFLAKE]: snowflake.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/integrations/microsoftSqlServer.ts

@@ -44,6 +44,7 @@ module MSSQLModule {
     description:
       "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
     friendlyName: "MS SQL Server",
+    type: "Relational",
     datasource: {
       user: {
         type: DatasourceFieldTypes.STRING,
@@ -246,6 +247,7 @@ module MSSQLModule {
             autocolumn: !!autoColumns.find((col: string) => col === name),
             name: name,
             ...convertSqlType(def.DATA_TYPE),
+            externalType: def.DATA_TYPE,
           }
         }
         tables[tableName] = {