@budibase/server 1.0.211 → 1.0.212-alpha.10

package/src/integrations/elasticsearch.ts

@@ -17,6 +17,7 @@ module ElasticsearchModule {
     description:
       "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.",
     friendlyName: "ElasticSearch",
+    type: "Non-relational",
     datasource: {
       url: {
         type: DatasourceFieldTypes.STRING,

package/src/integrations/firebase.ts

@@ -16,6 +16,7 @@ module Firebase {
   const SCHEMA: Integration = {
     docs: "https://firebase.google.com/docs/firestore/quickstart",
     friendlyName: "Firestore",
+    type: "Non-relational",
     description:
       "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.",
     datasource: {

package/src/integrations/googlesheets.ts

@@ -49,6 +49,7 @@ module GoogleSheetsModule {
     description:
       "Create and collaborate on online spreadsheets in real-time and from any device. ",
     friendlyName: "Google Sheets",
+    type: "Spreadsheet",
     datasource: {
       spreadsheetId: {
         display: "Google Sheet URL",

package/src/integrations/index.ts

@@ -12,6 +12,7 @@ const rest = require("./rest")
 const googlesheets = require("./googlesheets")
 const firebase = require("./firebase")
 const redis = require("./redis")
+const snowflake = require("./snowflake")
 const { SourceNames } = require("../definitions/datasource")
 const environment = require("../environment")
 
@@ -29,6 +30,7 @@ const DEFINITIONS = {
   [SourceNames.REST]: rest.schema,
   [SourceNames.FIRESTORE]: firebase.schema,
   [SourceNames.REDIS]: redis.schema,
+  [SourceNames.SNOWFLAKE]: snowflake.schema,
 }
 
 const INTEGRATIONS = {
@@ -46,6 +48,8 @@ const INTEGRATIONS = {
   [SourceNames.FIRESTORE]: firebase.integration,
   [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
   [SourceNames.REDIS]: redis.integration,
+  [SourceNames.FIREBASE]: firebase.integration,
+  [SourceNames.SNOWFLAKE]: snowflake.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/integrations/microsoftSqlServer.ts

@@ -44,6 +44,7 @@ module MSSQLModule {
     description:
       "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
     friendlyName: "MS SQL Server",
+    type: "Relational",
     datasource: {
       user: {
         type: DatasourceFieldTypes.STRING,
@@ -246,6 +247,7 @@ module MSSQLModule {
             autocolumn: !!autoColumns.find((col: string) => col === name),
             name: name,
             ...convertSqlType(def.DATA_TYPE),
+            externalType: def.DATA_TYPE,
           }
         }
         tables[tableName] = {

package/src/integrations/mongodb.ts

@@ -24,6 +24,7 @@ module MongoDBModule {
   const SCHEMA: Integration = {
     docs: "https://github.com/mongodb/node-mongodb-native",
     friendlyName: "MongoDB",
+    type: "Non-relational",
     description:
       "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.",
     datasource: {

package/src/integrations/mysql.ts

@@ -36,6 +36,7 @@ module MySQLModule {
     docs: "https://github.com/sidorares/node-mysql2",
     plus: true,
     friendlyName: "MySQL",
+    type: "Relational",
     description:
       "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ",
     datasource: {
@@ -232,6 +233,7 @@ module MySQLModule {
               autocolumn: isAuto,
               constraints,
               ...convertSqlType(column.Type),
+              externalType: column.Type,
             }
           }
           if (!tables[tableName]) {

package/src/integrations/oracle.ts

@@ -40,6 +40,7 @@ module OracleModule {
     docs: "https://github.com/oracle/node-oracledb",
     plus: true,
     friendlyName: "Oracle",
+    type: "Relational",
     description:
       "Oracle Database is an object-relational database management system developed by Oracle Corporation",
     datasource: {

package/src/integrations/postgres.ts

@@ -47,6 +47,7 @@ module PostgresModule {
     docs: "https://node-postgres.com",
     plus: true,
     friendlyName: "PostgreSQL",
+    type: "Relational",
     description:
       "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
     datasource: {
@@ -271,6 +272,7 @@ module PostgresModule {
             autocolumn: isAuto,
             name: columnName,
             ...convertSqlType(column.data_type),
+            externalType: column.data_type,
           }
         }
 

package/src/integrations/redis.ts

@@ -17,6 +17,7 @@ module RedisModule {
     docs: "https://redis.io/docs/",
     description: "",
     friendlyName: "Redis",
+    type: "Non-relational",
     datasource: {
       host: {
         type: "string",

package/src/integrations/rest.ts

@@ -64,6 +64,7 @@ module RestModule {
     description:
       "With the REST API datasource, you can connect, query and pull data from multiple REST APIs. You can then use the retrieved data to build apps.",
     friendlyName: "REST API",
+    type: "API",
     datasource: {
       url: {
         type: DatasourceFieldTypes.STRING,

package/src/integrations/s3.ts

@@ -17,6 +17,7 @@ module S3Module {
     description:
       "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
     friendlyName: "Amazon S3",
+    type: "Object store",
     datasource: {
       region: {
         type: "string",

package/src/integrations/snowflake.ts

@@ -0,0 +1,99 @@
+import { Integration, QueryTypes, SqlQuery } from "../definitions/datasource"
+import { Snowflake } from "snowflake-promise"
+
+module SnowflakeModule {
+  interface SnowflakeConfig {
+    account: string
+    username: string
+    password: string
+    warehouse: string
+    database: string
+    schema: string
+  }
+
+  const SCHEMA: Integration = {
+    docs: "https://developers.snowflake.com/",
+    description:
+      "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.",
+    friendlyName: "Snowflake",
+    type: "Relational",
+    datasource: {
+      account: {
+        type: "string",
+        required: true,
+      },
+      username: {
+        type: "string",
+        required: true,
+      },
+      password: {
+        type: "password",
+        required: true,
+      },
+      warehouse: {
+        type: "string",
+        required: true,
+      },
+      database: {
+        type: "string",
+        required: true,
+      },
+      schema: {
+        type: "string",
+        required: true,
+      },
+    },
+    query: {
+      create: {
+        type: QueryTypes.SQL,
+      },
+      read: {
+        type: QueryTypes.SQL,
+      },
+      update: {
+        type: QueryTypes.SQL,
+      },
+      delete: {
+        type: QueryTypes.SQL,
+      },
+    },
+  }
+
+  class SnowflakeIntegration {
+    private client: Snowflake
+
+    constructor(config: SnowflakeConfig) {
+      this.client = new Snowflake(config)
+    }
+
+    async internalQuery(query: SqlQuery) {
+      await this.client.connect()
+      try {
+        return await this.client.execute(query.sql)
+      } catch (err: any) {
+        throw err?.message.split(":")[1] || err?.message
+      }
+    }
+
+    async create(query: SqlQuery) {
+      return this.internalQuery(query)
+    }
+
+    async read(query: SqlQuery) {
+      return this.internalQuery(query)
+    }
+
+    async update(query: SqlQuery) {
+      return this.internalQuery(query)
+    }
+
+    async delete(query: SqlQuery) {
+      return this.internalQuery(query)
+    }
+  }
+
+  module.exports = {
+    schema: SCHEMA,
+    integration: SnowflakeIntegration,
+  }
+}

package/src/threads/automation.js

@@ -218,7 +218,7 @@ class Orchestrator {
             }
           }
           if (
-            index === parseInt(env.AUTOMATION_MAX_ITERATIONS) ||
+            index === env.AUTOMATION_MAX_ITERATION ||
             index === parseInt(loopStep.inputs.iterations)
           ) {
             this.updateContextAndOutput(loopStepNumber, step, tempOutput, {

package/src/utilities/fileSystem/index.js

@@ -21,6 +21,7 @@ const env = require("../../environment")
 const {
   USER_METDATA_PREFIX,
   LINK_USER_METADATA_PREFIX,
+  TABLE_ROW_PREFIX,
 } = require("../../db/utils")
 const MemoryStream = require("memorystream")
 const { getAppId } = require("@budibase/backend-core/context")
@@ -109,6 +110,23 @@ exports.apiFileReturn = contents => {
   return fs.createReadStream(path)
 }
 
+exports.defineFilter = excludeRows => {
+  if (excludeRows) {
+    return doc =>
+      !(
+        doc._id.includes(USER_METDATA_PREFIX) ||
+        doc._id.includes(LINK_USER_METADATA_PREFIX) ||
+        doc._id.includes(TABLE_ROW_PREFIX)
+      )
+  } else if (!excludeRows) {
+    return doc =>
+      !(
+        doc._id.includes(USER_METDATA_PREFIX) ||
+        doc._id.includes(LINK_USER_METADATA_PREFIX)
+      )
+  }
+}
+
 /**
  * Local utility to back up the database state for an app, excluding global user
  * data or user relationships.
@@ -116,14 +134,10 @@ exports.apiFileReturn = contents => {
  * @param {object} config Config to send to export DB
  * @returns {*} either a string or a stream of the backup
  */
-const backupAppData = async (appId, config) => {
+const backupAppData = async (appId, config, includeRows) => {
   return await exports.exportDB(appId, {
     ...config,
-    filter: doc =>
-      !(
-        doc._id.includes(USER_METDATA_PREFIX) ||
-        doc._id.includes(LINK_USER_METADATA_PREFIX)
-      ),
+    filter: exports.defineFilter(includeRows),
   })
 }
 
@@ -142,8 +156,8 @@ exports.performBackup = async (appId, backupName) => {
  * @param {string} appId The ID of the app which is to be backed up.
  * @returns {*} a readable stream of the backup which is written in real time
  */
-exports.streamBackup = async appId => {
-  return await backupAppData(appId, { stream: true })
+exports.streamBackup = async (appId, includeRows) => {
+  return await backupAppData(appId, { stream: true }, includeRows)
 }
 
 /**

package/src/utilities/fileSystem/utilities.js

@@ -9,6 +9,7 @@ const {
   deleteFolder,
   uploadDirectory,
   downloadTarball,
+  downloadTarballDirect,
 } = require("@budibase/backend-core/objectStore")
 
 /***********************************
@@ -29,4 +30,5 @@ exports.retrieveToTmp = retrieveToTmp
 exports.deleteFolder = deleteFolder
 exports.uploadDirectory = uploadDirectory
 exports.downloadTarball = downloadTarball
+exports.downloadTarballDirect = downloadTarballDirect
 exports.deleteFiles = deleteFiles

package/src/utilities/index.js

@@ -54,11 +54,17 @@ exports.objectStoreUrl = () => {
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */
-exports.clientLibraryPath = (appId, version) => {
+exports.clientLibraryPath = (appId, version, ctx) => {
   if (env.isProd()) {
+    // TODO: remove - for beta testing UI
+    if (ctx && ctx.cookies.get("beta:design_ui")) {
+      return "https://cdn.budi.live/beta:design_ui/budibase-client.js"
+    }
+
     let url = `${exports.objectStoreUrl()}/${sanitizeKey(
       appId
     )}/budibase-client.js`
+
     // append app version to bust the cache
     if (version) {
       url += `?v=${version}`