@budibase/server 1.0.212-alpha.9 → 1.0.214

package/dist/utilities/fileSystem/index.js

@@ -17,7 +17,7 @@ const { ObjectStoreBuckets } = require("../../constants");
 const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
 const env = require("../../environment");
-const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, } = require("../../db/utils");
+const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, } = require("../../db/utils");
 const MemoryStream = require("memorystream");
 const { getAppId } = require("@budibase/backend-core/context");
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..");
@@ -98,17 +98,6 @@ exports.apiFileReturn = contents => {
     fs.writeFileSync(path, contents);
     return fs.createReadStream(path);
 };
-exports.defineFilter = excludeRows => {
-    if (excludeRows) {
-        return doc => !(doc._id.includes(USER_METDATA_PREFIX) ||
-            doc._id.includes(LINK_USER_METADATA_PREFIX) ||
-            doc._id.includes(TABLE_ROW_PREFIX));
-    }
-    else if (!excludeRows) {
-        return doc => !(doc._id.includes(USER_METDATA_PREFIX) ||
-            doc._id.includes(LINK_USER_METADATA_PREFIX));
-    }
-};
 /**
  * Local utility to back up the database state for an app, excluding global user
  * data or user relationships.
@@ -116,8 +105,9 @@ exports.defineFilter = excludeRows => {
  * @param {object} config Config to send to export DB
  * @returns {*} either a string or a stream of the backup
  */
-const backupAppData = (appId, config, includeRows) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: exports.defineFilter(includeRows) }));
+const backupAppData = (appId, config) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: doc => !(doc._id.includes(USER_METDATA_PREFIX) ||
+            doc._id.includes(LINK_USER_METADATA_PREFIX)) }));
 });
 /**
  * Takes a copy of the database state for an app to the object store.
@@ -133,8 +123,8 @@ exports.performBackup = (appId, backupName) => __awaiter(void 0, void 0, void 0,
  * @param {string} appId The ID of the app which is to be backed up.
  * @returns {*} a readable stream of the backup which is written in real time
  */
-exports.streamBackup = (appId, includeRows) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield backupAppData(appId, { stream: true }, includeRows);
+exports.streamBackup = (appId) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield backupAppData(appId, { stream: true });
 });
 /**
  * Exports a DB to either file or a variable (memory).

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.212-alpha.9",
+  "version": "1.0.214",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,10 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^1.0.212-alpha.9",
-    "@budibase/client": "^1.0.212-alpha.9",
-    "@budibase/pro": "1.0.212-alpha.8",
-    "@budibase/string-templates": "^1.0.212-alpha.9",
-    "@budibase/types": "^1.0.212-alpha.9",
+    "@budibase/backend-core": "^1.0.214",
+    "@budibase/client": "^1.0.214",
+    "@budibase/pro": "1.0.213",
+    "@budibase/string-templates": "^1.0.214",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -136,7 +135,6 @@
     "pouchdb-replication-stream": "1.2.9",
     "redis": "4",
     "server-destroy": "1.0.1",
-    "snowflake-promise": "^4.5.0",
     "svelte": "3.44.1",
     "swagger-parser": "10.0.3",
     "to-json-schema": "0.2.5",
@@ -152,6 +150,7 @@
     "@babel/core": "7.17.4",
     "@babel/preset-env": "7.16.11",
     "@budibase/standard-components": "^0.9.139",
+    "@budibase/types": "^1.0.214",
     "@jest/test-sequencer": "24.9.0",
     "@types/apidoc": "0.50.0",
     "@types/bson": "4.2.0",
@@ -195,5 +194,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "84013b1a5966f50f0915b96ec75568d73ad0af77"
+  "gitHead": "9c7248b4982f8f898d6ac1ced4b8b9bd2f690627"
 }

package/src/api/controllers/backup.js

@@ -3,12 +3,11 @@ const { events, context } = require("@budibase/backend-core")
 const { DocumentTypes } = require("../../db/utils")
 
 exports.exportAppDump = async function (ctx) {
-  let { appId, excludeRows } = ctx.query
+  const { appId } = ctx.query
   const appName = decodeURI(ctx.query.appname)
-  excludeRows = excludeRows === "true"
   const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
   ctx.attachment(backupIdentifier)
-  ctx.body = await streamBackup(appId, excludeRows)
+  ctx.body = await streamBackup(appId)
 
   await context.doInAppContext(appId, async () => {
     const appDb = context.getAppDB()

package/src/api/controllers/row/external.js

@@ -155,44 +155,34 @@ exports.validate = async () => {
 }
 
 exports.exportRows = async ctx => {
-  const { datasourceId } = breakExternalTableId(ctx.params.tableId)
+  const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
   const db = getAppDB()
-  const format = ctx.query.format
-  const { columns } = ctx.request.body
+  let format = ctx.query.format
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
+  const tables = datasource.entities
+  const table = tables[tableName]
   ctx.request.body = {
     query: {
       oneOf: {
-        _id: ctx.request.body.rows.map(row => JSON.parse(decodeURI(row))[0]),
+        [table.primaryDisplay]: ctx.request.body.rows.map(
+          id => breakRowIdField(id)[0]
+        ),
       },
     },
   }
 
   let result = await exports.search(ctx)
-  let rows = []
-
-  // Filter data to only specified columns if required
-  if (columns && columns.length) {
-    for (let i = 0; i < result.rows.length; i++) {
-      rows[i] = {}
-      for (let column of columns) {
-        rows[i][column] = result.rows[i][column]
-      }
-    }
-  } else {
-    rows = result.rows
-  }
 
-  let headers = Object.keys(rows[0])
+  let headers = Object.keys(result.rows[0])
   const exporter = exporters[format]
   const filename = `export.${format}`
 
   // send down the file
   ctx.attachment(filename)
-  return apiFileReturn(exporter(headers, rows))
+  return apiFileReturn(exporter(headers, result.rows))
 }
 
 exports.fetchEnrichedRow = async ctx => {

package/src/api/controllers/row/internalSearch.js

@@ -17,7 +17,6 @@ class QueryBuilder {
       notEqual: {},
       empty: {},
       notEmpty: {},
-      oneOf: {},
       ...base,
     }
     this.limit = 50
@@ -113,11 +112,6 @@ class QueryBuilder {
     return this
   }
 
-  addOneOf(key, value) {
-    this.query.oneOf[key] = value
-    return this
-  }
-
   /**
    * Preprocesses a value before going into a lucene search.
    * Transforms strings to lowercase and wraps strings and bools in quotes.
@@ -226,28 +220,6 @@ class QueryBuilder {
     if (this.query.notEmpty) {
       build(this.query.notEmpty, key => `${key}:["" TO *]`)
     }
-    if (this.query.oneOf) {
-      build(this.query.oneOf, (key, value) => {
-        if (!Array.isArray(value)) {
-          if (typeof value === "string") {
-            value = value.split(",")
-          } else {
-            return ""
-          }
-        }
-        let orStatement = `${builder.preprocess(
-          value[0],
-          allPreProcessingOpts
-        )}`
-        for (let i = 1; i < value.length; i++) {
-          orStatement += ` OR ${builder.preprocess(
-            value[i],
-            allPreProcessingOpts
-          )}`
-        }
-        return `${key}:(${orStatement})`
-      })
-    }
     return query
   }
 

package/src/api/controllers/row/utils.js

@@ -64,9 +64,6 @@ exports.validate = async ({ tableId, row, table }) => {
     // Validate.js doesn't seem to handle array
     if (type === FieldTypes.ARRAY && row[fieldName]) {
       if (row[fieldName].length) {
-        if (!Array.isArray(row[fieldName])) {
-          row[fieldName] = row[fieldName].split(",")
-        }
         row[fieldName].map(val => {
           if (
             !constraints.inclusion.includes(val) &&

package/src/automations/automationUtils.js

@@ -1,10 +1,5 @@
 const { getTable } = require("../api/controllers/table/utils")
-const {
-  findHBSBlocks,
-  decodeJSBinding,
-  isJSBinding,
-  encodeJSBinding,
-} = require("@budibase/string-templates")
+const { findHBSBlocks } = require("@budibase/string-templates")
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -82,21 +77,11 @@ exports.getError = err => {
 }
 
 exports.substituteLoopStep = (hbsString, substitute) => {
-  let blocks = []
-  let checkForJS = isJSBinding(hbsString)
-  if (checkForJS) {
-    hbsString = decodeJSBinding(hbsString)
-    blocks.push(hbsString)
-  } else {
-    blocks = findHBSBlocks(hbsString)
-  }
+  let blocks = findHBSBlocks(hbsString)
   for (let block of blocks) {
+    let oldBlock = block
     block = block.replace(/loop/, substitute)
-    if (checkForJS) {
-      hbsString = encodeJSBinding(block)
-    } else {
-      hbsString = block
-    }
+    hbsString = hbsString.replace(new RegExp(oldBlock, "g"), block)
   }
 
   return hbsString

package/src/automations/triggers.js

@@ -97,7 +97,7 @@ exports.externalTrigger = async function (
     // values are likely to be submitted as strings, so we shall convert to correct type
     const coercedFields = {}
     const fields = automation.definition.trigger.inputs.fields
-    for (let key of Object.keys(fields || {})) {
+    for (let key of Object.keys(fields)) {
       coercedFields[key] = coerce(params.fields[key], fields[key])
     }
     params.fields = coercedFields

package/src/db/utils.js

@@ -74,7 +74,6 @@ exports.isDevAppID = isDevAppID
 exports.isProdAppID = isProdAppID
 exports.USER_METDATA_PREFIX = `${DocumentTypes.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
 exports.LINK_USER_METADATA_PREFIX = `${DocumentTypes.LINK}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
-exports.TABLE_ROW_PREFIX = `${DocumentTypes.ROW}${SEPARATOR}${DocumentTypes.TABLE}`
 exports.ViewNames = ViewNames
 exports.InternalTables = InternalTables
 exports.DocumentTypes = DocumentTypes

package/src/definitions/common.ts

@@ -12,7 +12,6 @@ export interface Application extends Base {
 export interface FieldSchema {
   // TODO: replace with field types enum when done
   type: string
-  externalType?: string
   fieldName?: string
   name: string
   tableId?: string

package/src/definitions/datasource.ts

@@ -50,7 +50,6 @@ export enum SourceNames {
   GOOGLE_SHEETS = "GOOGLE_SHEETS",
   FIRESTORE = "FIRESTORE",
   REDIS = "REDIS",
-  SNOWFLAKE = "SNOWFLAKE",
 }
 
 export enum IncludeRelationships {
@@ -94,7 +93,6 @@ export interface Integration {
   relationships?: boolean
   description: string
   friendlyName: string
-  type?: string
   datasource: {}
   query: {
     [key: string]: QueryDefinition

package/src/integrations/airtable.ts

@@ -18,7 +18,6 @@ module AirtableModule {
     description:
       "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.",
     friendlyName: "Airtable",
-    type: "Spreadsheet",
     datasource: {
       apiKey: {
         type: DatasourceFieldTypes.PASSWORD,

package/src/integrations/arangodb.ts

@@ -19,7 +19,6 @@ module ArangoModule {
   const SCHEMA: Integration = {
     docs: "https://github.com/arangodb/arangojs",
     friendlyName: "ArangoDB",
-    type: "Non-relational",
     description:
       "ArangoDB is a scalable open-source multi-model database natively supporting graph, document and search. All supported data models & access patterns can be combined in queries allowing for maximal flexibility. ",
     datasource: {

package/src/integrations/base/sql.ts

@@ -89,28 +89,6 @@ function parseFilters(filters: SearchFilters | undefined): SearchFilters {
   return filters
 }
 
-function generateSelectStatement(
-  json: QueryJson,
-  knex: Knex
-): (string | Knex.Raw)[] {
-  const { resource, meta } = json
-  const schema = meta?.table?.schema
-  return resource.fields.map(field => {
-    const fieldNames = field.split(/\./g)
-    const tableName = fieldNames[0]
-    const columnName = fieldNames[1]
-    if (columnName && knex.client.config.client === SqlClients.POSTGRES) {
-      const externalType = schema?.[columnName].externalType
-      if (externalType?.includes("money")) {
-        return knex.raw(
-          `"${tableName}"."${columnName}"::money::numeric as "${field}"`
-        )
-      }
-    }
-    return `${field} as ${field}`
-  })
-}
-
 class InternalBuilder {
   private readonly client: string
 
@@ -238,7 +216,9 @@ class InternalBuilder {
   }
 
   addRelationships(
+    knex: Knex,
     query: KnexQuery,
+    fields: string | string[],
     fromTable: string,
     relationships: RelationshipsJson[] | undefined
   ): KnexQuery {
@@ -343,12 +323,12 @@ class InternalBuilder {
     if (!resource) {
       resource = { fields: [] }
     }
-    let selectStatement: string | (string | Knex.Raw)[] = "*"
+    let selectStatement: string | string[] = "*"
     // handle select
     if (resource.fields && resource.fields.length > 0) {
       // select the resources as the format "table.columnName" - this is what is provided
       // by the resource builder further up
-      selectStatement = generateSelectStatement(json, knex)
+      selectStatement = resource.fields.map(field => `${field} as ${field}`)
     }
     let foundLimit = limit || BASE_LIMIT
     // handle pagination
@@ -383,7 +363,13 @@ class InternalBuilder {
       preQuery = this.addSorting(preQuery, json)
     }
     // handle joins
-    query = this.addRelationships(preQuery, tableName, relationships)
+    query = this.addRelationships(
+      knex,
+      preQuery,
+      selectStatement,
+      tableName,
+      relationships
+    )
     return this.addFilters(query, filters, { relationship: true })
   }
 

package/src/integrations/couchdb.ts

@@ -16,7 +16,6 @@ module CouchDBModule {
   const SCHEMA: Integration = {
     docs: "https://docs.couchdb.org/en/stable/",
     friendlyName: "CouchDB",
-    type: "Non-relational",
     description:
       "Apache CouchDB is an open-source document-oriented NoSQL database, implemented in Erlang.",
     datasource: {

package/src/integrations/dynamodb.ts

@@ -21,7 +21,6 @@ module DynamoModule {
     description:
       "Amazon DynamoDB is a key-value and document database that delivers single-digit millisecond performance at any scale.",
     friendlyName: "DynamoDB",
-    type: "Non-relational",
     datasource: {
       region: {
         type: DatasourceFieldTypes.STRING,

package/src/integrations/elasticsearch.ts

@@ -17,7 +17,6 @@ module ElasticsearchModule {
     description:
       "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.",
     friendlyName: "ElasticSearch",
-    type: "Non-relational",
     datasource: {
       url: {
         type: DatasourceFieldTypes.STRING,

package/src/integrations/firebase.ts

@@ -16,7 +16,6 @@ module Firebase {
   const SCHEMA: Integration = {
     docs: "https://firebase.google.com/docs/firestore/quickstart",
     friendlyName: "Firestore",
-    type: "Non-relational",
     description:
       "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.",
     datasource: {

package/src/integrations/googlesheets.ts

@@ -49,7 +49,6 @@ module GoogleSheetsModule {
     description:
       "Create and collaborate on online spreadsheets in real-time and from any device. ",
     friendlyName: "Google Sheets",
-    type: "Spreadsheet",
     datasource: {
       spreadsheetId: {
         display: "Google Sheet URL",

package/src/integrations/index.ts

@@ -12,7 +12,6 @@ const rest = require("./rest")
 const googlesheets = require("./googlesheets")
 const firebase = require("./firebase")
 const redis = require("./redis")
-const snowflake = require("./snowflake")
 const { SourceNames } = require("../definitions/datasource")
 const environment = require("../environment")
 
@@ -30,7 +29,6 @@ const DEFINITIONS = {
   [SourceNames.REST]: rest.schema,
   [SourceNames.FIRESTORE]: firebase.schema,
   [SourceNames.REDIS]: redis.schema,
-  [SourceNames.SNOWFLAKE]: snowflake.schema,
 }
 
 const INTEGRATIONS = {
@@ -48,8 +46,6 @@ const INTEGRATIONS = {
   [SourceNames.FIRESTORE]: firebase.integration,
   [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
   [SourceNames.REDIS]: redis.integration,
-  [SourceNames.FIREBASE]: firebase.integration,
-  [SourceNames.SNOWFLAKE]: snowflake.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/integrations/microsoftSqlServer.ts

@@ -44,7 +44,6 @@ module MSSQLModule {
     description:
       "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
     friendlyName: "MS SQL Server",
-    type: "Relational",
     datasource: {
       user: {
         type: DatasourceFieldTypes.STRING,
@@ -247,7 +246,6 @@ module MSSQLModule {
             autocolumn: !!autoColumns.find((col: string) => col === name),
             name: name,
             ...convertSqlType(def.DATA_TYPE),
-            externalType: def.DATA_TYPE,
           }
         }
         tables[tableName] = {

package/src/integrations/mongodb.ts

@@ -24,7 +24,6 @@ module MongoDBModule {
   const SCHEMA: Integration = {
     docs: "https://github.com/mongodb/node-mongodb-native",
     friendlyName: "MongoDB",
-    type: "Non-relational",
     description:
       "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.",
     datasource: {

package/src/integrations/mysql.ts

@@ -36,7 +36,6 @@ module MySQLModule {
     docs: "https://github.com/sidorares/node-mysql2",
     plus: true,
     friendlyName: "MySQL",
-    type: "Relational",
     description:
       "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ",
     datasource: {
@@ -233,7 +232,6 @@ module MySQLModule {
               autocolumn: isAuto,
               constraints,
               ...convertSqlType(column.Type),
-              externalType: column.Type,
             }
           }
           if (!tables[tableName]) {

package/src/integrations/oracle.ts

@@ -40,7 +40,6 @@ module OracleModule {
     docs: "https://github.com/oracle/node-oracledb",
     plus: true,
     friendlyName: "Oracle",
-    type: "Relational",
     description:
       "Oracle Database is an object-relational database management system developed by Oracle Corporation",
     datasource: {

package/src/integrations/postgres.ts

@@ -47,7 +47,6 @@ module PostgresModule {
     docs: "https://node-postgres.com",
     plus: true,
     friendlyName: "PostgreSQL",
-    type: "Relational",
     description:
       "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
     datasource: {
@@ -272,7 +271,6 @@ module PostgresModule {
             autocolumn: isAuto,
             name: columnName,
             ...convertSqlType(column.data_type),
-            externalType: column.data_type,
           }
         }
 

package/src/integrations/redis.ts

@@ -17,7 +17,6 @@ module RedisModule {
     docs: "https://redis.io/docs/",
     description: "",
     friendlyName: "Redis",
-    type: "Non-relational",
     datasource: {
       host: {
         type: "string",

package/src/integrations/rest.ts

@@ -64,7 +64,6 @@ module RestModule {
     description:
       "With the REST API datasource, you can connect, query and pull data from multiple REST APIs. You can then use the retrieved data to build apps.",
     friendlyName: "REST API",
-    type: "API",
     datasource: {
       url: {
         type: DatasourceFieldTypes.STRING,

package/src/integrations/s3.ts

@@ -17,7 +17,6 @@ module S3Module {
     description:
       "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
     friendlyName: "Amazon S3",
-    type: "Object store",
     datasource: {
       region: {
         type: "string",

package/src/threads/automation.js

@@ -218,7 +218,7 @@ class Orchestrator {
             }
           }
           if (
-            index === env.AUTOMATION_MAX_ITERATION ||
+            index === parseInt(env.AUTOMATION_MAX_ITERATIONS) ||
             index === parseInt(loopStep.inputs.iterations)
           ) {
             this.updateContextAndOutput(loopStepNumber, step, tempOutput, {

package/src/utilities/fileSystem/index.js

@@ -21,7 +21,6 @@ const env = require("../../environment")
 const {
   USER_METDATA_PREFIX,
   LINK_USER_METADATA_PREFIX,
-  TABLE_ROW_PREFIX,
 } = require("../../db/utils")
 const MemoryStream = require("memorystream")
 const { getAppId } = require("@budibase/backend-core/context")
@@ -110,23 +109,6 @@ exports.apiFileReturn = contents => {
   return fs.createReadStream(path)
 }
 
-exports.defineFilter = excludeRows => {
-  if (excludeRows) {
-    return doc =>
-      !(
-        doc._id.includes(USER_METDATA_PREFIX) ||
-        doc._id.includes(LINK_USER_METADATA_PREFIX) ||
-        doc._id.includes(TABLE_ROW_PREFIX)
-      )
-  } else if (!excludeRows) {
-    return doc =>
-      !(
-        doc._id.includes(USER_METDATA_PREFIX) ||
-        doc._id.includes(LINK_USER_METADATA_PREFIX)
-      )
-  }
-}
-
 /**
  * Local utility to back up the database state for an app, excluding global user
  * data or user relationships.
@@ -134,10 +116,14 @@ exports.defineFilter = excludeRows => {
  * @param {object} config Config to send to export DB
  * @returns {*} either a string or a stream of the backup
  */
-const backupAppData = async (appId, config, includeRows) => {
+const backupAppData = async (appId, config) => {
   return await exports.exportDB(appId, {
     ...config,
-    filter: exports.defineFilter(includeRows),
+    filter: doc =>
+      !(
+        doc._id.includes(USER_METDATA_PREFIX) ||
+        doc._id.includes(LINK_USER_METADATA_PREFIX)
+      ),
   })
 }
 
@@ -156,8 +142,8 @@ exports.performBackup = async (appId, backupName) => {
  * @param {string} appId The ID of the app which is to be backed up.
  * @returns {*} a readable stream of the backup which is written in real time
  */
-exports.streamBackup = async (appId, includeRows) => {
-  return await backupAppData(appId, { stream: true }, includeRows)
+exports.streamBackup = async appId => {
+  return await backupAppData(appId, { stream: true })
 }
 
 /**