@budibase/server 0.9.184 → 0.9.185-alpha.11

package/src/integrations/postgres.ts

@@ -6,17 +6,18 @@ import {
   SqlQuery,
 } from "../definitions/datasource"
 import { Table } from "../definitions/common"
-import { getSqlQuery } from "./utils"
+import {
+  getSqlQuery,
+  buildExternalTableId,
+  convertSqlType,
+  finaliseExternalTables,
+  SqlClients,
+} from "./utils"
+import { DatasourcePlus } from "./base/datasourcePlus"
 
 module PostgresModule {
   const { Pool } = require("pg")
   const Sql = require("./base/sql")
-  const { FieldTypes } = require("../constants")
-  const {
-    buildExternalTableId,
-    convertType,
-    copyExistingPropsOver,
-  } = require("./utils")
   const { escapeDangerousCharacters } = require("../utilities")
 
   const JSON_REGEX = /'{.*}'::json/s
@@ -27,6 +28,7 @@ module PostgresModule {
     database: string
     user: string
     password: string
+    schema: string
     ssl?: boolean
     ca?: string
     rejectUnauthorized?: boolean
@@ -64,6 +66,11 @@ module PostgresModule {
         default: "root",
         required: true,
       },
+      schema: {
+        type: DatasourceFieldTypes.STRING,
+        default: "public",
+        required: true,
+      },
       ssl: {
         type: DatasourceFieldTypes.BOOLEAN,
         default: false,
@@ -96,22 +103,6 @@ module PostgresModule {
     },
   }
 
-  const TYPE_MAP = {
-    text: FieldTypes.LONGFORM,
-    varchar: FieldTypes.STRING,
-    integer: FieldTypes.NUMBER,
-    bigint: FieldTypes.NUMBER,
-    decimal: FieldTypes.NUMBER,
-    smallint: FieldTypes.NUMBER,
-    real: FieldTypes.NUMBER,
-    "double precision": FieldTypes.NUMBER,
-    timestamp: FieldTypes.DATETIME,
-    time: FieldTypes.DATETIME,
-    boolean: FieldTypes.BOOLEAN,
-    json: FieldTypes.JSON,
-    date: FieldTypes.DATETIME,
-  }
-
   async function internalQuery(client: any, query: SqlQuery) {
     // need to handle a specific issue with json data types in postgres,
     // new lines inside the JSON data will break it
@@ -132,13 +123,14 @@ module PostgresModule {
     }
   }
 
-  class PostgresIntegration extends Sql {
+  class PostgresIntegration extends Sql implements DatasourcePlus {
     static pool: any
     private readonly client: any
     private readonly config: PostgresConfig
+    public tables: Record<string, Table> = {}
+    public schemaErrors: Record<string, string> = {}
 
-    COLUMNS_SQL =
-      "select * from information_schema.columns where not table_schema = 'information_schema' and not table_schema = 'pg_catalog'"
+    COLUMNS_SQL!: string
 
     PRIMARY_KEYS_SQL = `
     select tc.table_schema, tc.table_name, kc.column_name as primary_key 
@@ -151,7 +143,7 @@ module PostgresModule {
     `
 
     constructor(config: PostgresConfig) {
-      super("pg")
+      super(SqlClients.POSTGRES)
       this.config = config
 
       let newConfig = {
@@ -168,6 +160,17 @@ module PostgresModule {
       }
 
       this.client = this.pool
+      this.setSchema()
+    }
+
+    setSchema() {
+      if (!this.config.schema) {
+        this.config.schema = "public"
+      }
+      this.client.on("connect", (client: any) => {
+        client.query(`SET search_path TO ${this.config.schema}`)
+      })
+      this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
     }
 
     /**
@@ -207,13 +210,13 @@ module PostgresModule {
         if (!tables[tableName] || !tables[tableName].schema) {
           tables[tableName] = {
             _id: buildExternalTableId(datasourceId, tableName),
-            primary: tableKeys[tableName] || ["id"],
+            primary: tableKeys[tableName] || [],
             name: tableName,
             schema: {},
           }
         }
 
-        const type: string = convertType(column.data_type, TYPE_MAP)
+        const type: string = convertSqlType(column.data_type)
         const identity = !!(
           column.identity_generation ||
           column.identity_start ||
@@ -232,10 +235,9 @@ module PostgresModule {
         }
       }
 
-      for (let tableName of Object.keys(tables)) {
-        copyExistingPropsOver(tableName, tables, entities)
-      }
-      this.tables = tables
+      const final = finaliseExternalTables(tables, entities)
+      this.tables = final.tables
+      this.schemaErrors = final.errors
     }
 
     async create(query: SqlQuery | string) {
@@ -261,8 +263,16 @@ module PostgresModule {
     async query(json: QueryJson) {
       const operation = this._operation(json).toLowerCase()
       const input = this._query(json)
-      const response = await internalQuery(this.client, input)
-      return response.rows.length ? response.rows : [{ [operation]: true }]
+      if (Array.isArray(input)) {
+        const responses = []
+        for (let query of input) {
+          responses.push(await internalQuery(this.client, query))
+        }
+        return responses
+      } else {
+        const response = await internalQuery(this.client, input)
+        return response.rows.length ? response.rows : [{ [operation]: true }]
+      }
     }
   }
 

package/src/integrations/rest.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module RestModule {
   const fetch = require("node-fetch")
@@ -131,7 +132,7 @@ module RestModule {
     },
   }
 
-  class RestIntegration {
+  class RestIntegration implements IntegrationBase {
     private config: RestConfig
     private headers: {
       [key: string]: string

package/src/integrations/s3.ts

@@ -1,4 +1,5 @@
 import { Integration, QueryTypes } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module S3Module {
   const AWS = require("aws-sdk")
@@ -42,7 +43,7 @@ module S3Module {
     },
   }
 
-  class S3Integration {
+  class S3Integration implements IntegrationBase {
     private readonly config: S3Config
     private client: any
     private connectionPromise: Promise<any>

package/src/integrations/tests/microsoftSqlServer.spec.js

@@ -9,32 +9,39 @@ class TestConfiguration {
 }
 
 describe("MS SQL Server Integration", () => {
-  let config 
+  let config
 
-  beforeEach(() => {
+  beforeEach(async () => {
     config = new TestConfiguration()
   })
 
-  it("calls the create method with the correct params", async () => {
-    const sql = "insert into users (name, age) values ('Joe', 123);"
-    const response = await config.integration.create({ 
-      sql
+  describe("check sql used", () => {
+    beforeEach(async () => {
+      await config.integration.connect()
+    })
+
+    it("calls the create method with the correct params", async () => {
+      const sql = "insert into users (name, age) values ('Joe', 123);"
+      const response = await config.integration.create({
+        sql
+      })
+      expect(config.integration.client.request).toHaveBeenCalledWith()
+      expect(response[0]).toEqual(sql)
     })
-    expect(config.integration.client.query).toHaveBeenCalledWith(sql, {})
-  })
 
-  it("calls the read method with the correct params", async () => {
-    const sql = "select * from users;"
-    const response = await config.integration.read({ 
-      sql
+    it("calls the read method with the correct params", async () => {
+      const sql = "select * from users;"
+      const response = await config.integration.read({
+        sql
+      })
+      expect(config.integration.client.request).toHaveBeenCalledWith()
+      expect(response[0]).toEqual(sql)
     })
-    expect(config.integration.client.query).toHaveBeenCalledWith(sql, {})
   })
 
   describe("no rows returned", () => {
     beforeEach(async () => {
       await config.integration.connect()
-      config.integration.client.query.mockImplementation(() => ({ rows: [] }))
     })
 
     it("returns the correct response when the create response has no rows", async () => {
@@ -42,7 +49,7 @@ describe("MS SQL Server Integration", () => {
       const response = await config.integration.create({ 
         sql
       })
-      expect(response).toEqual([{ created: true }])
+      expect(response[0]).toEqual(sql)
     })
   })
 })

package/src/integrations/tests/postgres.spec.js

@@ -15,6 +15,10 @@ describe("Postgres Integration", () => {
     config = new TestConfiguration()
   })
 
+  it("calls the connection callback", async () => {
+    expect(pg.on).toHaveBeenCalledWith('connect', expect.anything())
+  })
+
   it("calls the create method with the correct params", async () => {
     const sql = "insert into users (name, age) values ('Joe', 123);"
     await config.integration.create({

package/src/integrations/utils.ts

@@ -1,12 +1,45 @@
 import { SqlQuery } from "../definitions/datasource"
-import { Datasource } from "../definitions/common"
+import { Datasource, Table } from "../definitions/common"
 import { SourceNames } from "../definitions/datasource"
 const { DocumentTypes, SEPARATOR } = require("../db/utils")
-const { FieldTypes } = require("../constants")
+const { FieldTypes, BuildSchemaErrors } = require("../constants")
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 
+const SQL_TYPE_MAP = {
+  text: FieldTypes.LONGFORM,
+  varchar: FieldTypes.STRING,
+  integer: FieldTypes.NUMBER,
+  bigint: FieldTypes.NUMBER,
+  decimal: FieldTypes.NUMBER,
+  smallint: FieldTypes.NUMBER,
+  real: FieldTypes.NUMBER,
+  "double precision": FieldTypes.NUMBER,
+  timestamp: FieldTypes.DATETIME,
+  time: FieldTypes.DATETIME,
+  boolean: FieldTypes.BOOLEAN,
+  json: FieldTypes.JSON,
+  date: FieldTypes.DATETIME,
+  blob: FieldTypes.LONGFORM,
+  enum: FieldTypes.STRING,
+  float: FieldTypes.NUMBER,
+  int: FieldTypes.NUMBER,
+  numeric: FieldTypes.NUMBER,
+  mediumint: FieldTypes.NUMBER,
+  dec: FieldTypes.NUMBER,
+  double: FieldTypes.NUMBER,
+  fixed: FieldTypes.NUMBER,
+  datetime: FieldTypes.DATETIME,
+  tinyint: FieldTypes.BOOLEAN,
+}
+
+export enum SqlClients {
+  MS_SQL = "mssql",
+  POSTGRES = "pg",
+  MY_SQL = "mysql",
+}
+
 export function isExternalTable(tableId: string) {
   return tableId.includes(DocumentTypes.DATASOURCE)
 }
@@ -68,8 +101,8 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
   }
 }
 
-export function convertType(type: string, map: { [key: string]: any }) {
-  for (let [external, internal] of Object.entries(map)) {
+export function convertSqlType(type: string) {
+  for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) {
     if (type.toLowerCase().includes(external)) {
       return internal
     }
@@ -102,14 +135,14 @@ export function isIsoDateString(str: string) {
 }
 
 // add the existing relationships from the entities if they exist, to prevent them from being overridden
-export function copyExistingPropsOver(
+function copyExistingPropsOver(
   tableName: string,
-  tables: { [key: string]: any },
+  table: Table,
   entities: { [key: string]: any }
 ) {
   if (entities && entities[tableName]) {
     if (entities[tableName].primaryDisplay) {
-      tables[tableName].primaryDisplay = entities[tableName].primaryDisplay
+      table.primaryDisplay = entities[tableName].primaryDisplay
     }
     const existingTableSchema = entities[tableName].schema
     for (let key in existingTableSchema) {
@@ -117,8 +150,27 @@ export function copyExistingPropsOver(
         continue
       }
       if (existingTableSchema[key].type === "link") {
-        tables[tableName].schema[key] = existingTableSchema[key]
+        table.schema[key] = existingTableSchema[key]
       }
     }
   }
+  return table
+}
+
+export function finaliseExternalTables(
+  tables: { [key: string]: any },
+  entities: { [key: string]: any }
+) {
+  const finalTables: { [key: string]: any } = {}
+  const errors: { [key: string]: string } = {}
+  for (let [name, table] of Object.entries(tables)) {
+    // make sure every table has a key
+    if (table.primary == null || table.primary.length === 0) {
+      errors[name] = BuildSchemaErrors.NO_KEY
+      continue
+    }
+    // make sure all previous props have been added back
+    finalTables[name] = copyExistingPropsOver(name, table, entities)
+  }
+  return { tables: finalTables, errors }
 }

package/src/middleware/authorized.js

@@ -15,7 +15,8 @@ module.exports =
   (permType, permLevel = null) =>
   async (ctx, next) => {
     // webhooks don't need authentication, each webhook unique
-    if (isWebhookEndpoint(ctx)) {
+    // also internal requests (between services) don't need authorized
+    if (isWebhookEndpoint(ctx) || ctx.internal) {
       return next()
     }
 

package/src/middleware/builder.js

@@ -8,6 +8,7 @@ const {
 const CouchDB = require("../db")
 const { DocumentTypes } = require("../db/utils")
 const { PermissionTypes } = require("@budibase/auth/permissions")
+const { app: appCache } = require("@budibase/auth/cache")
 
 const DEBOUNCE_TIME_SEC = 30
 
@@ -50,7 +51,9 @@ async function updateAppUpdatedAt(ctx) {
   const db = new CouchDB(appId)
   const metadata = await db.get(DocumentTypes.APP_METADATA)
   metadata.updatedAt = new Date().toISOString()
-  await db.put(metadata)
+  const response = await db.put(metadata)
+  metadata._rev = response.rev
+  await appCache.invalidateAppMetadata(appId, metadata)
   // set a new debounce record with a short TTL
   await setDebounce(appId, DEBOUNCE_TIME_SEC)
 }

package/src/tests/utilities/TestConfiguration.js

@@ -311,7 +311,8 @@ class TestConfiguration {
 
   async createDatasource(config = null) {
     config = config || basicDatasource()
-    this.datasource = await this._req(config, null, controllers.datasource.save)
+    const response = await this._req(config, null, controllers.datasource.save)
+    this.datasource = response.datasource
     return this.datasource
   }
 

package/src/{automations/thread.js → threads/automation.js}

@@ -1,5 +1,5 @@
-const actions = require("./actions")
-const automationUtils = require("./automationUtils")
+const actions = require("../automations/actions")
+const automationUtils = require("../automations/automationUtils")
 const AutomationEmitter = require("../events/AutomationEmitter")
 const { processObject } = require("@budibase/string-templates")
 const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
@@ -8,8 +8,10 @@ const { DocumentTypes, isDevAppID } = require("../db/utils")
 const { doInTenant } = require("@budibase/auth/tenancy")
 const env = require("../environment")
 const usage = require("../utilities/usageQuota")
+const { definitions: triggerDefs } = require("../automations/triggerInfo")
 
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
+const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: false, status: "STOPPED" }
 
 /**
@@ -23,6 +25,8 @@ class Orchestrator {
     this._chainCount = this._metadata ? this._metadata.automationChainCount : 0
     this._appId = triggerOutput.appId
     this._app = null
+    const triggerStepId = automation.definition.trigger.stepId
+    triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
     // remove from context
     delete triggerOutput.appId
     delete triggerOutput.metadata
@@ -34,11 +38,17 @@ class Orchestrator {
     this._emitter = new AutomationEmitter(this._chainCount + 1)
     this.executionOutput = { trigger: {}, steps: [] }
     // setup the execution output
-    const triggerStepId = automation.definition.trigger.stepId
     const triggerId = automation.definition.trigger.id
     this.updateExecutionOutput(triggerId, triggerStepId, null, triggerOutput)
   }
 
+  cleanupTriggerOutputs(stepId, triggerOutput) {
+    if (stepId === CRON_STEP_ID) {
+      triggerOutput.timestamp = Date.now()
+    }
+    return triggerOutput
+  }
+
   async getStepFunctionality(stepId) {
     let step = await actions.getAction(stepId)
     if (step == null) {
@@ -119,10 +129,17 @@ class Orchestrator {
   }
 }
 
-module.exports = async job => {
+module.exports = (input, callback) => {
   const automationOrchestrator = new Orchestrator(
-    job.data.automation,
-    job.data.event
+    input.data.automation,
+    input.data.event
   )
-  return automationOrchestrator.execute()
+  automationOrchestrator
+    .execute()
+    .then(response => {
+      callback(null, response)
+    })
+    .catch(err => {
+      callback(err)
+    })
 }

package/src/threads/index.js

@@ -0,0 +1,60 @@
+const workerFarm = require("worker-farm")
+const env = require("../environment")
+
+const ThreadType = {
+  QUERY: "query",
+  AUTOMATION: "automation",
+}
+
+function typeToFile(type) {
+  let filename = null
+  switch (type) {
+    case ThreadType.QUERY:
+      filename = "./query"
+      break
+    case ThreadType.AUTOMATION:
+      filename = "./automation"
+      break
+    default:
+      throw "Unknown thread type"
+  }
+  return require.resolve(filename)
+}
+
+class Thread {
+  constructor(type, opts = { timeoutMs: null, count: 1 }) {
+    this.type = type
+    if (!env.isTest()) {
+      const workerOpts = {
+        autoStart: true,
+        maxConcurrentWorkers: opts.count ? opts.count : 1,
+      }
+      if (opts.timeoutMs) {
+        workerOpts.maxCallTime = opts.timeoutMs
+      }
+      this.workers = workerFarm(workerOpts, typeToFile(type))
+    }
+  }
+
+  run(data) {
+    return new Promise((resolve, reject) => {
+      let fncToCall
+      // if in test then don't use threading
+      if (env.isTest()) {
+        fncToCall = require(typeToFile(this.type))
+      } else {
+        fncToCall = this.workers
+      }
+      fncToCall(data, (err, response) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve(response)
+        }
+      })
+    })
+  }
+}
+
+module.exports.Thread = Thread
+module.exports.ThreadType = ThreadType

package/src/threads/query.js

@@ -0,0 +1,63 @@
+const ScriptRunner = require("../utilities/scriptRunner")
+const { integrations } = require("../integrations")
+
+function formatResponse(resp) {
+  if (typeof resp === "string") {
+    try {
+      resp = JSON.parse(resp)
+    } catch (err) {
+      resp = { response: resp }
+    }
+  }
+  return resp
+}
+
+async function runAndTransform(datasource, queryVerb, query, transformer) {
+  const Integration = integrations[datasource.source]
+  if (!Integration) {
+    throw "Integration type does not exist."
+  }
+  const integration = new Integration(datasource.config)
+
+  let rows = formatResponse(await integration[queryVerb](query))
+
+  // transform as required
+  if (transformer) {
+    const runner = new ScriptRunner(transformer, { data: rows })
+    rows = runner.execute()
+  }
+
+  // needs to an array for next step
+  if (!Array.isArray(rows)) {
+    rows = [rows]
+  }
+
+  // map into JSON if just raw primitive here
+  if (rows.find(row => typeof row !== "object")) {
+    rows = rows.map(value => ({ value }))
+  }
+
+  // get all the potential fields in the schema
+  let keys = rows.flatMap(Object.keys)
+
+  if (integration.end) {
+    integration.end()
+  }
+
+  return { rows, keys }
+}
+
+module.exports = (input, callback) => {
+  runAndTransform(
+    input.datasource,
+    input.queryVerb,
+    input.query,
+    input.transformer
+  )
+    .then(response => {
+      callback(null, response)
+    })
+    .catch(err => {
+      callback(err)
+    })
+}

package/src/utilities/csvParser.js

@@ -51,7 +51,7 @@ function parse(csvString, parsers) {
     })
     result.subscribe(row => {
       // For each CSV row parse all the columns that need parsed
-      for (let key in parsers) {
+      for (let key of Object.keys(parsers)) {
         if (!schema[key] || schema[key].success) {
           // get the validator for the column type
           const validator = VALIDATORS[parsers[key].type]
@@ -76,16 +76,58 @@ function parse(csvString, parsers) {
   })
 }
 
-async function transform({ schema, csvString }) {
+function updateSchema({ schema, existingTable }) {
+  if (!schema) {
+    return schema
+  }
+  const finalSchema = {}
+  const schemaKeyMap = {}
+  Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
+  for (let [key, field] of Object.entries(existingTable.schema)) {
+    const lcKey = key.toLowerCase()
+    const foundKey = schemaKeyMap[lcKey]
+    if (foundKey) {
+      finalSchema[key] = schema[foundKey]
+      finalSchema[key].type = field.type
+    }
+  }
+  return finalSchema
+}
+
+async function transform({ schema, csvString, existingTable }) {
   const colParser = {}
 
-  for (let key in schema) {
+  // make sure the table has all the columns required for import
+  if (existingTable) {
+    schema = updateSchema({ schema, existingTable })
+  }
+
+  for (let key of Object.keys(schema)) {
     colParser[key] = PARSERS[schema[key].type] || schema[key].type
   }
 
   try {
-    const json = await csv({ colParser }).fromString(csvString)
-    return json
+    const data = await csv({ colParser }).fromString(csvString)
+    const schemaKeyMap = {}
+    Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
+    for (let element of data) {
+      if (!data) {
+        continue
+      }
+      for (let key of Object.keys(element)) {
+        const mappedKey = schemaKeyMap[key.toLowerCase()]
+        // isn't a column in the table, remove it
+        if (mappedKey == null) {
+          delete element[key]
+        }
+        // casing is different, fix it in row
+        else if (key !== mappedKey) {
+          element[mappedKey] = element[key]
+          delete element[key]
+        }
+      }
+    }
+    return data
   } catch (err) {
     console.error(`Error transforming CSV to JSON for data import`, err)
     throw err
@@ -95,4 +137,5 @@ async function transform({ schema, csvString }) {
 module.exports = {
   parse,
   transform,
+  updateSchema,
 }