@budibase/server 0.9.185-alpha.0 → 0.9.185-alpha.12

package/src/api/controllers/user.js

@@ -97,6 +97,7 @@ exports.syncUser = async function (ctx) {
       .map(([appId]) => appId)
   }
   for (let prodAppId of prodAppIds) {
+    const roleId = roles[prodAppId]
     const devAppId = getDevelopmentAppID(prodAppId)
     for (let appId of [prodAppId, devAppId]) {
       if (!(await doesDatabaseExist(appId))) {
@@ -115,17 +116,21 @@ exports.syncUser = async function (ctx) {
           tableId: InternalTables.USER_METADATA,
         }
       }
-      let combined
-      if (deleting) {
-        combined = {
-          ...metadata,
-          status: UserStatus.INACTIVE,
-          metadata: BUILTIN_ROLE_IDS.PUBLIC,
-        }
-      } else {
-        combined = combineMetadataAndUser(user, metadata)
+      // assign the roleId for the metadata doc
+      if (roleId) {
+        metadata.roleId = roleId
+      }
+      let combined = !deleting
+        ? combineMetadataAndUser(user, metadata)
+        : {
+            ...metadata,
+            status: UserStatus.INACTIVE,
+            metadata: BUILTIN_ROLE_IDS.PUBLIC,
+          }
+      // if its null then there was no updates required
+      if (combined) {
+        await db.put(combined)
       }
-      await db.put(combined)
     }
   }
   ctx.body = {

package/src/api/routes/query.js

@@ -66,6 +66,7 @@ router
   )
   .get(
     "/api/queries/:queryId",
+    paramResource("queryId"),
     authorized(PermissionTypes.QUERY, PermissionLevels.READ),
     queryController.find
   )

package/src/api/routes/table.js

@@ -53,5 +53,16 @@ router
     authorized(BUILDER),
     tableController.destroy
   )
+  // this is currently builder only, but in the future
+  // it could be carried out by an end user in app,
+  // however some thought will need to be had about
+  // implications for automations (triggers)
+  // new trigger type, bulk rows created
+  .post(
+    "/api/tables/:tableId/import",
+    paramResource("tableId"),
+    authorized(BUILDER),
+    tableController.bulkImport
+  )
 
 module.exports = router

package/src/api/routes/tests/misc.spec.js

@@ -75,7 +75,11 @@ describe("run misc tests", () => {
         },
       })
       const dataImport = {
-        csvString: "a,b,c,d\n1,2,3,4"
+        csvString: "a,b,c,d\n1,2,3,4",
+        schema: {},
+      }
+      for (let col of ["a", "b", "c", "d"]) {
+        dataImport.schema[col] = { type: "string" }
       }
       await tableUtils.handleDataImport(
         config.getAppId(),

package/src/app.ts

@@ -41,16 +41,8 @@ app.use(
 )
 
 if (!env.isTest()) {
-  const bullApp = bullboard.init()
-  app.use(async (ctx: ExtendableContext, next: () => any) => {
-    if (ctx.path.startsWith(bullboard.pathPrefix)) {
-      ctx.status = 200
-      ctx.respond = false
-      bullApp(ctx.req, ctx.res)
-    } else {
-      await next()
-    }
-  })
+  const plugin = bullboard.init()
+  app.use(plugin)
 }
 
 app.context.eventEmitter = eventEmitter

package/src/automations/bullboard.js

@@ -1,6 +1,6 @@
-const { createBullBoard } = require("bull-board")
-const { BullAdapter } = require("bull-board/bullAdapter")
-const express = require("express")
+const { createBullBoard } = require("@bull-board/api")
+const { BullAdapter } = require("@bull-board/api/bullAdapter")
+const { KoaAdapter } = require("@bull-board/koa")
 const env = require("../environment")
 const Queue = env.isTest()
   ? require("../utilities/queue/inMemoryQueue")
@@ -9,23 +9,40 @@ const { JobQueues } = require("../constants")
 const { utils } = require("@budibase/auth/redis")
 const { opts, redisProtocolUrl } = utils.getRedisOptions()
 
-const redisConfig = redisProtocolUrl || { redis: opts }
-let automationQueue = new Queue(JobQueues.AUTOMATIONS, redisConfig)
+const CLEANUP_PERIOD_MS = 60 * 1000
+const queueConfig = redisProtocolUrl || { redis: opts }
+let cleanupInternal = null
 
-exports.pathPrefix = "/bulladmin"
+let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig)
+
+async function cleanup() {
+  await automationQueue.clean(CLEANUP_PERIOD_MS, "completed")
+}
+
+const PATH_PREFIX = "/bulladmin"
 
 exports.init = () => {
-  const expressApp = express()
+  // cleanup the events every 5 minutes
+  if (!cleanupInternal) {
+    cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS)
+    // fire off an initial cleanup
+    cleanup().catch(err => {
+      console.error(`Unable to cleanup automation queue initially - ${err}`)
+    })
+  }
   // Set up queues for bull board admin
   const queues = [automationQueue]
   const adapters = []
+  const serverAdapter = new KoaAdapter()
   for (let queue of queues) {
     adapters.push(new BullAdapter(queue))
   }
-  const { router } = createBullBoard(adapters)
-
-  expressApp.use(exports.pathPrefix, router)
-  return expressApp
+  createBullBoard({
+    queues: adapters,
+    serverAdapter,
+  })
+  serverAdapter.setBasePath(PATH_PREFIX)
+  return serverAdapter.registerPlugin()
 }
 
 exports.queue = automationQueue

package/src/automations/tests/automation.spec.js

@@ -1,5 +1,5 @@
 jest.mock("../../utilities/usageQuota")
-jest.mock("../thread")
+jest.mock("../../threads/automation")
 jest.mock("../../utilities/redis", () => ({
   init: jest.fn(),
   checkTestFlag: () => {
@@ -11,8 +11,7 @@ jest.spyOn(global.console, "error")
 
 require("../../environment")
 const automation = require("../index")
-const usageQuota = require("../../utilities/usageQuota")
-const thread = require("../thread")
+const thread = require("../../threads/automation")
 const triggers = require("../triggers")
 const { basicAutomation } = require("../../tests/utilities/structures")
 const { wait } = require("../../utilities")
@@ -62,7 +61,7 @@ describe("Run through some parts of the automations system", () => {
           }
         }
       }
-    }))
+    }), expect.any(Function))
   })
 
   it("should be able to clean inputs with the utilities", () => {

package/src/automations/triggers.js

@@ -11,6 +11,10 @@ const utils = require("./utils")
 const env = require("../environment")
 
 const TRIGGER_DEFINITIONS = definitions
+const JOB_OPTS = {
+  removeOnComplete: true,
+  removeOnFail: true,
+}
 
 async function queueRelevantRowAutomations(event, eventType) {
   if (event.appId == null) {
@@ -47,7 +51,7 @@ async function queueRelevantRowAutomations(event, eventType) {
       automationTrigger.inputs &&
       automationTrigger.inputs.tableId === event.row.tableId
     ) {
-      await queue.add({ automation, event })
+      await queue.add({ automation, event }, JOB_OPTS)
     }
   }
 }
@@ -86,7 +90,7 @@ exports.externalTrigger = async function (
     automation.definition.trigger != null &&
     automation.definition.trigger.stepId === definitions.APP.stepId &&
     automation.definition.trigger.stepId === "APP" &&
-    !checkTestFlag(automation._id)
+    !(await checkTestFlag(automation._id))
   ) {
     // values are likely to be submitted as strings, so we shall convert to correct type
     const coercedFields = {}
@@ -100,7 +104,7 @@ exports.externalTrigger = async function (
   if (getResponses) {
     return utils.processEvent({ data })
   } else {
-    return queue.add(data)
+    return queue.add(data, JOB_OPTS)
   }
 }
 

package/src/automations/utils.js

@@ -1,4 +1,4 @@
-const runner = require("./thread")
+const { Thread, ThreadType } = require("../threads")
 const { definitions } = require("./triggerInfo")
 const webhooks = require("../api/controllers/webhook")
 const CouchDB = require("../db")
@@ -10,11 +10,12 @@ const { getDeployedAppID } = require("@budibase/auth/db")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
+const Runner = new Thread(ThreadType.AUTOMATION)
 
 exports.processEvent = async job => {
   try {
     // need to actually await these so that an error can be captured properly
-    return await runner(job)
+    return await Runner.run(job)
   } catch (err) {
     console.error(
       `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
@@ -162,3 +163,12 @@ exports.checkForWebhooks = async ({ appId, oldAuto, newAuto }) => {
   }
   return newAuto
 }
+
+/**
+ * When removing an app/unpublishing it need to make sure automations are cleaned up (cron).
+ * @param appId {string} the app that is being removed.
+ * @return {Promise<void>} clean is complete if this succeeds.
+ */
+exports.cleanupAutomations = async appId => {
+  await exports.disableAllCrons(appId)
+}

package/src/constants/index.js

@@ -69,6 +69,7 @@ exports.DataSourceOperation = {
   READ: "READ",
   UPDATE: "UPDATE",
   DELETE: "DELETE",
+  BULK_CREATE: "BULK_CREATE",
   CREATE_TABLE: "CREATE_TABLE",
   UPDATE_TABLE: "UPDATE_TABLE",
   DELETE_TABLE: "DELETE_TABLE",

package/src/definitions/datasource.ts

@@ -1,10 +1,11 @@
-import { Table } from "./common"
+import { Row, Table } from "./common"
 
 export enum Operation {
   CREATE = "CREATE",
   READ = "READ",
   UPDATE = "UPDATE",
   DELETE = "DELETE",
+  BULK_CREATE = "BULK_CREATE",
   CREATE_TABLE = "CREATE_TABLE",
   UPDATE_TABLE = "UPDATE_TABLE",
   DELETE_TABLE = "DELETE_TABLE",
@@ -144,7 +145,7 @@ export interface QueryJson {
   filters?: SearchFilters
   sort?: SortJson
   paginate?: PaginationJson
-  body?: object
+  body?: Row | Row[]
   table?: Table
   meta?: {
     table?: Table

package/src/integrations/airtable.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module AirtableModule {
   const Airtable = require("airtable")
@@ -73,7 +74,7 @@ module AirtableModule {
     },
   }
 
-  class AirtableIntegration {
+  class AirtableIntegration implements IntegrationBase {
     private config: AirtableConfig
     private client: any
 

package/src/integrations/arangodb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module ArangoModule {
   const { Database, aql } = require("arangojs")
@@ -55,7 +56,7 @@ module ArangoModule {
     },
   }
 
-  class ArangoDBIntegration {
+  class ArangoDBIntegration implements IntegrationBase {
     private config: ArangodbConfig
     private client: any
 

package/src/integrations/base/IntegrationBase.ts

@@ -0,0 +1,6 @@
+export interface IntegrationBase {
+  create?(query: any): Promise<[any]>
+  read?(query: any): Promise<[any]>
+  update?(query: any): Promise<[any]>
+  delete?(query: any): Promise<[any]>
+}

package/src/integrations/base/datasourcePlus.ts

@@ -1,6 +1,7 @@
 import { Table } from "../../definitions/common"
+import { IntegrationBase } from "./IntegrationBase"
 
-export interface DatasourcePlus {
+export interface DatasourcePlus extends IntegrationBase {
   tables: Record<string, Table>
   schemaErrors: Record<string, string>
 

package/src/integrations/base/sql.ts

@@ -13,22 +13,50 @@ import SqlTableQueryBuilder from "./sqlTable"
 const BASE_LIMIT = 5000
 
 type KnexQuery = Knex.QueryBuilder | Knex
+// these are invalid dates sent by the client, need to convert them to a real max date
+const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z"
+const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z"
+
+function parse(input: any) {
+  if (Array.isArray(input)) {
+    return JSON.stringify(input)
+  }
+  if (typeof input !== "string") {
+    return input
+  }
+  if (input === MAX_ISO_DATE) {
+    return new Date(8640000000000000)
+  }
+  if (input === MIN_ISO_DATE) {
+    return new Date(-8640000000000000)
+  }
+  if (isIsoDateString(input)) {
+    return new Date(input)
+  }
+  return input
+}
 
 function parseBody(body: any) {
   for (let [key, value] of Object.entries(body)) {
-    if (Array.isArray(value)) {
-      body[key] = JSON.stringify(value)
-    }
-    if (typeof value !== "string") {
-      continue
-    }
-    if (isIsoDateString(value)) {
-      body[key] = new Date(value)
-    }
+    body[key] = parse(value)
   }
   return body
 }
 
+function parseFilters(filters: SearchFilters): SearchFilters {
+  for (let [key, value] of Object.entries(filters)) {
+    let parsed
+    if (typeof value === "object") {
+      parsed = parseFilters(value)
+    } else {
+      parsed = parse(value)
+    }
+    // @ts-ignore
+    filters[key] = parsed
+  }
+  return filters
+}
+
 class InternalBuilder {
   private readonly client: string
 
@@ -53,6 +81,7 @@ class InternalBuilder {
     if (!filters) {
       return query
     }
+    filters = parseFilters(filters)
     // if all or specified in filters, then everything is an or
     const allOr = filters.allOr
     if (filters.oneOf) {
@@ -179,6 +208,16 @@ class InternalBuilder {
     }
   }
 
+  bulkCreate(knex: Knex, json: QueryJson): KnexQuery {
+    const { endpoint, body } = json
+    let query: KnexQuery = knex(endpoint.entityId)
+    if (!Array.isArray(body)) {
+      return query
+    }
+    const parsedBody = body.map(row => parseBody(row))
+    return query.insert(parsedBody)
+  }
+
   read(knex: Knex, json: QueryJson, limit: number): KnexQuery {
     let { endpoint, resource, filters, sort, paginate, relationships } = json
     const tableName = endpoint.entityId
@@ -294,6 +333,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
       case Operation.DELETE:
         query = builder.delete(client, json, opts)
         break
+      case Operation.BULK_CREATE:
+        query = builder.bulkCreate(client, json)
+        break
       case Operation.CREATE_TABLE:
       case Operation.UPDATE_TABLE:
       case Operation.DELETE_TABLE:

package/src/integrations/base/sqlTable.ts

@@ -29,10 +29,7 @@ function generateSchema(
   for (let [key, column] of Object.entries(table.schema)) {
     // skip things that are already correct
     const oldColumn = oldTable ? oldTable.schema[key] : null
-    if (
-      (oldColumn && oldColumn.type) ||
-      (primaryKey === key && !isJunction)
-    ) {
+    if ((oldColumn && oldColumn.type) || (primaryKey === key && !isJunction)) {
       continue
     }
     switch (column.type) {

package/src/integrations/couchdb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module CouchDBModule {
   const PouchDB = require("pouchdb")
@@ -50,7 +51,7 @@ module CouchDBModule {
     },
   }
 
-  class CouchDBIntegration {
+  class CouchDBIntegration implements IntegrationBase {
     private config: CouchDBConfig
     private client: any
 

package/src/integrations/dynamodb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module DynamoModule {
   const AWS = require("aws-sdk")
@@ -113,7 +114,7 @@ module DynamoModule {
     },
   }
 
-  class DynamoDBIntegration {
+  class DynamoDBIntegration implements IntegrationBase {
     private config: DynamoDBConfig
     private client: any
 

package/src/integrations/elasticsearch.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module ElasticsearchModule {
   const { Client } = require("@elastic/elasticsearch")
@@ -74,7 +75,7 @@ module ElasticsearchModule {
     },
   }
 
-  class ElasticSearchIntegration {
+  class ElasticSearchIntegration implements IntegrationBase {
     private config: ElasticsearchConfig
     private client: any
 

package/src/integrations/mongodb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module MongoDBModule {
   const { MongoClient } = require("mongodb")
@@ -62,7 +63,7 @@ module MongoDBModule {
     },
   }
 
-  class MongoIntegration {
+  class MongoIntegration implements IntegrationBase {
     private config: MongoDBConfig
     private client: any
 

package/src/integrations/mysql.ts

@@ -184,7 +184,7 @@ module MySQLModule {
       return results.length ? results : [{ created: true }]
     }
 
-    read(query: SqlQuery | string) {
+    async read(query: SqlQuery | string) {
       return internalQuery(this.client, getSqlQuery(query))
     }
 

package/src/integrations/postgres.ts

@@ -130,7 +130,7 @@ module PostgresModule {
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
 
-    COLUMNS_SQL!: string 
+    COLUMNS_SQL!: string
 
     PRIMARY_KEYS_SQL = `
     select tc.table_schema, tc.table_name, kc.column_name as primary_key 
@@ -165,11 +165,11 @@ module PostgresModule {
 
     setSchema() {
       if (!this.config.schema) {
-        this.config.schema = 'public'
+        this.config.schema = "public"
       }
-      this.client.on('connect', (client: any) => {    
-        client.query(`SET search_path TO ${this.config.schema}`);
-      });
+      this.client.on("connect", (client: any) => {
+        client.query(`SET search_path TO ${this.config.schema}`)
+      })
       this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
     }
 

package/src/integrations/rest.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module RestModule {
   const fetch = require("node-fetch")
@@ -131,7 +132,7 @@ module RestModule {
     },
   }
 
-  class RestIntegration {
+  class RestIntegration implements IntegrationBase {
     private config: RestConfig
     private headers: {
       [key: string]: string

package/src/integrations/s3.ts

@@ -1,4 +1,5 @@
 import { Integration, QueryTypes } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module S3Module {
   const AWS = require("aws-sdk")
@@ -42,7 +43,7 @@ module S3Module {
     },
   }
 
-  class S3Integration {
+  class S3Integration implements IntegrationBase {
     private readonly config: S3Config
     private client: any
     private connectionPromise: Promise<any>

package/src/middleware/builder.js

@@ -8,6 +8,7 @@ const {
 const CouchDB = require("../db")
 const { DocumentTypes } = require("../db/utils")
 const { PermissionTypes } = require("@budibase/auth/permissions")
+const { app: appCache } = require("@budibase/auth/cache")
 
 const DEBOUNCE_TIME_SEC = 30
 
@@ -50,7 +51,9 @@ async function updateAppUpdatedAt(ctx) {
   const db = new CouchDB(appId)
   const metadata = await db.get(DocumentTypes.APP_METADATA)
   metadata.updatedAt = new Date().toISOString()
-  await db.put(metadata)
+  const response = await db.put(metadata)
+  metadata._rev = response.rev
+  await appCache.invalidateAppMetadata(appId, metadata)
   // set a new debounce record with a short TTL
   await setDebounce(appId, DEBOUNCE_TIME_SEC)
 }