@budibase/server 2.2.12-alpha.2 → 2.2.12-alpha.21

package/src/api/routes/user.ts

@@ -48,4 +48,4 @@ router
     controller.getFlags
   )
 
-export = router
+export default router

package/src/api/routes/utils/validators.ts

@@ -18,7 +18,7 @@ export function tableValidator() {
     schema: Joi.object().required(),
     name: Joi.string().required(),
     views: Joi.object(),
-    dataImport: Joi.object(),
+    rows: Joi.array(),
   }).unknown(true))
 }
 

package/src/api/routes/view.ts

@@ -31,4 +31,4 @@ router
   )
   .post("/api/views", authorized(permissions.BUILDER), viewController.save)
 
-export = router
+export default router

package/src/api/routes/webhook.ts

@@ -24,4 +24,4 @@ router
   // this shouldn't have authorisation, right now its always public
   .post("/api/webhooks/trigger/:instance/:id", controller.trigger)
 
-export = router
+export default router

package/src/app.ts

@@ -1,5 +1,5 @@
 // need to load environment first
-import * as env from "./environment"
+import env from "./environment"
 
 // enable APM if configured
 if (process.env.ELASTIC_APM_ENABLED) {
@@ -19,7 +19,7 @@ import * as api from "./api"
 import * as automations from "./automations"
 import { Thread } from "./threads"
 import * as redis from "./utilities/redis"
-import { events, logging } from "@budibase/backend-core"
+import { events, logging, middleware } from "@budibase/backend-core"
 import { initialise as initialiseWebsockets } from "./websocket"
 import { startup } from "./startup"
 const Sentry = require("@sentry/node")
@@ -40,6 +40,8 @@ app.use(
   })
 )
 
+app.use(middleware.logging)
+
 if (env.isProd()) {
   env._set("NODE_ENV", "production")
   Sentry.init()
@@ -77,7 +79,7 @@ server.on("close", async () => {
   }
 })
 
-export = server.listen(env.PORT || 0, async () => {
+export default server.listen(env.PORT || 0, async () => {
   await startup(app, server)
 })
 

package/src/automations/logging/index.ts

@@ -1,4 +1,4 @@
-import * as env from "../../environment"
+import env from "../../environment"
 import { AutomationResults, Automation, App } from "@budibase/types"
 import { automations } from "@budibase/pro"
 import { db as dbUtils } from "@budibase/backend-core"

package/src/db/index.ts

@@ -1,5 +1,5 @@
 import { init as coreInit } from "@budibase/backend-core"
-import env = require("../environment")
+import env from "../environment"
 
 export function init() {
   const dbConfig: any = {

package/src/db/linkedRows/LinkController.ts

@@ -441,4 +441,4 @@ class LinkController {
   }
 }
 
-export = LinkController
+export default LinkController

package/src/db/linkedRows/LinkDocument.ts

@@ -57,4 +57,4 @@ class LinkDocumentImpl implements LinkDocument {
   }
 }
 
-export = LinkDocumentImpl
+export default LinkDocumentImpl

package/src/db/newid.ts

@@ -1,5 +1,5 @@
 const { v4 } = require("uuid")
 
-export = function (): string {
+export default function (): string {
   return v4().replace(/-/g, "")
 }

package/src/db/tests/linkController.spec.js

@@ -1,6 +1,6 @@
 const TestConfig = require("../../tests/utilities/TestConfiguration")
 const { basicRow, basicLinkedRow, basicTable } = require("../../tests/utilities/structures")
-const LinkController = require("../linkedRows/LinkController")
+const LinkController = require("../linkedRows/LinkController").default
 const { context } = require("@budibase/backend-core")
 const { RelationshipTypes } = require("../../constants")
 const { cloneDeep } = require("lodash/fp")

package/src/environment.ts

@@ -125,4 +125,4 @@ for (let [key, value] of Object.entries(environment)) {
   }
 }
 
-export = environment
+export default environment

package/src/events/AutomationEmitter.ts

@@ -55,4 +55,4 @@ class AutomationEmitter {
   }
 }
 
-export = AutomationEmitter
+export default AutomationEmitter

package/src/events/BudibaseEmitter.ts

@@ -26,4 +26,4 @@ class BudibaseEmitter extends EventEmitter {
   }
 }
 
-export = BudibaseEmitter
+export default BudibaseEmitter

package/src/events/index.ts

@@ -2,4 +2,4 @@ import BudibaseEmitter from "./BudibaseEmitter"
 
 const emitter = new BudibaseEmitter()
 
-export = emitter
+export default emitter

package/src/integrations/index.ts

@@ -17,8 +17,8 @@ import oracle from "./oracle"
 import { getPlugins } from "../api/controllers/plugin"
 import { SourceName, Integration, PluginType } from "@budibase/types"
 import { getDatasourcePlugin } from "../utilities/fileSystem"
-const environment = require("../environment")
-const { cloneDeep } = require("lodash")
+import env from "../environment"
+import { cloneDeep } from "lodash"
 
 const DEFINITIONS: { [key: string]: Integration } = {
   [SourceName.POSTGRES]: postgres.schema,
@@ -69,7 +69,7 @@ if (
 
 export async function getDefinitions() {
   const pluginSchemas: { [key: string]: Integration } = {}
-  if (environment.SELF_HOSTED) {
+  if (env.SELF_HOSTED) {
     const plugins = await getPlugins(PluginType.DATASOURCE)
     // extract the actual schema from each custom
     for (let plugin of plugins) {
@@ -93,7 +93,7 @@ export async function getIntegration(integration: string) {
   if (INTEGRATIONS[integration]) {
     return INTEGRATIONS[integration]
   }
-  if (environment.SELF_HOSTED) {
+  if (env.SELF_HOSTED) {
     const plugins = await getPlugins(PluginType.DATASOURCE)
     for (let plugin of plugins) {
       if (plugin.name === integration) {

package/src/integrations/tests/rest.spec.ts

@@ -1,16 +1,18 @@
-const mockFetch = jest.fn(() => ({
-  headers: {
-    raw: () => {
-      return { "content-type": ["application/json"] }
+jest.mock("node-fetch", () => {
+  return jest.fn(() => ({
+    headers: {
+      raw: () => {
+        return { "content-type": ["application/json"] }
+      },
+      get: () => ["application/json"],
     },
-    get: () => ["application/json"],
-  },
-  json: jest.fn(() => ({
-    my_next_cursor: 123,
-  })),
-  text: jest.fn(),
-}))
-jest.mock("node-fetch", () => mockFetch)
+    json: jest.fn(() => ({
+      my_next_cursor: 123,
+    })),
+    text: jest.fn(),
+  }))
+})
+
 import fetch from "node-fetch"
 import { default as RestIntegration } from "../rest"
 const FormData = require("form-data")
@@ -579,7 +581,7 @@ describe("REST Integration", () => {
     })
     await config.integration.read({})
 
-    const calls: any = mockFetch.mock.calls[0]
+    const calls: any = fetch.mock.calls[0]
     const url = calls[0]
     expect(url).toBe(`${BASE_URL}/`)
 

package/src/middleware/authorized.ts

@@ -63,7 +63,11 @@ const checkAuthorizedResource = async (
   }
 }
 
-export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
+export default (
+    permType: any,
+    permLevel: any = null,
+    opts = { schema: false }
+  ) =>
   async (ctx: any, next: any) => {
     // webhooks don't need authentication, each webhook unique
     // also internal requests (between services) don't need authorized

package/src/middleware/builder.ts

@@ -64,7 +64,7 @@ async function updateAppUpdatedAt(ctx: BBContext) {
   })
 }
 
-export = async function builder(ctx: BBContext, permType: string) {
+export default async function builder(ctx: BBContext, permType: string) {
   const appId = ctx.appId
   // this only functions within an app context
   if (!appId) {

package/src/middleware/currentapp.ts

@@ -12,7 +12,7 @@ import env from "../environment"
 import { isWebhookEndpoint } from "./utils"
 import { BBContext } from "@budibase/types"
 
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   // try to get the appID from the request
   let requestAppId = await utils.getAppIdFromCtx(ctx)
   // get app cookie if it exists
@@ -25,6 +25,7 @@ export = async (ctx: BBContext, next: any) => {
   if (!appCookie && !requestAppId) {
     return next()
   }
+
   // check the app exists referenced in cookie
   if (appCookie) {
     const appId = appCookie.appId
@@ -51,7 +52,7 @@ export = async (ctx: BBContext, next: any) => {
 
   let appId: string | undefined,
     roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
-  if (!ctx.user) {
+  if (!ctx.user?._id) {
     // not logged in, try to set a cookie for public apps
     appId = requestAppId
   } else if (requestAppId != null) {
@@ -96,7 +97,7 @@ export = async (ctx: BBContext, next: any) => {
     // need to judge this only based on the request app ID,
     if (
       env.MULTI_TENANCY &&
-      ctx.user &&
+      ctx.user?._id &&
       requestAppId &&
       !tenancy.isUserInAppTenant(requestAppId, ctx.user)
     ) {

package/src/middleware/publicApi.ts

@@ -1,7 +1,7 @@
 import { constants, utils } from "@budibase/backend-core"
 import { BBContext } from "@budibase/types"
 
-export = function ({ requiresAppId }: { requiresAppId?: boolean } = {}) {
+export default function ({ requiresAppId }: { requiresAppId?: boolean } = {}) {
   return async (ctx: BBContext, next: any) => {
     const appId = await utils.getAppIdFromCtx(ctx)
     if (requiresAppId && !appId) {

package/src/middleware/selfhost.ts

@@ -3,7 +3,7 @@ import { BBContext } from "@budibase/types"
 
 // if added as a middleware will stop requests unless builder is in self host mode
 // or cloud is in self host
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   if (env.SELF_HOSTED) {
     await next()
     return

package/src/middleware/tests/authorized.spec.js

@@ -7,7 +7,7 @@ jest.mock("../../environment", () => ({
     }
   })
 )
-const authorizedMiddleware = require("../authorized")
+const authorizedMiddleware = require("../authorized").default
 const env = require("../../environment")
 const { permissions } = require("@budibase/backend-core")
 

package/src/middleware/tests/currentapp.spec.js

@@ -124,7 +124,7 @@ class TestConfiguration {
     // import as late as possible for mocks
     jest.resetModules()
     require("../../db").init()
-    const currentAppMiddleware = require("../currentapp")
+    const currentAppMiddleware = require("../currentapp").default
     return currentAppMiddleware(this.ctx, this.next)
   }
 }

package/src/middleware/tests/selfhost.spec.js

@@ -1,4 +1,4 @@
-const selfHostMiddleware = require("../selfhost")
+const selfHostMiddleware = require("../selfhost").default
 const env = require("../../environment")
 jest.mock("../../environment")
 

package/src/startup.ts

@@ -1,16 +1,11 @@
-import * as env from "./environment"
+import env from "./environment"
 import * as redis from "./utilities/redis"
 import {
   createAdminUser,
   generateApiKey,
   getChecklist,
 } from "./utilities/workerRequests"
-import {
-  installation,
-  pinoSettings,
-  tenancy,
-  logging,
-} from "@budibase/backend-core"
+import { installation, tenancy, logging } from "@budibase/backend-core"
 import fs from "fs"
 import { watch } from "./watch"
 import * as automations from "./automations"
@@ -26,7 +21,7 @@ const pino = require("koa-pino-logger")
 let STARTUP_RAN = false
 
 async function initRoutes(app: any) {
-  app.use(pino(pinoSettings()))
+  app.use(pino(logging.pinoSettings()))
 
   if (!env.isTest()) {
     const plugin = await bullboard.init()

package/src/tests/jestEnv.ts

@@ -0,0 +1,13 @@
+import env from "../environment"
+import { tmpdir } from "os"
+
+env._set("SELF_HOSTED", "1")
+env._set("NODE_ENV", "jest")
+env._set("JWT_SECRET", "test-jwtsecret")
+env._set("CLIENT_ID", "test-client-id")
+env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
+env._set("LOG_LEVEL", "silent")
+env._set("PORT", 0)
+env._set("MINIO_URL", "http://localhost")
+env._set("MINIO_ACCESS_KEY", "test")
+env._set("MINIO_SECRET_KEY", "test")

package/src/tests/jestSetup.ts

@@ -1,17 +1,3 @@
-const { tmpdir } = require("os")
-import env from "../environment"
-
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-env._set("JWT_SECRET", "test-jwtsecret")
-env._set("CLIENT_ID", "test-client-id")
-env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
-env._set("LOG_LEVEL", "silent")
-env._set("PORT", 0)
-env._set("MINIO_URL", "http://localhost")
-env._set("MINIO_ACCESS_KEY", "test")
-env._set("MINIO_SECRET_KEY", "test")
-
 import { mocks } from "@budibase/backend-core/tests"
 
 // mock all dates to 2020-01-01T00:00:00.000Z

package/src/tests/utilities/TestConfiguration.ts

@@ -67,7 +67,7 @@ class TestConfiguration {
     if (openServer) {
       // use a random port because it doesn't matter
       env.PORT = "0"
-      this.server = require("../../app")
+      this.server = require("../../app").default
       // we need the request for logging in, involves cookies, hard to fake
       this.request = supertest(this.server)
       this.started = true

package/src/threads/index.ts

@@ -1,5 +1,5 @@
 import workerFarm from "worker-farm"
-import * as env from "../environment"
+import env from "../environment"
 
 export const ThreadType = {
   QUERY: "query",

package/src/utilities/fileSystem/filesystem.ts

@@ -3,7 +3,7 @@ const { budibaseTempDir } = require("../budibaseDir")
 const fs = require("fs")
 const { join } = require("path")
 const uuid = require("uuid/v4")
-const env = require("../../environment")
+import env from "../../environment"
 import tar from "tar"
 
 export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")

package/src/utilities/schema.ts

@@ -0,0 +1,141 @@
+import { FieldTypes } from "../constants"
+
+interface SchemaColumn {
+  readonly name: string
+  readonly type: FieldTypes
+  readonly autocolumn?: boolean
+}
+
+interface Schema {
+  readonly [index: string]: SchemaColumn
+}
+
+interface Row {
+  [index: string]: any
+}
+
+type Rows = Array<Row>
+
+interface SchemaValidation {
+  [index: string]: boolean
+}
+
+interface ValidationResults {
+  schemaValidation: SchemaValidation
+  allValid: boolean
+  invalidColumns: Array<string>
+}
+
+const PARSERS: any = {
+  [FieldTypes.NUMBER]: (attribute?: string) => {
+    if (!attribute) {
+      return attribute
+    }
+    return Number(attribute)
+  },
+  [FieldTypes.DATETIME]: (attribute?: string) => {
+    if (!attribute) {
+      return attribute
+    }
+    return new Date(attribute).toISOString()
+  },
+}
+
+export function isSchema(schema: any): schema is Schema {
+  return (
+    typeof schema === "object" &&
+    Object.values(schema).every(rawColumn => {
+      const column = rawColumn as SchemaColumn
+
+      return (
+        column !== null &&
+        typeof column === "object" &&
+        typeof column.type === "string" &&
+        Object.values(FieldTypes).includes(column.type as FieldTypes)
+      )
+    })
+  )
+}
+
+export function isRows(rows: any): rows is Rows {
+  return Array.isArray(rows) && rows.every(row => typeof row === "object")
+}
+
+export function validate(rows: Rows, schema: Schema): ValidationResults {
+  const results: ValidationResults = {
+    schemaValidation: {},
+    allValid: false,
+    invalidColumns: [],
+  }
+
+  rows.forEach(row => {
+    Object.entries(row).forEach(([columnName, columnData]) => {
+      const columnType = schema[columnName]?.type
+      const isAutoColumn = schema[columnName]?.autocolumn
+
+      // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
+      if (typeof columnType !== "string") {
+        results.invalidColumns.push(columnName)
+      } else if (
+        // If there's no data for this field don't bother with further checks
+        // If the field is already marked as invalid there's no need for further checks
+        results.schemaValidation[columnName] === false ||
+        columnData == null ||
+        isAutoColumn
+      ) {
+        return
+      } else if (
+        columnType === FieldTypes.NUMBER &&
+        isNaN(Number(columnData))
+      ) {
+        // If provided must be a valid number
+        results.schemaValidation[columnName] = false
+      } else if (
+        // If provided must be a valid date
+        columnType === FieldTypes.DATETIME &&
+        isNaN(new Date(columnData).getTime())
+      ) {
+        results.schemaValidation[columnName] = false
+      } else {
+        results.schemaValidation[columnName] = true
+      }
+    })
+  })
+
+  results.allValid =
+    Object.values(results.schemaValidation).length > 0 &&
+    Object.values(results.schemaValidation).every(column => column)
+
+  // Select unique values
+  results.invalidColumns = [...new Set(results.invalidColumns)]
+  return results
+}
+
+export function parse(rows: Rows, schema: Schema): Rows {
+  return rows.map(row => {
+    const parsedRow: Row = {}
+
+    Object.entries(row).forEach(([columnName, columnData]) => {
+      if (!(columnName in schema) || schema[columnName]?.autocolumn) {
+        // Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case
+        return
+      }
+
+      const columnType = schema[columnName].type
+
+      if (columnType === FieldTypes.NUMBER) {
+        // If provided must be a valid number
+        parsedRow[columnName] = columnData ? Number(columnData) : columnData
+      } else if (columnType === FieldTypes.DATETIME) {
+        // If provided must be a valid date
+        parsedRow[columnName] = columnData
+          ? new Date(columnData).toISOString()
+          : columnData
+      } else {
+        parsedRow[columnName] = columnData
+      }
+    })
+
+    return parsedRow
+  })
+}

package/src/utilities/scriptRunner.ts

@@ -25,4 +25,4 @@ class ScriptRunner {
   }
 }
 
-export = ScriptRunner
+export default ScriptRunner

package/src/utilities/statusCodes.ts

@@ -1,4 +1,4 @@
-export = {
+export default {
   OK: 200,
   UNAUTHORIZED: 401,
   FORBIDDEN: 403,

package/src/utilities/workerRequests.ts

@@ -1,7 +1,12 @@
 import fetch from "node-fetch"
 import env from "../environment"
 import { checkSlashesInUrl } from "./index"
-import { db as dbCore, constants, tenancy } from "@budibase/backend-core"
+import {
+  db as dbCore,
+  constants,
+  tenancy,
+  logging,
+} from "@budibase/backend-core"
 import { updateAppRole } from "./global"
 import { BBContext, User } from "@budibase/types"
 
@@ -27,6 +32,10 @@ export function request(ctx?: BBContext, request?: any) {
   if (ctx && ctx.headers) {
     request.headers = ctx.headers
   }
+
+  // add x-budibase-correlation-id header
+  logging.correlation.setHeader(request.headers)
+
   return request
 }
 

package/src/watch.ts

@@ -1,5 +1,5 @@
 import path from "path"
-import * as env from "./environment"
+import env from "./environment"
 import chokidar from "chokidar"
 import fs from "fs"
 import { constants, tenancy } from "@budibase/backend-core"