@budibase/server 2.2.12-alpha.4 → 2.2.12-alpha.41

package/src/api/controllers/user.ts

@@ -4,19 +4,21 @@ import { getGlobalUsers, getRawGlobalUser } from "../../utilities/global"
 import { getFullUser } from "../../utilities/users"
 import {
   context,
-  constants,
   roles as rolesCore,
   db as dbCore,
 } from "@budibase/backend-core"
-import { BBContext, User } from "@budibase/types"
+import { BBContext, Ctx, SyncUserRequest, User } from "@budibase/types"
 import sdk from "../../sdk"
 
-export async function syncUser(ctx: BBContext) {
+export async function syncUser(ctx: Ctx<SyncUserRequest>) {
   let deleting = false,
     user: User | any
   const userId = ctx.params.id
+
+  const previousUser = ctx.request.body?.previousUser
+
   try {
-    user = await getRawGlobalUser(userId)
+    user = (await getRawGlobalUser(userId)) as User
   } catch (err: any) {
     if (err && err.status === 404) {
       user = {}
@@ -25,6 +27,11 @@ export async function syncUser(ctx: BBContext) {
       throw err
     }
   }
+
+  let previousApps = previousUser
+    ? Object.keys(previousUser.roles).map(appId => appId)
+    : []
+
   const roles = deleting ? {} : user.roles
   // remove props which aren't useful to metadata
   delete user.password
@@ -40,8 +47,9 @@ export async function syncUser(ctx: BBContext) {
       .filter(entry => entry[1] !== rolesCore.BUILTIN_ROLE_IDS.PUBLIC)
       .map(([appId]) => appId)
   }
-  for (let prodAppId of prodAppIds) {
+  for (let prodAppId of new Set([...prodAppIds, ...previousApps])) {
     const roleId = roles[prodAppId]
+    const deleteFromApp = !roleId
     const devAppId = dbCore.getDevelopmentAppID(prodAppId)
     for (let appId of [prodAppId, devAppId]) {
       if (!(await dbCore.dbExists(appId))) {
@@ -54,24 +62,24 @@ export async function syncUser(ctx: BBContext) {
         try {
           metadata = await db.get(metadataId)
         } catch (err) {
-          if (deleting) {
+          if (deleteFromApp) {
             return
           }
           metadata = {
             tableId: InternalTables.USER_METADATA,
           }
         }
+
+        if (deleteFromApp) {
+          await db.remove(metadata)
+          return
+        }
+
         // assign the roleId for the metadata doc
         if (roleId) {
           metadata.roleId = roleId
         }
-        let combined = !deleting
-          ? sdk.users.combineMetadataAndUser(user, metadata)
-          : {
-              ...metadata,
-              status: constants.UserStatus.INACTIVE,
-              metadata: rolesCore.BUILTIN_ROLE_IDS.PUBLIC,
-            }
+        let combined = sdk.users.combineMetadataAndUser(user, metadata)
         // if its null then there was no updates required
         if (combined) {
           await db.put(combined)

package/src/api/controllers/view/exporters.ts

@@ -1,4 +1,4 @@
-import { Row } from "@budibase/types"
+import { Row, TableSchema } from "@budibase/types"
 
 export function csv(headers: string[], rows: Row[]) {
   let csv = headers.map(key => `"${key}"`).join(",")
@@ -18,11 +18,26 @@ export function csv(headers: string[], rows: Row[]) {
   return csv
 }
 
-export function json(headers: string[], rows: Row[]) {
+export function json(rows: Row[]) {
   return JSON.stringify(rows, undefined, 2)
 }
 
-export const ExportFormats = {
-  CSV: "csv",
-  JSON: "json",
+export function jsonWithSchema(schema: TableSchema, rows: Row[]) {
+  const newSchema: TableSchema = {}
+  Object.values(schema).forEach(column => {
+    if (!column.autocolumn) {
+      newSchema[column.name] = column
+    }
+  })
+  return JSON.stringify({ schema: newSchema, rows }, undefined, 2)
+}
+
+export enum Format {
+  CSV = "csv",
+  JSON = "json",
+  JSON_WITH_SCHEMA = "jsonWithSchema",
+}
+
+export function isFormat(format: any): format is Format {
+  return Object.values(Format).includes(format as Format)
 }

package/src/api/controllers/view/index.ts

@@ -1,6 +1,6 @@
 import viewTemplate from "./viewBuilder"
 import { apiFileReturn } from "../../../utilities/fileSystem"
-import * as exporters from "./exporters"
+import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters"
 import { deleteView, getView, getViews, saveView } from "./utils"
 import { fetchView } from "../row"
 import { FieldTypes } from "../../../constants"
@@ -127,9 +127,13 @@ export async function exportView(ctx: BBContext) {
   const viewName = decodeURIComponent(ctx.query.view as string)
   const view = await getView(viewName)
 
-  const format = ctx.query.format as string
-  if (!format || !Object.values(exporters.ExportFormats).includes(format)) {
-    ctx.throw(400, "Format must be specified, either csv or json")
+  const format = ctx.query.format as unknown
+
+  if (!isFormat(format)) {
+    ctx.throw(
+      400,
+      "Format must be specified, either csv, json or jsonWithSchema"
+    )
   }
 
   if (view) {
@@ -171,7 +175,7 @@ export async function exportView(ctx: BBContext) {
   })
 
   // make sure no "undefined" entries appear in the CSV
-  if (format === exporters.ExportFormats.CSV) {
+  if (format === Format.CSV) {
     const schemaKeys = Object.keys(schema)
     for (let key of schemaKeys) {
       for (let row of rows) {
@@ -182,13 +186,18 @@ export async function exportView(ctx: BBContext) {
     }
   }
 
-  // Export part
-  let headers = Object.keys(schema)
-  const exporter = format === "csv" ? exporters.csv : exporters.json
-  const filename = `${viewName}.${format}`
-  // send down the file
-  ctx.attachment(filename)
-  ctx.body = apiFileReturn(exporter(headers, rows))
+  if (format === Format.CSV) {
+    ctx.attachment(`${viewName}.csv`)
+    ctx.body = apiFileReturn(csv(Object.keys(schema), rows))
+  } else if (format === Format.JSON) {
+    ctx.attachment(`${viewName}.json`)
+    ctx.body = apiFileReturn(json(rows))
+  } else if (format === Format.JSON_WITH_SCHEMA) {
+    ctx.attachment(`${viewName}.json`)
+    ctx.body = apiFileReturn(jsonWithSchema(schema, rows))
+  } else {
+    throw "Format not recognised"
+  }
 
   if (viewName.startsWith(DocumentType.TABLE)) {
     await events.table.exported(table, format as TableExportFormat)

package/src/api/controllers/view/tests/viewBuilder.spec.js

@@ -1,4 +1,4 @@
-const viewTemplate = require("../viewBuilder");
+const viewTemplate = require("../viewBuilder").default;
 
 describe("viewBuilder", () => {
 

package/src/api/controllers/view/viewBuilder.ts

@@ -136,7 +136,7 @@ function parseEmitExpression(field: string, groupBy: string) {
  * filters: Array of filter objects containing predicates that are parsed into a JS expression
  * calculation: an optional calculation to be performed over the view data.
  */
-export = function ({
+export default function ({
   field,
   tableId,
   groupBy,

package/src/api/routes/analytics.ts

@@ -6,4 +6,4 @@ const router: Router = new Router()
 router.get("/api/bbtel", controller.isEnabled)
 router.post("/api/bbtel/ping", controller.ping)
 
-export = router
+export default router

package/src/api/routes/apikeys.ts

@@ -9,4 +9,4 @@ router
   .get("/api/keys", authorized(permissions.BUILDER), controller.fetch)
   .put("/api/keys/:key", authorized(permissions.BUILDER), controller.update)
 
-export = router
+export default router

package/src/api/routes/application.ts

@@ -54,4 +54,4 @@ router
     controller.destroy
   )
 
-export = router
+export default router

package/src/api/routes/auth.ts

@@ -5,4 +5,4 @@ const router: Router = new Router()
 
 router.get("/api/self", controller.fetchSelf)
 
-export = router
+export default router

package/src/api/routes/automation.ts

@@ -84,4 +84,4 @@ router
     controller.test
   )
 
-export = router
+export default router

package/src/api/routes/backup.ts

@@ -11,4 +11,4 @@ router.get(
   controller.exportAppDump
 )
 
-export = router
+export default router

package/src/api/routes/cloud.ts

@@ -15,4 +15,4 @@ router
   .post("/api/cloud/import", controller.importApps)
   .get("/api/cloud/import/complete", controller.hasBeenImported)
 
-export = router
+export default router

package/src/api/routes/component.ts

@@ -11,4 +11,4 @@ router.get(
   controller.fetchAppComponentDefinitions
 )
 
-export = router
+export default router

package/src/api/routes/datasource.ts

@@ -57,4 +57,4 @@ router
     datasourceController.destroy
   )
 
-export = router
+export default router

package/src/api/routes/deploy.ts

@@ -17,4 +17,4 @@ router
     controller.deploymentProgress
   )
 
-export = router
+export default router

package/src/api/routes/dev.ts

@@ -35,4 +35,4 @@ router
     controller.revert
   )
 
-export = router
+export default router

package/src/api/routes/integration.ts

@@ -13,4 +13,4 @@ router
     controller.find
   )
 
-export = router
+export default router

package/src/api/routes/layout.ts

@@ -13,4 +13,4 @@ router
     controller.destroy
   )
 
-export = router
+export default router

package/src/api/routes/metadata.ts

@@ -35,4 +35,4 @@ router
     controller.getMetadata
   )
 
-export = router
+export default router

package/src/api/routes/migrations.ts

@@ -11,4 +11,4 @@ router
     auth.internalApi,
     migrationsController.fetchDefinitions
   )
-export = router
+export default router

package/src/api/routes/permission.ts

@@ -38,4 +38,4 @@ router
     controller.removePermission
   )
 
-export = router
+export default router

package/src/api/routes/query.ts

@@ -58,4 +58,4 @@ router
     queryController.executeV2 as any
   )
 
-export = router
+export default router

package/src/api/routes/role.ts

@@ -21,4 +21,4 @@ router
     controller.destroy
   )
 
-export = router
+export default router

package/src/api/routes/routing.ts

@@ -11,4 +11,4 @@ router
   // gets the full structure, not just the correct screen ID for user role
   .get("/api/routing", authorized(permissions.BUILDER), controller.fetch)
 
-export = router
+export default router

package/src/api/routes/screen.ts

@@ -20,4 +20,4 @@ router
     controller.destroy
   )
 
-export = router
+export default router

package/src/api/routes/script.ts

@@ -7,4 +7,4 @@ const router: Router = new Router()
 
 router.post("/api/script", authorized(permissions.BUILDER), controller.save)
 
-export = router
+export default router

package/src/api/routes/static.ts

@@ -3,7 +3,7 @@ import * as controller from "../controllers/static"
 import { budibaseTempDir } from "../../utilities/budibaseDir"
 import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
-import * as env from "../../environment"
+import env from "../../environment"
 import { paramResource } from "../../middleware/resourceId"
 const { BUILDER, PermissionType, PermissionLevel } = permissions
 
@@ -62,4 +62,4 @@ router
     controller.getSignedUploadURL
   )
 
-export = router
+export default router

package/src/api/routes/table.ts

@@ -67,10 +67,7 @@ router
    * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field
    * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix
    * the rows in the table. This functionality is only available for internal tables.
-   * @apiParam (Body) {object} [dataImport] When creating an internal table it can be built from a CSV, by using the
-   * CSV validation endpoint. Send the CSV data to the validation endpoint, then put the results of that call
-   * into this property, along with the CSV and a table/rows will be built from it. This is not supported when updating
-   * or for external tables.
+   * @apiParam (Body) {object[]} [rows] When creating a table using a compatible data source, an array of objects to be imported into the new table can be provided.
    *
    * @apiParamExample {json} Example:
    * {
@@ -99,15 +96,7 @@ router
    *     "old": "columnName",
    *     "updated": "newColumnName",
    *   },
-   *   "dataImport": {
-   *     "csvString": "column\nvalue",
-   *     "primaryDisplay": "column",
-   *     "schema": {
-   *       "column": {
-   *         "type": "string"
-   *       }
-   *     }
-   *   }
+   *   "rows": []
    * }
    *
    * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and
@@ -121,30 +110,20 @@ router
     tableValidator(),
     tableController.save
   )
-  /**
-   * @api {post} /api/tables/csv/validate Validate a CSV for a table
-   * @apiName Validate a CSV for a table
-   * @apiGroup tables
-   * @apiPermission builder
-   * @apiDescription When creating a new table, or importing a CSV to an existing table the CSV must be validated and
-   * converted into a Budibase schema; this endpoint does this.
-   *
-   * @apiParam (Body) {string} csvString The CSV which is to be validated as a string.
-   * @apiParam (Body) {object} [schema] When a CSV has been validated it is possible to re-validate after changing the
-   * type of a field, by default everything will be strings as there is no way to infer types. The returned schema can
-   * be updated and then returned to the endpoint to re-validate and check if the type will work for the CSV, e.g.
-   * using a number instead of strings.
-   * @apiParam (Body) {string} [tableId] If importing data to an existing table this will pull the current table and
-   * remove any fields from the CSV schema which do not exist on the table/don't match the type of the table. When
-   * importing a CSV to an existing table only fields that are present on the table can be imported.
-   *
-   * @apiSuccess {object} schema The response body will contain a "schema" object that represents the schema found for
-   * the CSV - this will be in the same format used for table schema.s
-   */
   .post(
-    "/api/tables/csv/validate",
+    "/api/convert/csvToJson",
+    authorized(BUILDER),
+    tableController.csvToJson
+  )
+  .post(
+    "/api/tables/validateNewTableImport",
+    authorized(BUILDER),
+    tableController.validateNewTableImport
+  )
+  .post(
+    "/api/tables/validateExistingTableImport",
     authorized(BUILDER),
-    tableController.validateCSVSchema
+    tableController.validateExistingTableImport
   )
   /**
    * @api {post} /api/tables/:tableId/:revId Delete a table
@@ -177,9 +156,7 @@ router
    *
    * @apiParam {string} tableId The ID of the table which the data should be imported to.
    *
-   * @apiParam (Body) {object} dataImport This is the same as the structure used when creating an internal table with
-   * a CSV, it will have the "schema" returned from the CSV validation endpoint and the "csvString" which is to be
-   * turned into rows.
+   * @apiParam (Body) {object[]} rows An array of objects representing the rows to be imported, key-value pairs not matching the table schema will be ignored.
    *
    * @apiSuccess {string} message A message stating that the data was imported successfully.
    */
@@ -190,4 +167,4 @@ router
     tableController.bulkImport
   )
 
-export = router
+export default router

package/src/api/routes/templates.ts

@@ -13,4 +13,4 @@ router
     controller.downloadTemplate
   )
 
-export = router
+export default router

package/src/api/routes/tests/misc.spec.js

@@ -42,7 +42,7 @@ describe("run misc tests", () => {
   })
 
   describe("test table utilities", () => {
-    it("should be able to import a CSV", async () => {
+    it("should be able to import data", async () => {
       return config.doInContext(null, async () => {
         const table = await config.createTable({
           name: "table",
@@ -75,17 +75,11 @@ describe("run misc tests", () => {
             },
           },
         })
-        const dataImport = {
-          csvString: "a,b,c,d\n1,2,3,4",
-          schema: {},
-        }
-        for (let col of ["a", "b", "c", "d"]) {
-          dataImport.schema[col] = { type: "string" }
-        }
+
         await tableUtils.handleDataImport(
           { userId: "test" },
           table,
-          dataImport
+          [{ a: '1', b: '2', c: '3', d: '4'}]
         )
         const rows = await config.getRows()
         expect(rows[0].a).toEqual("1")
@@ -94,4 +88,4 @@ describe("run misc tests", () => {
       })
     })
   })
-})
+})

package/src/api/routes/tests/plugin.spec.ts

@@ -1,8 +1,3 @@
-let mockObjectStore = jest.fn().mockImplementation(() => {
-  return [{ name: "test.js" }]
-})
-
-let deleteFolder = jest.fn().mockImplementation()
 jest.mock("@budibase/backend-core", () => {
   const core = jest.requireActual("@budibase/backend-core")
   return {
@@ -10,15 +5,20 @@ jest.mock("@budibase/backend-core", () => {
     objectStore: {
       ...core.objectStore,
       upload: jest.fn(),
-      uploadDirectory: mockObjectStore,
-      deleteFolder: deleteFolder,
+      uploadDirectory: jest.fn().mockImplementation(() => {
+        return [{ name: "test.js" }]
+      }),
+      deleteFolder: jest.fn().mockImplementation(),
     },
   }
 })
 
-import { events } from "@budibase/backend-core"
+import { events, objectStore } from "@budibase/backend-core"
 import * as setup from "./utilities"
 
+const mockUploadDirectory = objectStore.uploadDirectory as jest.Mock
+const mockDeleteFolder = objectStore.deleteFolder as jest.Mock
+
 describe("/plugins", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -57,7 +57,7 @@ describe("/plugins", () => {
     })
 
     it("should not be able to create a plugin if there is an error", async () => {
-      mockObjectStore.mockImplementationOnce(() => {
+      mockUploadDirectory.mockImplementationOnce(() => {
         throw new Error()
       })
       let res = await createPlugin(400)
@@ -92,7 +92,7 @@ describe("/plugins", () => {
       expect(events.plugin.deleted).toHaveBeenCalledTimes(1)
     })
     it("should handle an error deleting a plugin", async () => {
-      deleteFolder.mockImplementationOnce(() => {
+      mockDeleteFolder.mockImplementationOnce(() => {
         throw new Error()
       })
 

package/src/api/routes/tests/table.spec.js

@@ -43,21 +43,18 @@ describe("/tables", () => {
       expect(events.table.created).toBeCalledWith(res.body)
     })
 
-    it("creates a table via data import CSV", async () => {
+    it("creates a table via data import", async () => {
       const table = basicTable()
-      table.dataImport = {
-        csvString: "\"name\",\"description\"\n\"test-name\",\"test-desc\"",
-      }
-      table.dataImport.schema = table.schema
+      table.rows = [{ name: 'test-name', description: 'test-desc' }]
 
       const res = await createTable(table)
 
       expect(events.table.created).toBeCalledTimes(1)
       expect(events.table.created).toBeCalledWith(res.body)
       expect(events.table.imported).toBeCalledTimes(1)
-      expect(events.table.imported).toBeCalledWith(res.body, "csv")
+      expect(events.table.imported).toBeCalledWith(res.body)
       expect(events.rows.imported).toBeCalledTimes(1)
-      expect(events.rows.imported).toBeCalledWith(res.body, "csv", 1)
+      expect(events.rows.imported).toBeCalledWith(res.body, 1)
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -155,11 +152,10 @@ describe("/tables", () => {
     it("imports rows successfully", async () => {
       const table = await config.createTable()
       const importRequest = {
-        dataImport: {
-          csvString: "\"name\",\"description\"\n\"test-name\",\"test-desc\"",
-          schema: table.schema
-        }
+        schema: table.schema,
+        rows: [{ name: 'test-name', description: 'test-desc' }]
       }
+
       jest.clearAllMocks()
 
       await request
@@ -171,7 +167,7 @@ describe("/tables", () => {
 
       expect(events.table.created).not.toHaveBeenCalled()
       expect(events.rows.imported).toBeCalledTimes(1)
-      expect(events.rows.imported).toBeCalledWith(table, "csv", 1)
+      expect(events.rows.imported).toBeCalledWith(table, 1)
     })
   })
 
@@ -206,24 +202,6 @@ describe("/tables", () => {
     })
   })
 
-  describe("validate csv", () => {
-    it("should be able to validate a CSV layout", async () => {
-      const res = await request
-        .post(`/api/tables/csv/validate`)
-        .send({
-          csvString: "a,b,c,d\n1,2,3,4"
-        })
-        .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
-        .expect(200)
-      expect(res.body.schema).toBeDefined()
-      expect(res.body.schema.a).toEqual({
-        type: "string",
-        success: true,
-      })
-    })
-  })
-
   describe("indexing", () => {
     it("should be able to create a table with indexes", async () => {
       await context.doInAppContext(appId, async () => {

package/src/api/routes/tests/user.spec.js

@@ -171,9 +171,28 @@ describe("/users", () => {
         .expect("Content-Type", /json/)
       expect(res.body.message).toEqual('User synced.')
     })
-  })
 
 
+    it("should sync the user when a previous user is specified", async () => {
+      const app1 = await config.createApp('App 1')
+      const app2 = await config.createApp('App 2')
 
+      let user = await config.createUser(
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        false,
+        true,
+        { [app1.appId]: 'ADMIN' })
+      let res = await request
+        .post(`/api/users/metadata/sync/${user._id}`)
+        .set(config.defaultHeaders())
+        .send({ previousUser: { ...user, roles: { ...user.roles, [app2.appId]: 'BASIC' } } })
+        .expect(200)
+        .expect("Content-Type", /json/)
 
+      expect(res.body.message).toEqual('User synced.')
+    })
+  })
 })

package/src/api/routes/tests/utilities/TestFunctions.ts

@@ -3,7 +3,7 @@ import * as appController from "../../../controllers/application"
 import { AppStatus } from "../../../../db/utils"
 import { roles, tenancy, context } from "@budibase/backend-core"
 import { TENANT_ID } from "../../../../tests/utilities/structures"
-import * as env from "../../../../environment"
+import env from "../../../../environment"
 
 class Request {
   appId: any