@budibase/server 2.2.12-alpha.4 → 2.2.12-alpha.41

package/dist/utilities/fileSystem/filesystem.js

@@ -17,7 +17,7 @@ const { budibaseTempDir } = require("../budibaseDir");
 const fs = require("fs");
 const { join } = require("path");
 const uuid = require("uuid/v4");
-const env = require("../../environment");
+const environment_1 = __importDefault(require("../../environment"));
 const tar_1 = __importDefault(require("tar"));
 exports.TOP_LEVEL_PATH = join(__dirname, "..", "..", "..");
 /**
@@ -38,7 +38,7 @@ const init = () => {
         }
     }
     const clientLibPath = join(budibaseTempDir(), "budibase-client.js");
-    if (env.isTest() && !fs.existsSync(clientLibPath)) {
+    if (environment_1.default.isTest() && !fs.existsSync(clientLibPath)) {
         fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath);
     }
 };
@@ -48,7 +48,7 @@ exports.init = init;
  * everything required to function is ready.
  */
 const checkDevelopmentEnvironment = () => {
-    if (!env.isDev() || env.isTest()) {
+    if (!environment_1.default.isDev() || environment_1.default.isTest()) {
         return;
     }
     if (!fs.existsSync(budibaseTempDir())) {

package/dist/utilities/schema.js

@@ -0,0 +1,108 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parse = exports.validate = exports.isRows = exports.isSchema = void 0;
+const constants_1 = require("../constants");
+const PARSERS = {
+    [constants_1.FieldTypes.NUMBER]: (attribute) => {
+        if (!attribute) {
+            return attribute;
+        }
+        return Number(attribute);
+    },
+    [constants_1.FieldTypes.DATETIME]: (attribute) => {
+        if (!attribute) {
+            return attribute;
+        }
+        return new Date(attribute).toISOString();
+    },
+};
+function isSchema(schema) {
+    return (typeof schema === "object" &&
+        Object.values(schema).every(rawColumn => {
+            const column = rawColumn;
+            return (column !== null &&
+                typeof column === "object" &&
+                typeof column.type === "string" &&
+                Object.values(constants_1.FieldTypes).includes(column.type));
+        }));
+}
+exports.isSchema = isSchema;
+function isRows(rows) {
+    return Array.isArray(rows) && rows.every(row => typeof row === "object");
+}
+exports.isRows = isRows;
+function validate(rows, schema) {
+    const results = {
+        schemaValidation: {},
+        allValid: false,
+        invalidColumns: [],
+    };
+    rows.forEach(row => {
+        Object.entries(row).forEach(([columnName, columnData]) => {
+            var _a, _b;
+            const columnType = (_a = schema[columnName]) === null || _a === void 0 ? void 0 : _a.type;
+            const isAutoColumn = (_b = schema[columnName]) === null || _b === void 0 ? void 0 : _b.autocolumn;
+            // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
+            if (typeof columnType !== "string") {
+                results.invalidColumns.push(columnName);
+            }
+            else if (
+            // If there's no data for this field don't bother with further checks
+            // If the field is already marked as invalid there's no need for further checks
+            results.schemaValidation[columnName] === false ||
+                columnData == null ||
+                isAutoColumn) {
+                return;
+            }
+            else if (columnType === constants_1.FieldTypes.NUMBER &&
+                isNaN(Number(columnData))) {
+                // If provided must be a valid number
+                results.schemaValidation[columnName] = false;
+            }
+            else if (
+            // If provided must be a valid date
+            columnType === constants_1.FieldTypes.DATETIME &&
+                isNaN(new Date(columnData).getTime())) {
+                results.schemaValidation[columnName] = false;
+            }
+            else {
+                results.schemaValidation[columnName] = true;
+            }
+        });
+    });
+    results.allValid =
+        Object.values(results.schemaValidation).length > 0 &&
+            Object.values(results.schemaValidation).every(column => column);
+    // Select unique values
+    results.invalidColumns = [...new Set(results.invalidColumns)];
+    return results;
+}
+exports.validate = validate;
+function parse(rows, schema) {
+    return rows.map(row => {
+        const parsedRow = {};
+        Object.entries(row).forEach(([columnName, columnData]) => {
+            var _a;
+            if (!(columnName in schema) || ((_a = schema[columnName]) === null || _a === void 0 ? void 0 : _a.autocolumn)) {
+                // Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case
+                return;
+            }
+            const columnType = schema[columnName].type;
+            if (columnType === constants_1.FieldTypes.NUMBER) {
+                // If provided must be a valid number
+                parsedRow[columnName] = columnData ? Number(columnData) : columnData;
+            }
+            else if (columnType === constants_1.FieldTypes.DATETIME) {
+                // If provided must be a valid date
+                parsedRow[columnName] = columnData
+                    ? new Date(columnData).toISOString()
+                    : columnData;
+            }
+            else {
+                parsedRow[columnName] = columnData;
+            }
+        });
+        return parsedRow;
+    });
+}
+exports.parse = parse;

package/dist/utilities/scriptRunner.js

@@ -2,6 +2,7 @@
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
+Object.defineProperty(exports, "__esModule", { value: true });
 const node_fetch_1 = __importDefault(require("node-fetch"));
 const vm2_1 = require("vm2");
 const JS_TIMEOUT_MS = 1000;
@@ -22,4 +23,4 @@ class ScriptRunner {
         return this.results.out;
     }
 }
-module.exports = ScriptRunner;
+exports.default = ScriptRunner;

package/dist/utilities/statusCodes.js

@@ -1,5 +1,6 @@
 "use strict";
-module.exports = {
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = {
     OK: 200,
     UNAUTHORIZED: 401,
     FORBIDDEN: 403,

package/dist/utilities/workerRequests.js

@@ -41,6 +41,8 @@ function request(ctx, request) {
     if (ctx && ctx.headers) {
         request.headers = ctx.headers;
     }
+    // add x-budibase-correlation-id header
+    backend_core_1.logging.correlation.setHeader(request.headers);
     return request;
 }
 exports.request = request;

package/dist/watch.js

@@ -1,27 +1,4 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -37,13 +14,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.watch = void 0;
 const path_1 = __importDefault(require("path"));
-const env = __importStar(require("./environment"));
+const environment_1 = __importDefault(require("./environment"));
 const chokidar_1 = __importDefault(require("chokidar"));
 const fs_1 = __importDefault(require("fs"));
 const backend_core_1 = require("@budibase/backend-core");
 const plugin_1 = require("./api/controllers/plugin");
 function watch() {
-    const watchPath = path_1.default.join(env.PLUGINS_DIR, "./**/*.tar.gz");
+    const watchPath = path_1.default.join(environment_1.default.PLUGINS_DIR, "./**/*.tar.gz");
     chokidar_1.default
         .watch(watchPath, {
         ignored: "**/node_modules",

package/jest.config.ts

@@ -2,15 +2,18 @@ import { Config } from "@jest/types"
 import * as fs from "fs"
 
 const config: Config.InitialOptions = {
-  preset: "ts-jest",
   testEnvironment: "node",
-  setupFiles: ["./src/tests/jestSetup.ts"],
+  setupFiles: ["./src/tests/jestEnv.ts"],
+  setupFilesAfterEnv: ["./src/tests/jestSetup.ts"],
   collectCoverageFrom: [
     "src/**/*.{js,ts}",
     // The use of coverage with couchdb view functions breaks tests
     "!src/db/views/staticViews.*",
   ],
   coverageReporters: ["lcov", "json", "clover"],
+  transform: {
+    "^.+\\.ts?$": "@swc/jest",
+  },
 }
 
 if (!process.env.CI) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.2.12-alpha.4",
+  "version": "2.2.12-alpha.41",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.2.12-alpha.4",
-    "@budibase/client": "2.2.12-alpha.4",
-    "@budibase/pro": "2.2.12-alpha.3",
-    "@budibase/string-templates": "2.2.12-alpha.4",
-    "@budibase/types": "2.2.12-alpha.4",
+    "@budibase/backend-core": "2.2.12-alpha.41",
+    "@budibase/client": "2.2.12-alpha.41",
+    "@budibase/pro": "2.2.12-alpha.40",
+    "@budibase/string-templates": "2.2.12-alpha.41",
+    "@budibase/types": "2.2.12-alpha.41",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -77,7 +77,7 @@
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
-    "knex": "0.95.15",
+    "knex": "2.4.0",
     "koa": "2.13.4",
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
@@ -123,6 +123,8 @@
     "@babel/preset-env": "7.16.11",
     "@budibase/standard-components": "^0.9.139",
     "@jest/test-sequencer": "24.9.0",
+    "@swc/core": "^1.3.25",
+    "@swc/jest": "^0.2.24",
     "@types/apidoc": "0.50.0",
     "@types/bson": "4.2.0",
     "@types/global-agent": "2.1.1",
@@ -167,5 +169,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "90b3aaddb3f131078b05acf21651c8535b9c7d03"
+  "gitHead": "b5da4944f2a33a3a70df34852833b83f536cde92"
 }

package/src/api/controllers/query/index.ts

@@ -4,12 +4,12 @@ import { Thread, ThreadType } from "../../../threads"
 import { save as saveDatasource } from "../datasource"
 import { RestImporter } from "./import"
 import { invalidateDynamicVariables } from "../../../threads/utils"
-import { QUERY_THREAD_TIMEOUT } from "../../../environment"
+import env from "../../../environment"
 import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
 
 const Runner = new Thread(ThreadType.QUERY, {
-  timeoutMs: QUERY_THREAD_TIMEOUT || 10000,
+  timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
 })
 
 // simple function to append "readable" to all read queries

package/src/api/controllers/row/ExternalRequest.ts

@@ -681,6 +681,12 @@ export class ExternalRequest {
       config,
       table
     )
+    //if the sort column is a formula, remove it
+    for (let sortColumn of Object.keys(sort || {})) {
+      if (table.schema[sortColumn]?.type === "formula") {
+        delete sort?.[sortColumn]
+      }
+    }
     filters = buildFilters(id, filters || {}, table)
     const relationships = this.buildRelationships(table)
     // clean up row on ingress using schema

package/src/api/controllers/row/internal.ts

@@ -27,7 +27,7 @@ import {
 import { cloneDeep } from "lodash/fp"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
-import * as exporters from "../view/exporters"
+import { csv, json, jsonWithSchema, Format, isFormat } from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
   Ctx,
@@ -412,14 +412,15 @@ export async function exportRows(ctx: Ctx) {
     rows = result
   }
 
-  let headers = Object.keys(rows[0])
-  // @ts-ignore
-  const exporter = exporters[format]
-  const filename = `export.${format}`
-
-  // send down the file
-  ctx.attachment(filename)
-  return apiFileReturn(exporter(headers, rows))
+  if (format === Format.CSV) {
+    ctx.attachment("export.csv")
+    return apiFileReturn(csv(Object.keys(rows[0]), rows))
+  } else if (format === Format.JSON) {
+    ctx.attachment("export.json")
+    return apiFileReturn(json(rows))
+  } else {
+    throw "Format not recognised"
+  }
 }
 
 export async function fetchEnrichedRow(ctx: Ctx) {

package/src/api/controllers/static/index.ts

@@ -10,7 +10,7 @@ const {
   NODE_MODULES_PATH,
   TOP_LEVEL_PATH,
 } = require("../../../utilities/fileSystem")
-const env = require("../../../environment")
+import env from "../../../environment"
 const { DocumentType } = require("../../../db/utils")
 const { context, objectStore, utils } = require("@budibase/backend-core")
 const AWS = require("aws-sdk")

package/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -2,7 +2,6 @@
   export let title = ""
   export let favicon = ""
   export let metaImage = ""
-  export let url = ""
 
   export let clientLibPath
   export let usedPlugins

package/src/api/controllers/table/external.ts

@@ -10,9 +10,9 @@ import {
 } from "./utils"
 import { FieldTypes, RelationshipTypes } from "../../../constants"
 import { makeExternalQuery } from "../../../integrations/base/query"
-import * as csvParser from "../../../utilities/csvParser"
 import { handleRequest } from "../row/external"
 import { events, context } from "@budibase/backend-core"
+import { parse, isRows, isSchema } from "../../../utilities/schema"
 import {
   Datasource,
   Table,
@@ -197,7 +197,7 @@ export async function save(ctx: BBContext) {
   const table: TableRequest = ctx.request.body
   const renamed = table?._rename
   // can't do this right now
-  delete table.dataImport
+  delete table.rows
   const datasourceId = getDatasourceId(ctx.request.body)!
   // table doesn't exist already, note that it is created
   if (!table._id) {
@@ -338,17 +338,17 @@ export async function destroy(ctx: BBContext) {
 
 export async function bulkImport(ctx: BBContext) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
-  const { dataImport } = ctx.request.body
-  if (!dataImport || !dataImport.schema || !dataImport.csvString) {
+  const { rows }: { rows: unknown } = ctx.request.body
+  const schema: unknown = table.schema
+
+  if (!rows || !isRows(rows) || !isSchema(schema)) {
     ctx.throw(400, "Provided data import information is invalid.")
   }
-  const rows = await csvParser.transform({
-    ...dataImport,
-    existingTable: table,
-  })
+
+  const parsedRows = await parse(rows, schema)
   await handleRequest(Operation.BULK_CREATE, table._id!, {
-    rows,
+    rows: parsedRows,
   })
-  await events.rows.imported(table, "csv", rows.length)
+  await events.rows.imported(table, parsedRows.length)
   return table
 }

package/src/api/controllers/table/index.ts

@@ -1,11 +1,16 @@
 import * as internal from "./internal"
 import * as external from "./external"
-import * as csvParser from "../../../utilities/csvParser"
+import {
+  validate as validateSchema,
+  isSchema,
+  isRows,
+} from "../../../utilities/schema"
 import { isExternalTable, isSQL } from "../../../integrations/utils"
 import { getDatasourceParams } from "../../../db/utils"
 import { context, events } from "@budibase/backend-core"
 import { Table, BBContext } from "@budibase/types"
 import sdk from "../../../sdk"
+import csv from "csvtojson"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
@@ -56,16 +61,16 @@ export async function find(ctx: BBContext) {
 export async function save(ctx: BBContext) {
   const appId = ctx.appId
   const table = ctx.request.body
-  const importFormat =
-    table.dataImport && table.dataImport.csvString ? "csv" : undefined
+  const isImport = table.rows
+
   const savedTable = await pickApi({ table }).save(ctx)
   if (!table._id) {
     await events.table.created(savedTable)
   } else {
     await events.table.updated(savedTable)
   }
-  if (importFormat) {
-    await events.table.imported(savedTable, importFormat)
+  if (isImport) {
+    await events.table.imported(savedTable)
   }
   ctx.status = 200
   ctx.message = `Table ${table.name} saved successfully.`
@@ -96,19 +101,43 @@ export async function bulkImport(ctx: BBContext) {
   ctx.body = { message: `Bulk rows created.` }
 }
 
-export async function validateCSVSchema(ctx: BBContext) {
-  // tableId being specified means its an import to an existing table
-  const { csvString, schema = {}, tableId } = ctx.request.body
-  let existingTable
+export async function csvToJson(ctx: BBContext) {
+  const { csvString } = ctx.request.body
+
+  const result = await csv().fromString(csvString)
+
+  ctx.status = 200
+  ctx.body = result
+}
+
+export async function validateNewTableImport(ctx: BBContext) {
+  const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body
+
+  if (isRows(rows) && isSchema(schema)) {
+    ctx.status = 200
+    ctx.body = validateSchema(rows, schema)
+  } else {
+    ctx.status = 422
+  }
+}
+
+export async function validateExistingTableImport(ctx: BBContext) {
+  const { rows, tableId }: { rows: unknown; tableId: unknown } =
+    ctx.request.body
+
+  let schema = null
   if (tableId) {
-    existingTable = await sdk.tables.getTable(tableId)
+    const table = await sdk.tables.getTable(tableId)
+    schema = table.schema
+  } else {
+    ctx.status = 422
+    return
   }
-  let result: Record<string, any> | undefined = await csvParser.parse(
-    csvString,
-    schema
-  )
-  if (existingTable) {
-    result = csvParser.updateSchema({ schema: result, existingTable })
+
+  if (tableId && isRows(rows) && isSchema(schema)) {
+    ctx.status = 200
+    ctx.body = validateSchema(rows, schema)
+  } else {
+    ctx.status = 422
   }
-  ctx.body = { schema: result }
 }

package/src/api/controllers/table/internal.ts

@@ -3,7 +3,7 @@ import { getRowParams, generateTableID } from "../../../db/utils"
 import { FieldTypes } from "../../../constants"
 import { TableSaveFunctions, hasTypeChanged, handleDataImport } from "./utils"
 import { context } from "@budibase/backend-core"
-import { isTest } from "../../../environment"
+import env from "../../../environment"
 import {
   cleanupAttachments,
   fixAutoColumnSubType,
@@ -13,7 +13,6 @@ import { Table } from "@budibase/types"
 import { quotas } from "@budibase/pro"
 import { isEqual } from "lodash"
 import { cloneDeep } from "lodash/fp"
-import env from "../../../environment"
 import sdk from "../../../sdk"
 
 function checkAutoColumns(table: Table, oldTable: Table) {
@@ -36,7 +35,7 @@ function checkAutoColumns(table: Table, oldTable: Table) {
 
 export async function save(ctx: any) {
   const db = context.getAppDB()
-  const { dataImport, ...rest } = ctx.request.body
+  const { rows, ...rest } = ctx.request.body
   let tableToSave = {
     type: "table",
     _id: generateTableID(),
@@ -62,7 +61,7 @@ export async function save(ctx: any) {
   const tableSaveFunctions = new TableSaveFunctions({
     user: ctx.user,
     oldTable,
-    dataImport,
+    importRows: rows,
   })
   tableToSave = await tableSaveFunctions.before(tableToSave)
 
@@ -164,7 +163,7 @@ export async function destroy(ctx: any) {
   await db.remove(tableToDelete._id, tableToDelete._rev)
 
   // remove table search index
-  if (!isTest() || env.COUCH_DB_URL) {
+  if (!env.isTest() || env.COUCH_DB_URL) {
     const currentIndexes = await db.getIndexes()
     const existingIndex = currentIndexes.indexes.find(
       (existing: any) => existing.name === `search:${ctx.params.tableId}`
@@ -186,7 +185,7 @@ export async function destroy(ctx: any) {
 
 export async function bulkImport(ctx: any) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
-  const { dataImport } = ctx.request.body
-  await handleDataImport(ctx.user, table, dataImport)
+  const { rows } = ctx.request.body
+  await handleDataImport(ctx.user, table, rows)
   return table
 }

package/src/api/controllers/table/utils.ts

@@ -1,4 +1,4 @@
-import { transform } from "../../../utilities/csvParser"
+import { parse, isSchema, isRows } from "../../../utilities/schema"
 import { getRowParams, generateRowID, InternalTables } from "../../../db/utils"
 import { isEqual } from "lodash"
 import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
@@ -128,24 +128,23 @@ export function importToRows(data: any, table: any, user: any = {}) {
   return finalData
 }
 
-export async function handleDataImport(user: any, table: any, dataImport: any) {
-  if (!dataImport || !dataImport.csvString) {
+export async function handleDataImport(user: any, table: any, rows: any) {
+  const schema: unknown = table.schema
+
+  if (!rows || !isRows(rows) || !isSchema(schema)) {
     return table
   }
 
   const db = context.getAppDB()
-  // Populate the table with rows imported from CSV in a bulk update
-  const data = await transform({
-    ...dataImport,
-    existingTable: table,
-  })
+  const data = parse(rows, schema)
 
   let finalData: any = importToRows(data, table, user)
 
   await quotas.addRows(finalData.length, () => db.bulkDocs(finalData), {
     tableId: table._id,
   })
-  await events.rows.imported(table, "csv", finalData.length)
+
+  await events.rows.imported(table, finalData.length)
   return table
 }
 
@@ -210,14 +209,14 @@ class TableSaveFunctions {
   db: any
   user: any
   oldTable: any
-  dataImport: any
+  importRows: any
   rows: any
 
-  constructor({ user, oldTable, dataImport }: any) {
+  constructor({ user, oldTable, importRows }: any) {
     this.db = context.getAppDB()
     this.user = user
     this.oldTable = oldTable
-    this.dataImport = dataImport
+    this.importRows = importRows
     // any rows that need updated
     this.rows = []
   }
@@ -241,7 +240,7 @@ class TableSaveFunctions {
   // after saving
   async after(table: any) {
     table = await handleSearchIndexes(table)
-    table = await handleDataImport(this.user, table, this.dataImport)
+    table = await handleDataImport(this.user, table, this.importRows)
     return table
   }