@budibase/server 1.0.35 → 1.0.39

package/dist/utilities/budibaseDir.js

@@ -2,7 +2,7 @@
 const { join } = require("./centralPath");
 const { homedir } = require("os");
 const env = require("../environment");
-const { budibaseTempDir } = require("@budibase/auth").objectStore;
+const { budibaseTempDir } = require("@budibase/backend-core/objectStore");
 module.exports.budibaseAppsDir = function () {
     return env.BUDIBASE_DIR || join(homedir(), ".budibase");
 };

package/dist/utilities/fileSystem/utilities.js

@@ -1,5 +1,5 @@
 "use strict";
-const { ObjectStore, makeSureBucketExists, upload, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, } = require("@budibase/auth").objectStore;
+const { ObjectStore, makeSureBucketExists, upload, deleteFiles, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, } = require("@budibase/backend-core/objectStore");
 /***********************************
  *              NOTE               *
  * This file purely exists so that *
@@ -17,3 +17,4 @@ exports.retrieveToTmp = retrieveToTmp;
 exports.deleteFolder = deleteFolder;
 exports.uploadDirectory = uploadDirectory;
 exports.downloadTarball = downloadTarball;
+exports.deleteFiles = deleteFiles;

package/dist/utilities/global.js

@@ -9,11 +9,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 const { getMultiIDParams, getGlobalIDFromUserMetadataID, } = require("../db/utils");
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles");
-const { getDeployedAppID } = require("@budibase/auth/db");
-const { getGlobalUserParams } = require("@budibase/auth/db");
-const { user: userCache } = require("@budibase/auth/cache");
-const { getGlobalDB, isUserInAppTenant } = require("@budibase/auth/tenancy");
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles");
+const { getDeployedAppID } = require("@budibase/backend-core/db");
+const { getGlobalUserParams } = require("@budibase/backend-core/db");
+const { user: userCache } = require("@budibase/backend-core/cache");
+const { getGlobalDB, isUserInAppTenant, } = require("@budibase/backend-core/tenancy");
 const env = require("../environment");
 exports.updateAppRole = (appId, user) => {
     if (!user || !user.roles) {

package/dist/utilities/index.js

@@ -10,7 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 const env = require("../environment");
 const { OBJ_STORE_DIRECTORY } = require("../constants");
-const { sanitizeKey } = require("@budibase/auth/src/objectStore");
+const { sanitizeKey } = require("@budibase/backend-core/objectStore");
 const CouchDB = require("../db");
 const { generateMetadataID } = require("../db/utils");
 const Readable = require("stream").Readable;
@@ -142,3 +142,13 @@ exports.doesDatabaseExist = (dbName) => __awaiter(void 0, void 0, void 0, functi
         return false;
     }
 });
+exports.formatBytes = bytes => {
+    const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
+    const byteIncrements = 1024;
+    let unit = 0;
+    let size = parseInt(bytes, 10) || 0;
+    while (size >= byteIncrements && ++unit) {
+        size /= byteIncrements;
+    }
+    return `${size.toFixed(size < 10 && unit > 0 ? 1 : 0)}${units[unit]}`;
+};

package/dist/utilities/redis.js

@@ -8,7 +8,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const { Client, utils } = require("@budibase/auth/redis");
+const { Client, utils } = require("@budibase/backend-core/redis");
 const { getGlobalIDFromUserMetadataID } = require("../db/utils");
 const APP_DEV_LOCK_SECONDS = 600;
 const AUTOMATION_TEST_FLAG_SECONDS = 60;

package/dist/utilities/rowProcessor/index.js

@@ -13,6 +13,10 @@ const { cloneDeep } = require("lodash/fp");
 const { FieldTypes, AutoFieldSubTypes } = require("../../constants");
 const { attachmentsRelativeURL } = require("../index");
 const { processFormulas } = require("./utils");
+const { deleteFiles } = require("../../utilities/fileSystem/utilities");
+const { ObjectStoreBuckets } = require("../../constants");
+const { isProdAppID, getDeployedAppID, dbExists, } = require("@budibase/backend-core/db");
+const CouchDB = require("../../db");
 const BASE_AUTO_ID = 1;
 /**
  * A map of how we convert various properties in rows to each other based on the row type.
@@ -89,7 +93,36 @@ const TYPE_TRANSFORM_MAP = {
     [FieldTypes.AUTO]: {
         parse: () => undefined,
     },
+    [FieldTypes.JSON]: {
+        parse: input => {
+            try {
+                if (input === "") {
+                    return undefined;
+                }
+                return JSON.parse(input);
+            }
+            catch (err) {
+                return input;
+            }
+        },
+    },
 };
+/**
+ * Given the old state of the row and the new one after an update, this will
+ * find the keys that have been removed in the updated row.
+ */
+function getRemovedAttachmentKeys(oldRow, row, attachmentKey) {
+    if (!oldRow[attachmentKey]) {
+        return [];
+    }
+    const oldKeys = oldRow[attachmentKey].map(attachment => attachment.key);
+    // no attachments in new row, all removed
+    if (!row[attachmentKey]) {
+        return oldKeys;
+    }
+    const newKeys = row[attachmentKey].map(attachment => attachment.key);
+    return oldKeys.filter(key => newKeys.indexOf(key) === -1);
+}
 /**
  * This will update any auto columns that are found on the row/table with the correct information based on
  * time now and the current logged in user making the request.
@@ -240,3 +273,46 @@ exports.outputProcessing = (ctx, table, rows, opts = { squash: true }) => __awai
     }
     return wasArray ? enriched : enriched[0];
 });
+/**
+ * Clean up any attachments that were attached to a row.
+ * @param {string} appId The ID of the app from which a row is being deleted.
+ * @param {object} table The table from which a row is being removed.
+ * @param {any} row optional - the row being removed.
+ * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
+ * @param {any} oldRow optional - if updating a row this will determine the difference.
+ * @return {Promise<void>} When all attachments have been removed this will return.
+ */
+exports.cleanupAttachments = (appId, table, { row, rows, oldRow }) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!isProdAppID(appId)) {
+        const prodAppId = getDeployedAppID(appId);
+        // if prod exists, then don't allow deleting
+        const exists = yield dbExists(CouchDB, prodAppId);
+        if (exists) {
+            return;
+        }
+    }
+    let files = [];
+    function addFiles(row, key) {
+        if (row[key]) {
+            files = files.concat(row[key].map(attachment => attachment.key));
+        }
+    }
+    for (let [key, schema] of Object.entries(table.schema)) {
+        if (schema.type !== FieldTypes.ATTACHMENT) {
+            continue;
+        }
+        // if updating, need to manage the differences
+        if (oldRow && row) {
+            files = files.concat(getRemovedAttachmentKeys(oldRow, row, key));
+        }
+        else if (row) {
+            addFiles(row, key);
+        }
+        else if (rows) {
+            rows.forEach(row => addFiles(row, key));
+        }
+    }
+    if (files.length > 0) {
+        return deleteFiles(ObjectStoreBuckets.APPS, files);
+    }
+});

package/dist/utilities/security.js

@@ -1,6 +1,6 @@
 "use strict";
-const { PermissionLevels, PermissionTypes, getBuiltinPermissionByID, isPermissionLevelHigherThanRead, } = require("@budibase/auth/permissions");
-const { lowerBuiltinRoleID, getBuiltinRoles } = require("@budibase/auth/roles");
+const { PermissionLevels, PermissionTypes, getBuiltinPermissionByID, isPermissionLevelHigherThanRead, } = require("@budibase/backend-core/permissions");
+const { lowerBuiltinRoleID, getBuiltinRoles, } = require("@budibase/backend-core/roles");
 const { DocumentTypes } = require("../db/utils");
 const CURRENTLY_SUPPORTED_LEVELS = [
     PermissionLevels.WRITE,

package/dist/utilities/usageQuota.js

@@ -9,8 +9,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 const env = require("../environment");
-const { getGlobalDB } = require("@budibase/auth/tenancy");
-const { StaticDatabases, generateNewUsageQuotaDoc, } = require("@budibase/auth/db");
+const { getGlobalDB } = require("@budibase/backend-core/tenancy");
+const { StaticDatabases, generateNewUsageQuotaDoc, } = require("@budibase/backend-core/db");
 function getNewQuotaReset() {
     return Date.now() + 2592000000;
 }

package/dist/utilities/workerRequests.js

@@ -11,10 +11,10 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 const fetch = require("node-fetch");
 const env = require("../environment");
 const { checkSlashesInUrl } = require("./index");
-const { getDeployedAppID } = require("@budibase/auth/db");
+const { getDeployedAppID } = require("@budibase/backend-core/db");
 const { updateAppRole } = require("./global");
-const { Headers } = require("@budibase/auth/constants");
-const { getTenantId, isTenantIdSet } = require("@budibase/auth/tenancy");
+const { Headers } = require("@budibase/backend-core/constants");
+const { getTenantId, isTenantIdSet } = require("@budibase/backend-core/tenancy");
 function request(ctx, request) {
     if (!request.headers) {
         request.headers = {};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.35",
+  "version": "1.0.39",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -69,9 +69,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/auth": "^1.0.35",
-    "@budibase/client": "^1.0.35",
-    "@budibase/string-templates": "^1.0.35",
+    "@apidevtools/swagger-parser": "^10.0.3",
+    "@budibase/backend-core": "^1.0.39",
+    "@budibase/client": "^1.0.39",
+    "@budibase/string-templates": "^1.0.39",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -85,12 +86,15 @@
     "bull": "^3.22.4",
     "chmodr": "1.2.0",
     "csvtojson": "2.0.10",
+    "curlconverter": "^3.21.0",
     "dotenv": "8.2.0",
     "download": "8.0.0",
     "fix-path": "3.0.0",
+    "form-data": "^4.0.0",
     "fs-extra": "8.1.0",
     "jimp": "0.16.1",
     "joi": "17.2.1",
+    "js-yaml": "^4.1.0",
     "jsonschema": "1.4.0",
     "knex": "^0.95.6",
     "koa": "2.7.0",
@@ -118,11 +122,13 @@
     "pouchdb-replication-stream": "1.2.9",
     "server-destroy": "1.0.1",
     "svelte": "^3.38.2",
+    "swagger-parser": "^10.0.3",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
     "vm2": "^3.9.3",
     "worker-farm": "^1.7.0",
+    "xml2js": "^0.4.23",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -146,6 +152,7 @@
     "eslint": "^6.8.0",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
+    "openapi-types": "^9.3.1",
     "path-to-regexp": "^6.2.0",
     "prettier": "^2.3.1",
     "rimraf": "^3.0.2",
@@ -158,5 +165,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "157877a60444104d17efedf937bc7d48fc3006c5"
+  "gitHead": "c1451c6e40a7c3c318881f3f60e203d724a9f096"
 }

package/scripts/replicateApp.js

@@ -7,7 +7,7 @@
 
 const CouchDB = require("../src/db")
 const { DocumentTypes } = require("../src/db/utils")
-const { getAllDbs } = require("@budibase/auth/db")
+const { getAllDbs } = require("@budibase/backend-core/db")
 
 const appName = process.argv[2].toLowerCase()
 const remoteUrl = process.argv[3]

package/src/api/controllers/apikeys.js

@@ -1,5 +1,5 @@
-const { StaticDatabases } = require("@budibase/auth/db")
-const { getGlobalDB } = require("@budibase/auth/tenancy")
+const { StaticDatabases } = require("@budibase/backend-core/db")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 
 const KEYS_DOC = StaticDatabases.GLOBAL.docs.apiKeys
 

package/src/api/controllers/application.js

@@ -19,7 +19,10 @@ const {
   DocumentTypes,
   AppStatus,
 } = require("../../db/utils")
-const { BUILTIN_ROLE_IDS, AccessController } = require("@budibase/auth/roles")
+const {
+  BUILTIN_ROLE_IDS,
+  AccessController,
+} = require("@budibase/backend-core/roles")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
 const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
@@ -28,7 +31,7 @@ const {
   isDevAppID,
   getDeployedAppID,
   Replication,
-} = require("@budibase/auth/db")
+} = require("@budibase/backend-core/db")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
 const {
   getDeployedApps,
@@ -41,9 +44,9 @@ const {
   backupClientLibrary,
   revertClientLibrary,
 } = require("../../utilities/fileSystem/clientLibrary")
-const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy")
+const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
 const { syncGlobalUsers } = require("./user")
-const { app: appCache } = require("@budibase/auth/cache")
+const { app: appCache } = require("@budibase/backend-core/cache")
 const { cleanupAutomations } = require("../../automations/utils")
 
 const URL_REGEX_SLASH = /\/|\\/g

package/src/api/controllers/auth.js

@@ -2,7 +2,7 @@ const CouchDB = require("../../db")
 const { outputProcessing } = require("../../utilities/rowProcessor")
 const { InternalTables } = require("../../db/utils")
 const { getFullUser } = require("../../utilities/users")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 
 exports.fetchSelf = async ctx => {
   const appId = ctx.appId

package/src/api/controllers/automation.js

@@ -5,11 +5,15 @@ const { getAutomationParams, generateAutomationID } = require("../../db/utils")
 const {
   checkForWebhooks,
   updateTestHistory,
+  removeDeprecated,
 } = require("../../automations/utils")
 const { deleteEntityMetadata } = require("../../utilities")
 const { MetadataTypes } = require("../../constants")
 const { setTestFlag, clearTestFlag } = require("../../utilities/redis")
 
+const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
+const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
+
 /*************************
  *                       *
  *   BUILDER FUNCTIONS   *
@@ -155,17 +159,17 @@ exports.destroy = async function (ctx) {
 }
 
 exports.getActionList = async function (ctx) {
-  ctx.body = actions.ACTION_DEFINITIONS
+  ctx.body = ACTION_DEFS
 }
 
 exports.getTriggerList = async function (ctx) {
-  ctx.body = triggers.TRIGGER_DEFINITIONS
+  ctx.body = TRIGGER_DEFS
 }
 
 module.exports.getDefinitionList = async function (ctx) {
   ctx.body = {
-    trigger: triggers.TRIGGER_DEFINITIONS,
-    action: actions.ACTION_DEFINITIONS,
+    trigger: TRIGGER_DEFS,
+    action: ACTION_DEFS,
   }
 }
 

package/src/api/controllers/cloud.js

@@ -1,5 +1,5 @@
 const env = require("../../environment")
-const { getAllApps } = require("@budibase/auth/db")
+const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../../db")
 const {
   exportDB,
@@ -7,7 +7,10 @@ const {
   readFileSync,
 } = require("../../utilities/fileSystem")
 const { stringToReadStream } = require("../../utilities")
-const { getGlobalDBName, getGlobalDB } = require("@budibase/auth/tenancy")
+const {
+  getGlobalDBName,
+  getGlobalDB,
+} = require("@budibase/backend-core/tenancy")
 const { create } = require("./application")
 const { getDocParams, DocumentTypes, isDevAppID } = require("../../db/utils")
 

package/src/api/controllers/datasource.js

@@ -10,6 +10,7 @@ const {
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
 const { integrations } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
+const { invalidateDynamicVariables } = require("../../threads/utils")
 
 exports.fetch = async function (ctx) {
   const database = new CouchDB(ctx.appId)
@@ -57,10 +58,43 @@ exports.buildSchemaFromDb = async function (ctx) {
   ctx.body = response
 }
 
+/**
+ * Check for variables that have been updated or removed and invalidate them.
+ */
+const invalidateVariables = async (existingDatasource, updatedDatasource) => {
+  const existingVariables = existingDatasource.config.dynamicVariables
+  const updatedVariables = updatedDatasource.config.dynamicVariables
+  const toInvalidate = []
+
+  if (!existingVariables) {
+    return
+  }
+
+  if (!updatedVariables) {
+    // invalidate all
+    toInvalidate.push(...existingVariables)
+  } else {
+    // invaldate changed / removed
+    existingVariables.forEach(existing => {
+      const unchanged = updatedVariables.find(
+        updated =>
+          existing.name === updated.name &&
+          existing.queryId === updated.queryId &&
+          existing.value === updated.value
+      )
+      if (!unchanged) {
+        toInvalidate.push(existing)
+      }
+    })
+  }
+  await invalidateDynamicVariables(toInvalidate)
+}
+
 exports.update = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const datasourceId = ctx.params.datasourceId
   let datasource = await db.get(datasourceId)
+  await invalidateVariables(datasource, ctx.request.body)
   datasource = { ...datasource, ...ctx.request.body }
 
   const response = await db.put(datasource)

package/src/api/controllers/deploy/index.js

@@ -1,12 +1,12 @@
 const CouchDB = require("../../../db")
 const Deployment = require("./Deployment")
-const { Replication, getDeployedAppID } = require("@budibase/auth/db")
+const { Replication, getDeployedAppID } = require("@budibase/backend-core/db")
 const { DocumentTypes, getAutomationParams } = require("../../../db/utils")
 const {
   disableAllCrons,
   enableCronTrigger,
 } = require("../../../automations/utils")
-const { app: appCache } = require("@budibase/auth/cache")
+const { app: appCache } = require("@budibase/backend-core/cache")
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000

package/src/api/controllers/dev.js

@@ -4,9 +4,9 @@ const env = require("../../environment")
 const { checkSlashesInUrl } = require("../../utilities")
 const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
-const { Replication } = require("@budibase/auth").db
+const { Replication } = require("@budibase/backend-core/db")
 const { DocumentTypes } = require("../../db/utils")
-const { app: appCache } = require("@budibase/auth/cache")
+const { app: appCache } = require("@budibase/backend-core/cache")
 
 async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params

package/src/api/controllers/hosting.js

@@ -1,7 +1,7 @@
 const CouchDB = require("../../db")
 const { getDeployedApps } = require("../../utilities/workerRequests")
-const { getScopedConfig } = require("@budibase/auth/db")
-const { Configs } = require("@budibase/auth").constants
+const { getScopedConfig } = require("@budibase/backend-core/db")
+const { Configs } = require("@budibase/backend-core/constants")
 const { checkSlashesInUrl } = require("../../utilities")
 
 exports.fetchUrls = async ctx => {

package/src/api/controllers/permission.js

@@ -1,10 +1,10 @@
-const { getBuiltinPermissions } = require("@budibase/auth/permissions")
+const { getBuiltinPermissions } = require("@budibase/backend-core/permissions")
 const {
   isBuiltin,
   getDBRoleID,
   getExternalRoleID,
   getBuiltinRoles,
-} = require("@budibase/auth/roles")
+} = require("@budibase/backend-core/roles")
 const { getRoleParams } = require("../../db/utils")
 const CouchDB = require("../../db")
 const {

package/src/api/controllers/query/import/index.ts

@@ -0,0 +1,85 @@
+import CouchDB from "../../../../db"
+import { queryValidation } from "../validation"
+import { generateQueryID } from "../../../../db/utils"
+import { ImportInfo, ImportSource } from "./sources/base"
+import { OpenAPI2 } from "./sources/openapi2"
+import { Query } from "./../../../../definitions/common"
+import { Curl } from "./sources/curl"
+interface ImportResult {
+  errorQueries: Query[]
+  queries: Query[]
+}
+
+export class RestImporter {
+  data: string
+  sources: ImportSource[]
+  source!: ImportSource
+
+  constructor(data: string) {
+    this.data = data
+    this.sources = [new OpenAPI2(), new Curl()]
+  }
+
+  init = async () => {
+    for (let source of this.sources) {
+      if (await source.isSupported(this.data)) {
+        this.source = source
+        break
+      }
+    }
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    return this.source.getInfo()
+  }
+
+  importQueries = async (
+    appId: string,
+    datasourceId: string
+  ): Promise<ImportResult> => {
+    // constuct the queries
+    let queries = await this.source.getQueries(datasourceId)
+
+    // validate queries
+    const errorQueries: Query[] = []
+    const schema = queryValidation()
+    queries = queries
+      .filter(query => {
+        const validation = schema.validate(query)
+        if (validation.error) {
+          errorQueries.push(query)
+          return false
+        }
+        return true
+      })
+      .map(query => {
+        query._id = generateQueryID(query.datasourceId)
+        return query
+      })
+
+    // persist queries
+    const db = new CouchDB(appId)
+    const response = await db.bulkDocs(queries)
+
+    // create index to seperate queries and errors
+    const queryIndex = queries.reduce((acc, query) => {
+      if (query._id) {
+        acc[query._id] = query
+      }
+      return acc
+    }, {} as { [key: string]: Query })
+
+    // check for failed writes
+    response.forEach((query: any) => {
+      if (!query.ok) {
+        errorQueries.push(queryIndex[query.id])
+        delete queryIndex[query.id]
+      }
+    })
+
+    return {
+      errorQueries,
+      queries: Object.values(queryIndex),
+    }
+  }
+}