@budibase/server 1.0.199 → 1.0.200-alpha.2

package/dist/migrations/index.js

@@ -31,44 +31,108 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.migrate = exports.MIGRATIONS = void 0;
-const { MIGRATION_TYPES, runMigrations, } = require("@budibase/backend-core/migrations");
+exports.migrate = exports.MIGRATIONS = exports.buildMigrations = void 0;
+const backend_core_1 = require("@budibase/backend-core");
+const types_1 = require("@budibase/types");
+const environment_1 = __importDefault(require("../environment"));
 // migration functions
 const userEmailViewCasing = __importStar(require("./functions/userEmailViewCasing"));
 const quota1 = __importStar(require("./functions/quotas1"));
 const appUrls = __importStar(require("./functions/appUrls"));
 const developerQuota = __importStar(require("./functions/developerQuota"));
 const publishedAppsQuota = __importStar(require("./functions/publishedAppsQuota"));
-exports.MIGRATIONS = [
-    {
-        type: MIGRATION_TYPES.GLOBAL,
-        name: "user_email_view_casing",
-        fn: userEmailViewCasing.run,
-    },
-    {
-        type: MIGRATION_TYPES.GLOBAL,
-        name: "quotas_1",
-        fn: quota1.run,
-    },
-    {
-        type: MIGRATION_TYPES.APP,
-        name: "app_urls",
-        opts: { all: true },
-        fn: appUrls.run,
-    },
-    {
-        type: MIGRATION_TYPES.GLOBAL,
-        name: "developer_quota",
-        fn: developerQuota.run,
-    },
-    {
-        type: MIGRATION_TYPES.GLOBAL,
-        name: "published_apps_quota",
-        fn: publishedAppsQuota.run,
-    },
-];
+const backfill = __importStar(require("./functions/backfill"));
+/**
+ * Populate the migration function and additional configuration from
+ * the static migration definitions.
+ */
+const buildMigrations = () => {
+    const definitions = backend_core_1.migrations.DEFINITIONS;
+    const serverMigrations = [];
+    for (const definition of definitions) {
+        switch (definition.name) {
+            case types_1.MigrationName.USER_EMAIL_VIEW_CASING: {
+                serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: userEmailViewCasing.run }));
+                break;
+            }
+            case types_1.MigrationName.QUOTAS_1: {
+                serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: quota1.run }));
+                break;
+            }
+            case types_1.MigrationName.APP_URLS: {
+                serverMigrations.push(Object.assign(Object.assign({}, definition), { appOpts: { all: true }, fn: appUrls.run }));
+                break;
+            }
+            case types_1.MigrationName.DEVELOPER_QUOTA: {
+                serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: developerQuota.run }));
+                break;
+            }
+            case types_1.MigrationName.PUBLISHED_APP_QUOTA: {
+                serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: publishedAppsQuota.run }));
+                break;
+            }
+            case types_1.MigrationName.EVENT_APP_BACKFILL: {
+                serverMigrations.push(Object.assign(Object.assign({}, definition), { appOpts: { all: true }, fn: backfill.app.run, silent: !!environment_1.default.SELF_HOSTED, preventRetry: !!environment_1.default.SELF_HOSTED }));
+                break;
+            }
+            case types_1.MigrationName.EVENT_GLOBAL_BACKFILL: {
+                serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: backfill.global.run, silent: !!environment_1.default.SELF_HOSTED, preventRetry: !!environment_1.default.SELF_HOSTED }));
+                break;
+            }
+            case types_1.MigrationName.EVENT_INSTALLATION_BACKFILL: {
+                serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: backfill.installation.run, silent: !!environment_1.default.SELF_HOSTED, preventRetry: !!environment_1.default.SELF_HOSTED }));
+                break;
+            }
+        }
+    }
+    return serverMigrations;
+};
+exports.buildMigrations = buildMigrations;
+exports.MIGRATIONS = (0, exports.buildMigrations)();
 const migrate = (options) => __awaiter(void 0, void 0, void 0, function* () {
-    yield runMigrations(exports.MIGRATIONS, options);
+    if (environment_1.default.SELF_HOSTED) {
+        // self host runs migrations on startup
+        // make sure only a single instance runs them
+        yield migrateWithLock(options);
+    }
+    else {
+        yield backend_core_1.migrations.runMigrations(exports.MIGRATIONS, options);
+    }
 });
 exports.migrate = migrate;
+const migrateWithLock = (options) => __awaiter(void 0, void 0, void 0, function* () {
+    // get a new lock client
+    const redlock = yield backend_core_1.redis.clients.getMigrationsRedlock();
+    // lock for 15 minutes
+    const ttl = 1000 * 60 * 15;
+    let migrationLock;
+    // acquire lock
+    try {
+        migrationLock = yield redlock.lock("migrations", ttl);
+    }
+    catch (e) {
+        if (e.name === "LockError") {
+            return;
+        }
+        else {
+            throw e;
+        }
+    }
+    // run migrations
+    try {
+        yield backend_core_1.migrations.runMigrations(exports.MIGRATIONS, options);
+    }
+    finally {
+        // release lock
+        try {
+            yield migrationLock.unlock();
+        }
+        catch (e) {
+            console.error("unable to release migration lock");
+        }
+    }
+});

package/dist/migrations/tests/helpers.js

@@ -0,0 +1,72 @@
+"use strict";
+// Mimic configs test configuration from worker, creation configs directly in database
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.saveSmtpConfig = exports.saveOIDCConfig = exports.saveGoogleConfig = exports.saveSettingsConfig = void 0;
+const structures = __importStar(require("./structures"));
+const backend_core_1 = require("@budibase/backend-core");
+const saveSettingsConfig = (globalDb) => __awaiter(void 0, void 0, void 0, function* () {
+    const config = structures.settings();
+    yield saveConfig(config, globalDb);
+});
+exports.saveSettingsConfig = saveSettingsConfig;
+const saveGoogleConfig = (globalDb) => __awaiter(void 0, void 0, void 0, function* () {
+    const config = structures.google();
+    yield saveConfig(config, globalDb);
+});
+exports.saveGoogleConfig = saveGoogleConfig;
+const saveOIDCConfig = (globalDb) => __awaiter(void 0, void 0, void 0, function* () {
+    const config = structures.oidc();
+    yield saveConfig(config, globalDb);
+});
+exports.saveOIDCConfig = saveOIDCConfig;
+const saveSmtpConfig = (globalDb) => __awaiter(void 0, void 0, void 0, function* () {
+    const config = structures.smtp();
+    yield saveConfig(config, globalDb);
+});
+exports.saveSmtpConfig = saveSmtpConfig;
+const saveConfig = (config, globalDb) => __awaiter(void 0, void 0, void 0, function* () {
+    config._id = backend_core_1.db.generateConfigID({ type: config.type });
+    let response;
+    try {
+        response = yield globalDb.get(config._id);
+        config._rev = response._rev;
+        yield globalDb.put(config);
+    }
+    catch (e) {
+        if (e.status === 404) {
+            yield globalDb.put(config);
+        }
+    }
+});

package/dist/migrations/tests/structures.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.settings = exports.smtp = exports.google = exports.oidc = void 0;
+const backend_core_1 = require("@budibase/backend-core");
+const types_1 = require("@budibase/types");
+const oidc = (conf) => {
+    return {
+        type: types_1.ConfigType.OIDC,
+        config: {
+            configs: [
+                Object.assign({ configUrl: "http://someconfigurl", clientID: "clientId", clientSecret: "clientSecret", logo: "Microsoft", name: "Active Directory", uuid: backend_core_1.utils.newid(), activated: true }, conf),
+            ],
+        },
+    };
+};
+exports.oidc = oidc;
+const google = (conf) => {
+    return {
+        type: types_1.ConfigType.GOOGLE,
+        config: Object.assign({ clientID: "clientId", clientSecret: "clientSecret", activated: true }, conf),
+    };
+};
+exports.google = google;
+const smtp = (conf) => {
+    return {
+        type: types_1.ConfigType.SMTP,
+        config: Object.assign({ port: 12345, host: "smtptesthost.com", from: "testfrom@test.com", subject: "Hello!", secure: false }, conf),
+    };
+};
+exports.smtp = smtp;
+const settings = (conf) => {
+    return {
+        type: types_1.ConfigType.SETTINGS,
+        config: Object.assign({ platformUrl: "http://mycustomdomain.com", logoUrl: "http://mylogourl,com", company: "mycompany" }, conf),
+    };
+};
+exports.settings = settings;

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@budibase/server",
     "email": "hi@budibase.com",
-    "version": "1.0.198",
+    "version": "1.0.200-alpha.1",
     "description": "Budibase Web Server",
     "main": "src/index.ts",
     "repository": {
@@ -9,7 +9,9 @@
         "url": "https://github.com/Budibase/budibase.git"
     },
     "scripts": {
-        "build": "rimraf dist/ && tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/ && yarn postbuild",
+        "prebuild": "rimraf dist/",
+        "build": "tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/",
+        "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
         "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
         "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
         "test": "jest --coverage --maxWorkers=2",
@@ -37,6 +39,11 @@
     "jest": {
         "preset": "ts-jest",
         "testEnvironment": "node",
+        "moduleNameMapper": {
+            "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
+            "@budibase/backend-core": "<rootDir>/../backend-core/src",
+            "@budibase/types": "<rootDir>/../types/src"
+        },
         "setupFiles": [
             "./scripts/jestSetup.js"
         ],
@@ -69,120 +76,123 @@
     "author": "Budibase",
     "license": "GPL-3.0",
     "dependencies": {
-        "@apidevtools/swagger-parser": "^10.0.3",
-        "@budibase/backend-core": "^1.0.198",
-        "@budibase/client": "^1.0.198",
-        "@budibase/pro": "1.0.198",
-        "@budibase/string-templates": "^1.0.198",
-        "@bull-board/api": "^3.7.0",
-        "@bull-board/koa": "^3.7.0",
+        "@apidevtools/swagger-parser": "10.0.3",
+        "@budibase/backend-core": "^1.0.200-alpha.1",
+        "@budibase/client": "^1.0.200-alpha.1",
+        "@budibase/pro": "1.0.200-alpha.1",
+        "@budibase/string-templates": "^1.0.200-alpha.1",
+        "@bull-board/api": "3.7.0",
+        "@bull-board/koa": "3.9.4",
         "@elastic/elasticsearch": "7.10.0",
-        "@google-cloud/firestore": "^5.0.2",
+        "@google-cloud/firestore": "5.0.2",
         "@koa/router": "8.0.0",
         "@sendgrid/mail": "7.1.1",
         "@sentry/node": "6.17.7",
-        "@types/global-agent": "^2.1.1",
-        "@types/koa__router": "^8.0.11",
         "airtable": "0.10.1",
         "arangojs": "7.2.0",
-        "aws-sdk": "^2.767.0",
+        "aws-sdk": "2.1030.0",
         "bcryptjs": "2.4.3",
-        "bull": "^3.22.4",
+        "bull": "3.29.3",
         "chmodr": "1.2.0",
         "csvtojson": "2.0.10",
-        "curlconverter": "^3.21.0",
+        "curlconverter": "3.21.0",
         "dotenv": "8.2.0",
         "download": "8.0.0",
         "fix-path": "3.0.0",
-        "form-data": "^4.0.0",
+        "form-data": "4.0.0",
         "fs-extra": "8.1.0",
-        "global-agent": "^3.0.0",
-        "google-auth-library": "^7.11.0",
-        "google-spreadsheet": "^3.2.0",
+        "global-agent": "3.0.0",
+        "google-auth-library": "7.12.0",
+        "google-spreadsheet": "3.2.0",
         "jimp": "0.16.1",
         "joi": "17.2.1",
-        "js-yaml": "^4.1.0",
+        "js-yaml": "4.1.0",
         "jsonschema": "1.4.0",
-        "knex": "^0.95.6",
+        "knex": "0.95.15",
         "koa": "2.7.0",
         "koa-body": "4.2.0",
         "koa-compress": "4.0.1",
-        "koa-connect": "^2.1.0",
+        "koa-connect": "2.1.0",
         "koa-pino-logger": "3.0.0",
         "koa-send": "5.0.0",
         "koa-session": "5.12.0",
         "koa-static": "5.0.0",
-        "koa2-ratelimit": "^1.1.0",
+        "koa2-ratelimit": "1.1.0",
         "lodash": "4.17.21",
-        "memorystream": "^0.3.1",
+        "memorystream": "0.3.1",
         "mongodb": "3.6.3",
         "mssql": "6.2.3",
-        "mysql2": "^2.3.1",
+        "mysql2": "2.3.3",
         "node-fetch": "2.6.7",
-        "open": "^8.4.0",
+        "open": "8.4.0",
         "pg": "8.5.1",
         "pino-pretty": "4.0.0",
-        "posthog-node": "^1.1.4",
+        "posthog-node": "1.3.0",
         "pouchdb": "7.3.0",
-        "pouchdb-adapter-memory": "^7.2.1",
+        "pouchdb-adapter-memory": "7.2.2",
         "pouchdb-all-dbs": "1.0.2",
-        "pouchdb-find": "^7.2.2",
+        "pouchdb-find": "7.2.2",
         "pouchdb-replication-stream": "1.2.9",
         "redis": "4",
         "server-destroy": "1.0.1",
-        "svelte": "^3.38.2",
-        "swagger-parser": "^10.0.3",
+        "svelte": "3.44.1",
+        "swagger-parser": "10.0.3",
         "to-json-schema": "0.2.5",
         "uuid": "3.3.2",
         "validate.js": "0.13.1",
-        "vm2": "^3.9.3",
-        "worker-farm": "^1.7.0",
-        "xml2js": "^0.4.23",
+        "vm2": "3.9.6",
+        "worker-farm": "1.7.0",
+        "xml2js": "0.4.23",
         "yargs": "13.2.4",
         "zlib": "1.0.5"
     },
     "devDependencies": {
-        "@babel/core": "^7.14.3",
-        "@babel/preset-env": "^7.14.4",
+        "@babel/core": "7.17.4",
+        "@babel/preset-env": "7.16.11",
         "@budibase/standard-components": "^0.9.139",
-        "@jest/test-sequencer": "^24.8.0",
-        "@types/apidoc": "^0.50.0",
-        "@types/bson": "^4.2.0",
-        "@types/bull": "^3.15.1",
-        "@types/google-spreadsheet": "^3.1.5",
-        "@types/jest": "^27.4.1",
-        "@types/koa": "^2.13.3",
-        "@types/koa-router": "^7.4.2",
+        "@budibase/types": "^1.0.200-alpha.1",
+        "@jest/test-sequencer": "24.9.0",
+        "@types/apidoc": "0.50.0",
+        "@types/bson": "4.2.0",
+        "@types/bull": "3.15.8",
+        "@types/global-agent": "2.1.1",
+        "@types/google-spreadsheet": "3.1.5",
+        "@types/jest": "27.5.1",
+        "@types/koa": "2.13.4",
+        "@types/koa__router": "8.0.0",
         "@types/lodash": "4.14.180",
         "@types/mongodb": "3.6.3",
-        "@types/node": "^15.12.4",
-        "@types/oracledb": "^5.2.1",
-        "@types/redis": "^4.0.11",
+        "@types/node": "14.18.20",
+        "@types/node-fetch": "2.6.1",
+        "@types/oracledb": "5.2.2",
+        "@types/redis": "4.0.11",
         "@typescript-eslint/parser": "5.12.0",
-        "apidoc": "^0.50.2",
-        "babel-jest": "^27.0.2",
-        "copyfiles": "^2.4.1",
-        "docker-compose": "^0.23.6",
-        "eslint": "^6.8.0",
-        "ioredis-mock": "^7.2.0",
-        "is-wsl": "^2.2.0",
-        "jest": "^27.0.5",
-        "jest-openapi": "^0.14.2",
-        "nodemon": "^2.0.4",
-        "openapi-types": "^9.3.1",
-        "openapi-typescript": "^5.2.0",
-        "path-to-regexp": "^6.2.0",
-        "prettier": "^2.3.1",
-        "rimraf": "^3.0.2",
-        "supertest": "^4.0.2",
-        "swagger-jsdoc": "^6.1.0",
-        "ts-jest": "^27.0.3",
-        "ts-node": "^10.0.0",
-        "typescript": "^4.5.5",
-        "update-dotenv": "^1.1.1"
+        "apidoc": "0.50.4",
+        "babel-jest": "27.5.1",
+        "copyfiles": "2.4.1",
+        "docker-compose": "0.23.17",
+        "eslint": "6.8.0",
+        "ioredis-mock": "7.2.0",
+        "is-wsl": "2.2.0",
+        "jest": "27.5.1",
+        "jest-openapi": "0.14.2",
+        "nodemon": "2.0.15",
+        "openapi-types": "9.3.1",
+        "openapi-typescript": "5.2.0",
+        "path-to-regexp": "6.2.0",
+        "prettier": "2.5.1",
+        "rimraf": "3.0.2",
+        "supertest": "4.0.2",
+        "swagger-jsdoc": "6.1.0",
+        "timekeeper": "2.2.0",
+        "ts-jest": "27.1.3",
+        "ts-node": "10.5.0",
+        "tsconfig-paths": "4.0.0",
+        "typescript": "4.6.2",
+        "update-dotenv": "1.1.1"
     },
     "optionalDependencies": {
-        "oracledb": "^5.3.0"
+        "oracledb": "5.3.0"
     },
     "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }