ghost 4.15.0 → 4.17.1

package/core/server/api/v3/settings.js

@@ -1,15 +1,14 @@
 const Promise = require('bluebird');
 const _ = require('lodash');
-const validator = require('@tryghost/validator');
 const models = require('../../models');
 const routeSettings = require('../../services/route-settings');
-const frontendSettings = require('../../../frontend/services/settings');
 const i18n = require('../../../shared/i18n');
-const {BadRequestError, NoPermissionError, NotFoundError} = require('@tryghost/errors');
+const {BadRequestError, NoPermissionError} = require('@tryghost/errors');
 const settingsService = require('../../services/settings');
-const settingsCache = require('../../../shared/settings-cache');
 const membersService = require('../../services/members');
 
+const settingsBREADService = settingsService.getSettingsBREADServiceInstance();
+
 module.exports = {
     docName: 'settings',
 
@@ -17,26 +16,7 @@ module.exports = {
         options: ['type', 'group'],
         permissions: true,
         query(frame) {
-            let settings = settingsCache.getAll();
-
-            // CASE: no context passed (functional call)
-            if (!frame.options.context) {
-                return Promise.resolve(settings.filter((setting) => {
-                    return setting.group === 'site';
-                }));
-            }
-
-            if (!frame.options.context.internal) {
-                // CASE: omit core settings unless internal request
-                settings = _.filter(settings, (setting) => {
-                    const isCore = setting.group === 'core';
-                    return !isCore;
-                });
-                // CASE: omit secret settings unless internal request
-                settings = settings.map(settingsService.hideValueIfSecret);
-            }
-
-            return settings;
+            return settingsBREADService.browse(frame.options.context);
         }
     },
 
@@ -55,41 +35,7 @@ module.exports = {
             }
         },
         query(frame) {
-            let setting;
-            if (frame.options.key === 'slack') {
-                const slackURL = settingsCache.get('slack_url', {resolve: false});
-                const slackUsername = settingsCache.get('slack_username', {resolve: false});
-
-                setting = slackURL || slackUsername;
-                setting.key = 'slack';
-                setting.value = [{
-                    url: slackURL && slackURL.value,
-                    username: slackUsername && slackUsername.value
-                }];
-            } else {
-                setting = settingsCache.get(frame.options.key, {resolve: false});
-            }
-
-            if (!setting) {
-                return Promise.reject(new NotFoundError({
-                    message: i18n.t('errors.api.settings.problemFindingSetting', {
-                        key: frame.options.key
-                    })
-                }));
-            }
-
-            // @TODO: handle in settings model permissible fn
-            if (setting.group === 'core' && !(frame.options.context && frame.options.context.internal)) {
-                return Promise.reject(new NoPermissionError({
-                    message: i18n.t('errors.api.settings.accessCoreSettingFromExtReq')
-                }));
-            }
-
-            setting = settingsService.hideValueIfSecret(setting);
-
-            return {
-                [frame.options.key]: setting
-            };
+            return settingsBREADService.read(frame.options.key, frame.options.context);
         }
     },
 
@@ -153,17 +99,7 @@ module.exports = {
         ],
         async query(frame) {
             const {email, type} = frame.data;
-            if (typeof email !== 'string' || !validator.isEmail(email)) {
-                throw new BadRequestError({
-                    message: i18n.t('errors.api.settings.invalidEmailReceived')
-                });
-            }
 
-            if (!type || !['fromAddressUpdate', 'supportAddressUpdate'].includes(type)) {
-                throw new BadRequestError({
-                    message: 'Invalid email type recieved'
-                });
-            }
             try {
                 // Send magic link to update fromAddress
                 await membersService.settings.sendEmailAddressUpdateMagicLink({
@@ -232,76 +168,20 @@ module.exports = {
             }
         },
         async query(frame) {
+            let stripeConnectData;
             const stripeConnectIntegrationToken = frame.data.settings.find(setting => setting.key === 'stripe_connect_integration_token');
 
-            const settings = frame.data.settings.filter((setting) => {
-                // The `stripe_connect_integration_token` "setting" is only used to set the `stripe_connect_*` settings.
-                return ![
-                    'stripe_connect_integration_token',
-                    'stripe_connect_publishable_key',
-                    'stripe_connect_secret_key',
-                    'stripe_connect_livemode',
-                    'stripe_connect_account_id',
-                    'stripe_connect_display_name'
-                ].includes(setting.key)
-                // Remove obfuscated settings
-                && !(setting.value === settingsService.obfuscatedSetting && settingsService.isSecretSetting(setting));
-            });
-
-            const getSetting = setting => settingsCache.get(setting.key, {resolve: false});
-
-            const firstUnknownSetting = settings.find(setting => !getSetting(setting));
-
-            if (firstUnknownSetting) {
-                throw new NotFoundError({
-                    message: i18n.t('errors.api.settings.problemFindingSetting', {
-                        key: firstUnknownSetting.key
-                    })
-                });
-            }
-
-            if (!(frame.options.context && frame.options.context.internal)) {
-                const firstCoreSetting = settings.find(setting => getSetting(setting).group === 'core');
-                if (firstCoreSetting) {
-                    throw new NoPermissionError({
-                        message: i18n.t('errors.api.settings.accessCoreSettingFromExtReq')
-                    });
-                }
-            }
-
             if (stripeConnectIntegrationToken && stripeConnectIntegrationToken.value) {
                 const getSessionProp = prop => frame.original.session[prop];
-                try {
-                    const data = await membersService.stripeConnect.getStripeConnectTokenData(stripeConnectIntegrationToken.value, getSessionProp);
-                    settings.push({
-                        key: 'stripe_connect_publishable_key',
-                        value: data.public_key
-                    });
-                    settings.push({
-                        key: 'stripe_connect_secret_key',
-                        value: data.secret_key
-                    });
-                    settings.push({
-                        key: 'stripe_connect_livemode',
-                        value: data.livemode
-                    });
-                    settings.push({
-                        key: 'stripe_connect_display_name',
-                        value: data.display_name
-                    });
-                    settings.push({
-                        key: 'stripe_connect_account_id',
-                        value: data.account_id
-                    });
-                } catch (err) {
-                    throw new BadRequestError({
-                        err,
-                        message: 'The Stripe Connect token could not be parsed.'
-                    });
-                }
+
+                stripeConnectData = await settingsBREADService.getStripeConnectData(
+                    stripeConnectIntegrationToken,
+                    getSessionProp,
+                    membersService.stripeConnect.getStripeConnectTokenData
+                );
             }
 
-            return models.Settings.edit(settings, frame.options);
+            return await settingsBREADService.edit(frame.data.settings, frame.options, stripeConnectData);
         }
     },
 
@@ -313,8 +193,8 @@ module.exports = {
             method: 'edit'
         },
         async query(frame) {
-            await routeSettings.setFromFilePath(frame.file.path);
-            const getRoutesHash = () => frontendSettings.getCurrentHash('routes');
+            await routeSettings.api.setFromFilePath(frame.file.path);
+            const getRoutesHash = () => routeSettings.api.getCurrentHash();
             await settingsService.syncRoutesHash(getRoutesHash);
         }
     },
@@ -333,7 +213,7 @@ module.exports = {
             method: 'browse'
         },
         query() {
-            return routeSettings.get();
+            return routeSettings.api.get();
         }
     }
 };

package/core/server/api/v3/utils/validators/input/settings.js

@@ -1,7 +1,13 @@
 const Promise = require('bluebird');
 const _ = require('lodash');
 const i18n = require('../../../../../../shared/i18n');
-const {NotFoundError, ValidationError} = require('@tryghost/errors');
+const {NotFoundError, ValidationError, BadRequestError} = require('@tryghost/errors');
+const validator = require('@tryghost/validator');
+
+const messages = {
+    invalidEmailReceived: 'Please send a valid email',
+    invalidEmailTypeReceived: 'Invalid email type received'
+};
 
 module.exports = {
     read(apiConfig, frame) {
@@ -62,5 +68,21 @@ module.exports = {
         if (errors.length) {
             return Promise.reject(errors[0]);
         }
+    },
+
+    updateMembersEmail(apiConfig, frame) {
+        const {email, type} = frame.data;
+
+        if (typeof email !== 'string' || !validator.isEmail(email)) {
+            throw new BadRequestError({
+                message: messages.invalidEmailReceived
+            });
+        }
+
+        if (!type || !['fromAddressUpdate', 'supportAddressUpdate'].includes(type)) {
+            throw new BadRequestError({
+                message: messages.invalidEmailTypeReceived
+            });
+        }
     }
 };

package/core/server/data/db/state-manager.js

@@ -39,7 +39,7 @@ class DatabaseStateManager {
             await this.knexMigrator.isDatabaseOK();
             return state;
         } catch (error) {
-            // CASE: database has not yet been initialised
+            // CASE: database has not yet been initialized
             if (error.code === 'DB_NOT_INITIALISED') {
                 state = states.NEEDS_INITIALISATION;
                 return state;

package/core/server/data/exporter/table-lists.js

@@ -36,7 +36,8 @@ const BACKUP_TABLES = [
     'members_status_events',
     'members_paid_subscription_events',
     'members_subscribe_events',
-    'members_product_events'
+    'members_product_events',
+    'offers'
 ];
 
 // NOTE: exposing only tables which are going to be included in a "default" export file
@@ -50,6 +51,7 @@ const TABLES_ALLOWLIST = [
     'roles',
     'roles_users',
     'settings',
+    'custom_theme_settings',
     'tags',
     'users'
 ];

package/core/server/data/importer/import-manager.js

@@ -0,0 +1,398 @@
+const _ = require('lodash');
+const Promise = require('bluebird');
+const fs = require('fs-extra');
+const path = require('path');
+const os = require('os');
+const glob = require('glob');
+const uuid = require('uuid');
+const {extract} = require('@tryghost/zip');
+const {pipeline, sequence} = require('@tryghost/promise');
+const i18n = require('../../../shared/i18n');
+const logging = require('@tryghost/logging');
+const errors = require('@tryghost/errors');
+const ImageHandler = require('./handlers/image');
+const JSONHandler = require('./handlers/json');
+const MarkdownHandler = require('./handlers/markdown');
+const ImageImporter = require('./importers/image');
+const DataImporter = require('./importers/data');
+
+// Glob levels
+const ROOT_ONLY = 0;
+
+const ROOT_OR_SINGLE_DIR = 1;
+const ALL_DIRS = 2;
+let defaults;
+
+defaults = {
+    extensions: ['.zip'],
+    contentTypes: ['application/zip', 'application/x-zip-compressed'],
+    directories: []
+};
+
+class ImportManager {
+    constructor() {
+        this.importers = [ImageImporter, DataImporter];
+        this.handlers = [ImageHandler, JSONHandler, MarkdownHandler];
+
+        // Keep track of file to cleanup at the end
+        this.fileToDelete = null;
+    }
+
+    /**
+     * Get an array of all the file extensions for which we have handlers
+     * @returns {string[]}
+     */
+    getExtensions() {
+        return _.flatten(_.union(_.map(this.handlers, 'extensions'), defaults.extensions));
+    }
+
+    /**
+     * Get an array of all the mime types for which we have handlers
+     * @returns {string[]}
+     */
+    getContentTypes() {
+        return _.flatten(_.union(_.map(this.handlers, 'contentTypes'), defaults.contentTypes));
+    }
+
+    /**
+     * Get an array of directories for which we have handlers
+     * @returns {string[]}
+     */
+    getDirectories() {
+        return _.flatten(_.union(_.map(this.handlers, 'directories'), defaults.directories));
+    }
+
+    /**
+     * Convert items into a glob string
+     * @param {String[]} items
+     * @returns {String}
+     */
+    getGlobPattern(items) {
+        return '+(' + _.reduce(items, function (memo, ext) {
+            return memo !== '' ? memo + '|' + ext : ext;
+        }, '') + ')';
+    }
+
+    /**
+     * @param {String[]} extensions
+     * @param {Number} level
+     * @returns {String}
+     */
+    getExtensionGlob(extensions, level) {
+        const prefix = level === ALL_DIRS ? '**/*' :
+            (level === ROOT_OR_SINGLE_DIR ? '{*/*,*}' : '*');
+
+        return prefix + this.getGlobPattern(extensions);
+    }
+
+    /**
+     *
+     * @param {String[]} directories
+     * @param {Number} level
+     * @returns {String}
+     */
+    getDirectoryGlob(directories, level) {
+        const prefix = level === ALL_DIRS ? '**/' :
+            (level === ROOT_OR_SINGLE_DIR ? '{*/,}' : '');
+
+        return prefix + this.getGlobPattern(directories);
+    }
+
+    /**
+     * Remove files after we're done (abstracted into a function for easier testing)
+     * @returns {Function}
+     */
+    cleanUp() {
+        const self = this;
+
+        if (self.fileToDelete === null) {
+            return;
+        }
+
+        fs.remove(self.fileToDelete, function (err) {
+            if (err) {
+                logging.error(new errors.GhostError({
+                    err: err,
+                    context: i18n.t('errors.data.importer.index.couldNotCleanUpFile.error'),
+                    help: i18n.t('errors.data.importer.index.couldNotCleanUpFile.context')
+                }));
+            }
+
+            self.fileToDelete = null;
+        });
+    }
+
+    /**
+     * Return true if the given file is a Zip
+     * @returns Boolean
+     */
+    isZip(ext) {
+        return _.includes(defaults.extensions, ext);
+    }
+
+    /**
+     * Checks the content of a zip folder to see if it is valid.
+     * Importable content includes any files or directories which the handlers can process
+     * Importable content must be found either in the root, or inside one base directory
+     *
+     * @param {String} directory
+     * @returns {Promise}
+     */
+    isValidZip(directory) {
+        // Globs match content in the root or inside a single directory
+        const extMatchesBase = glob.sync(this.getExtensionGlob(this.getExtensions(), ROOT_OR_SINGLE_DIR), {cwd: directory});
+
+        const extMatchesAll = glob.sync(
+            this.getExtensionGlob(this.getExtensions(), ALL_DIRS), {cwd: directory}
+        );
+
+        const dirMatches = glob.sync(
+            this.getDirectoryGlob(this.getDirectories(), ROOT_OR_SINGLE_DIR), {cwd: directory}
+        );
+
+        const oldRoonMatches = glob.sync(this.getDirectoryGlob(['drafts', 'published', 'deleted'], ROOT_OR_SINGLE_DIR),
+            {cwd: directory});
+
+        // This is a temporary extra message for the old format roon export which doesn't work with Ghost
+        if (oldRoonMatches.length > 0) {
+            throw new errors.UnsupportedMediaTypeError({message: i18n.t('errors.data.importer.index.unsupportedRoonExport')});
+        }
+
+        // If this folder contains importable files or a content or images directory
+        if (extMatchesBase.length > 0 || (dirMatches.length > 0 && extMatchesAll.length > 0)) {
+            return true;
+        }
+
+        if (extMatchesAll.length < 1) {
+            throw new errors.UnsupportedMediaTypeError({message: i18n.t('errors.data.importer.index.noContentToImport')});
+        }
+
+        throw new errors.UnsupportedMediaTypeError({message: i18n.t('errors.data.importer.index.invalidZipStructure')});
+    }
+
+    /**
+     * Use the extract module to extract the given zip file to a temp directory & return the temp directory path
+     * @param {String} filePath
+     * @returns {Promise[]} Files
+     */
+    extractZip(filePath) {
+        const tmpDir = path.join(os.tmpdir(), uuid.v4());
+        this.fileToDelete = tmpDir;
+
+        return extract(filePath, tmpDir).then(function () {
+            return tmpDir;
+        });
+    }
+
+    /**
+     * Use the handler extensions to get a globbing pattern, then use that to fetch all the files from the zip which
+     * are relevant to the given handler, and return them as a name and path combo
+     * @param {Object} handler
+     * @param {String} directory
+     * @returns [] Files
+     */
+    getFilesFromZip(handler, directory) {
+        const globPattern = this.getExtensionGlob(handler.extensions, ALL_DIRS);
+        return _.map(glob.sync(globPattern, {cwd: directory}), function (file) {
+            return {name: file, path: path.join(directory, file)};
+        });
+    }
+
+    /**
+     * Get the name of the single base directory if there is one, else return an empty string
+     * @param {String} directory
+     * @returns {String}
+     */
+    getBaseDirectory(directory) {
+        // Globs match root level only
+        const extMatches = glob.sync(this.getExtensionGlob(this.getExtensions(), ROOT_ONLY), {cwd: directory});
+
+        const dirMatches = glob.sync(this.getDirectoryGlob(this.getDirectories(), ROOT_ONLY), {cwd: directory});
+        let extMatchesAll;
+
+        // There is no base directory
+        if (extMatches.length > 0 || dirMatches.length > 0) {
+            return;
+        }
+        // There is a base directory, grab it from any ext match
+        extMatchesAll = glob.sync(
+            this.getExtensionGlob(this.getExtensions(), ALL_DIRS), {cwd: directory}
+        );
+        if (extMatchesAll.length < 1 || extMatchesAll[0].split('/') < 1) {
+            throw new errors.ValidationError({message: i18n.t('errors.data.importer.index.invalidZipFileBaseDirectory')});
+        }
+
+        return extMatchesAll[0].split('/')[0];
+    }
+
+    /**
+     * Process Zip
+     * Takes a reference to a zip file, extracts it, sends any relevant files from inside to the right handler, and
+     * returns an object in the importData format: {data: {}, images: []}
+     * The data key contains JSON representing any data that should be imported
+     * The image key contains references to images that will be stored (and where they will be stored)
+     * @param {File} file
+     * @returns {Promise(ImportData)}
+     */
+    processZip(file) {
+        const self = this;
+
+        return this.extractZip(file.path).then(function (zipDirectory) {
+            const ops = [];
+            const importData = {};
+            let baseDir;
+
+            self.isValidZip(zipDirectory);
+            baseDir = self.getBaseDirectory(zipDirectory);
+
+            _.each(self.handlers, function (handler) {
+                if (Object.prototype.hasOwnProperty.call(importData, handler.type)) {
+                    // This limitation is here to reduce the complexity of the importer for now
+                    return Promise.reject(new errors.UnsupportedMediaTypeError({
+                        message: i18n.t('errors.data.importer.index.zipContainsMultipleDataFormats')
+                    }));
+                }
+
+                const files = self.getFilesFromZip(handler, zipDirectory);
+
+                if (files.length > 0) {
+                    ops.push(function () {
+                        return handler.loadFile(files, baseDir).then(function (data) {
+                            importData[handler.type] = data;
+                        });
+                    });
+                }
+            });
+
+            if (ops.length === 0) {
+                return Promise.reject(new errors.UnsupportedMediaTypeError({
+                    message: i18n.t('errors.data.importer.index.noContentToImport')
+                }));
+            }
+
+            return sequence(ops).then(function () {
+                return importData;
+            });
+        });
+    }
+
+    /**
+     * Process File
+     * Takes a reference to a single file, sends it to the relevant handler to be loaded and returns an object in the
+     * importData format: {data: {}, images: []}
+     * The data key contains JSON representing any data that should be imported
+     * The image key contains references to images that will be stored (and where they will be stored)
+     * @param {File} file
+     * @returns {Promise(ImportData)}
+     */
+    processFile(file, ext) {
+        const fileHandler = _.find(this.handlers, function (handler) {
+            return _.includes(handler.extensions, ext);
+        });
+
+        return fileHandler.loadFile([_.pick(file, 'name', 'path')]).then(function (loadedData) {
+            // normalize the returned data
+            const importData = {};
+            importData[fileHandler.type] = loadedData;
+            return importData;
+        });
+    }
+
+    /**
+     * Import Step 1:
+     * Load the given file into usable importData in the format: {data: {}, images: []}, regardless of
+     * whether the file is a single importable file like a JSON file, or a zip file containing loads of files.
+     * @param {File} file
+     * @returns {Promise}
+     */
+    loadFile(file) {
+        const self = this;
+        const ext = path.extname(file.name).toLowerCase();
+        return this.isZip(ext) ? self.processZip(file) : self.processFile(file, ext);
+    }
+
+    /**
+     * Import Step 2:
+     * Pass the prepared importData through the preProcess function of the various importers, so that the importers can
+     * make any adjustments to the data based on relationships between it
+     * @param {ImportData} importData
+     * @returns {Promise(ImportData)}
+     */
+    preProcess(importData) {
+        const ops = [];
+        _.each(this.importers, function (importer) {
+            ops.push(function () {
+                return importer.preProcess(importData);
+            });
+        });
+
+        return pipeline(ops);
+    }
+
+    /**
+     * Import Step 3:
+     * Each importer gets passed the data from importData which has the key matching its type - i.e. it only gets the
+     * data that it should import. Each importer then handles actually importing that data into Ghost
+     * @param {ImportData} importData
+     * @param {Object} importOptions to allow override of certain import features such as locking a user
+     * @returns {Promise<any>}
+     */
+    doImport(importData, importOptions) {
+        importOptions = importOptions || {};
+        const ops = [];
+        _.each(this.importers, function (importer) {
+            if (Object.prototype.hasOwnProperty.call(importData, importer.type)) {
+                ops.push(function () {
+                    return importer.doImport(importData[importer.type], importOptions);
+                });
+            }
+        });
+
+        return sequence(ops).then(function (importResult) {
+            return importResult;
+        });
+    }
+
+    /**
+     * Import Step 4:
+     * Report on what was imported, currently a no-op
+     * @param {ImportData} importData
+     * @returns {Promise<ImportData>}
+     */
+    generateReport(importData) {
+        return Promise.resolve(importData);
+    }
+
+    /**
+     * Import From File
+     * The main method of the ImportManager, call this to kick everything off!
+     * @param {File} file
+     * @param {Object} importOptions to allow override of certain import features such as locking a user
+     * @returns {Promise}
+     */
+    importFromFile(file, importOptions = {}) {
+        const self = this;
+
+        // Step 1: Handle converting the file to usable data
+        return this.loadFile(file).then(function (importData) {
+            // Step 2: Let the importers pre-process the data
+            return self.preProcess(importData);
+        }).then(function (importData) {
+            // Step 3: Actually do the import
+            // @TODO: It would be cool to have some sort of dry run flag here
+            return self.doImport(importData, importOptions);
+        }).then(function (importData) {
+            // Step 4: Report on the import
+            return self.generateReport(importData);
+        }).finally(() => self.cleanUp()); // Step 5: Cleanup any files
+    }
+}
+
+/**
+ * A number, or a string containing a number.
+ * @typedef {Object} ImportData
+ * @property [Object] data
+ * @property [Array] images
+ */
+
+module.exports = new ImportManager();