ghost 4.13.0 → 4.16.0

package/core/server/data/importer/importers/data/index.js

@@ -1,162 +1 @@
-const _ = require('lodash');
-const Promise = require('bluebird');
-const semver = require('semver');
-const {IncorrectUsageError} = require('@tryghost/errors');
-const debug = require('@tryghost/debug')('importer:data');
-const {sequence} = require('@tryghost/promise');
-const models = require('../../../../models');
-const PostsImporter = require('./posts');
-const TagsImporter = require('./tags');
-const SettingsImporter = require('./settings');
-const UsersImporter = require('./users');
-const RolesImporter = require('./roles');
-let importers = {};
-let DataImporter;
-
-DataImporter = {
-    type: 'data',
-
-    preProcess: function preProcess(importData) {
-        importData.preProcessedByData = true;
-        return importData;
-    },
-
-    init: function init(importData) {
-        importers.users = new UsersImporter(importData.data);
-        importers.roles = new RolesImporter(importData.data);
-        importers.tags = new TagsImporter(importData.data);
-        importers.posts = new PostsImporter(importData.data);
-        importers.settings = new SettingsImporter(importData.data);
-
-        return importData;
-    },
-
-    // Allow importing with an options object that is passed through the importer
-    doImport: function doImport(importData, importOptions) {
-        importOptions = importOptions || {};
-
-        const ops = [];
-        let errors = [];
-        let results = [];
-
-        const modelOptions = {
-            importing: true,
-            context: {
-                internal: true
-            }
-        };
-
-        if (!Object.prototype.hasOwnProperty.call(importOptions, 'returnImportedData')) {
-            importOptions.returnImportedData = false;
-        }
-
-        if (importOptions.importPersistUser) {
-            modelOptions.importPersistUser = importOptions.importPersistUser;
-        }
-
-        if (!importData.meta) {
-            return Promise.reject(new IncorrectUsageError({
-                message: 'Wrong importer structure. `meta` is missing.',
-                help: 'https://ghost.org/docs/migration/custom/'
-            }));
-        }
-
-        if (!importData.meta.version) {
-            return Promise.reject(new IncorrectUsageError({
-                message: 'Wrong importer structure. `meta.version` is missing.',
-                help: 'https://ghost.org/docs/migration/custom/'
-            }));
-        }
-
-        // CASE: We deny LTS imports, because these are major version jumps. Only imports from v1 until the latest are supported.
-        //       We can detect a wrong structure by checking the meta version field. Ghost v0 doesn't use semver compliant versions.
-        if (!semver.valid(importData.meta.version)) {
-            return Promise.reject(new IncorrectUsageError({
-                message: 'Detected unsupported file structure.',
-                help: 'Please install Ghost 1.0, import the file and then update your blog to the latest Ghost version.\nVisit https://ghost.org/docs/update/ or ask for help in our https://forum.ghost.org.'
-            }));
-        }
-
-        this.init(importData);
-
-        return models.Base.transaction(function (transacting) {
-            modelOptions.transacting = transacting;
-
-            _.each(importers, function (importer) {
-                ops.push(function doModelImport() {
-                    return importer.fetchExisting(modelOptions, importOptions)
-                        .then(function () {
-                            return importer.beforeImport(modelOptions, importOptions);
-                        })
-                        .then(function () {
-                            if (importer.options.requiredImportedData.length) {
-                                _.each(importer.options.requiredImportedData, (key) => {
-                                    importer.requiredImportedData[key] = importers[key].importedData;
-                                });
-                            }
-
-                            if (importer.options.requiredExistingData.length) {
-                                _.each(importer.options.requiredExistingData, (key) => {
-                                    importer.requiredExistingData[key] = importers[key].existingData;
-                                });
-                            }
-
-                            return importer.replaceIdentifiers(modelOptions, importOptions);
-                        })
-                        .then(function () {
-                            return importer.doImport(modelOptions, importOptions)
-                                .then(function (_results) {
-                                    results = results.concat(_results);
-                                });
-                        });
-                });
-            });
-
-            sequence(ops)
-                .then(function () {
-                    results.forEach(function (promise) {
-                        if (!promise.isFulfilled()) {
-                            errors = errors.concat(promise.reason());
-                        }
-                    });
-
-                    if (errors.length === 0) {
-                        transacting.commit();
-                    } else {
-                        transacting.rollback(errors);
-                    }
-                });
-        }).then(function () {
-            /**
-             * data: imported data
-             * originalData: data from the json file
-             * problems: warnings
-             */
-            const toReturn = {
-                data: {},
-                originalData: importData.data,
-                problems: []
-            };
-
-            _.each(importers, function (importer) {
-                toReturn.problems = toReturn.problems.concat(importer.problems);
-
-                if (importOptions.returnImportedData) {
-                    toReturn.data[importer.dataKeyToImport] = importer.importedDataToReturn;
-                }
-            });
-
-            return toReturn;
-        }).catch(function (err) {
-            debug(err);
-            return Promise.reject(err);
-        }).finally(() => {
-            // release memory
-            importers = {};
-            results = null;
-            importData = null;
-        });
-    }
-};
-
-module.exports = DataImporter;
+module.exports = require('./data-importer');

package/core/server/data/importer/index.js

@@ -1,379 +1 @@
-const _ = require('lodash');
-const Promise = require('bluebird');
-const fs = require('fs-extra');
-const path = require('path');
-const os = require('os');
-const glob = require('glob');
-const uuid = require('uuid');
-const {extract} = require('@tryghost/zip');
-const {pipeline, sequence} = require('@tryghost/promise');
-const i18n = require('../../../shared/i18n');
-const logging = require('@tryghost/logging');
-const errors = require('@tryghost/errors');
-const ImageHandler = require('./handlers/image');
-const JSONHandler = require('./handlers/json');
-const MarkdownHandler = require('./handlers/markdown');
-const ImageImporter = require('./importers/image');
-const DataImporter = require('./importers/data');
-
-// Glob levels
-const ROOT_ONLY = 0;
-
-const ROOT_OR_SINGLE_DIR = 1;
-const ALL_DIRS = 2;
-let defaults;
-
-defaults = {
-    extensions: ['.zip'],
-    contentTypes: ['application/zip', 'application/x-zip-compressed'],
-    directories: []
-};
-
-function ImportManager() {
-    this.importers = [ImageImporter, DataImporter];
-    this.handlers = [ImageHandler, JSONHandler, MarkdownHandler];
-    // Keep track of file to cleanup at the end
-    this.fileToDelete = null;
-}
-
-/**
- * A number, or a string containing a number.
- * @typedef {Object} ImportData
- * @property [Object] data
- * @property [Array] images
- */
-
-_.extend(ImportManager.prototype, {
-    /**
-     * Get an array of all the file extensions for which we have handlers
-     * @returns {string[]}
-     */
-    getExtensions: function () {
-        return _.flatten(_.union(_.map(this.handlers, 'extensions'), defaults.extensions));
-    },
-    /**
-     * Get an array of all the mime types for which we have handlers
-     * @returns {string[]}
-     */
-    getContentTypes: function () {
-        return _.flatten(_.union(_.map(this.handlers, 'contentTypes'), defaults.contentTypes));
-    },
-    /**
-     * Get an array of directories for which we have handlers
-     * @returns {string[]}
-     */
-    getDirectories: function () {
-        return _.flatten(_.union(_.map(this.handlers, 'directories'), defaults.directories));
-    },
-    /**
-     * Convert items into a glob string
-     * @param {String[]} items
-     * @returns {String}
-     */
-    getGlobPattern: function (items) {
-        return '+(' + _.reduce(items, function (memo, ext) {
-            return memo !== '' ? memo + '|' + ext : ext;
-        }, '') + ')';
-    },
-    /**
-     * @param {String[]} extensions
-     * @param {Number} level
-     * @returns {String}
-     */
-    getExtensionGlob: function (extensions, level) {
-        const prefix = level === ALL_DIRS ? '**/*' :
-            (level === ROOT_OR_SINGLE_DIR ? '{*/*,*}' : '*');
-
-        return prefix + this.getGlobPattern(extensions);
-    },
-    /**
-     *
-     * @param {String[]} directories
-     * @param {Number} level
-     * @returns {String}
-     */
-    getDirectoryGlob: function (directories, level) {
-        const prefix = level === ALL_DIRS ? '**/' :
-            (level === ROOT_OR_SINGLE_DIR ? '{*/,}' : '');
-
-        return prefix + this.getGlobPattern(directories);
-    },
-    /**
-     * Remove files after we're done (abstracted into a function for easier testing)
-     * @returns {Function}
-     */
-    cleanUp: function () {
-        const self = this;
-
-        if (self.fileToDelete === null) {
-            return;
-        }
-
-        fs.remove(self.fileToDelete, function (err) {
-            if (err) {
-                logging.error(new errors.GhostError({
-                    err: err,
-                    context: i18n.t('errors.data.importer.index.couldNotCleanUpFile.error'),
-                    help: i18n.t('errors.data.importer.index.couldNotCleanUpFile.context')
-                }));
-            }
-
-            self.fileToDelete = null;
-        });
-    },
-    /**
-     * Return true if the given file is a Zip
-     * @returns Boolean
-     */
-    isZip: function (ext) {
-        return _.includes(defaults.extensions, ext);
-    },
-    /**
-     * Checks the content of a zip folder to see if it is valid.
-     * Importable content includes any files or directories which the handlers can process
-     * Importable content must be found either in the root, or inside one base directory
-     *
-     * @param {String} directory
-     * @returns {Promise}
-     */
-    isValidZip: function (directory) {
-        // Globs match content in the root or inside a single directory
-        const extMatchesBase = glob.sync(this.getExtensionGlob(this.getExtensions(), ROOT_OR_SINGLE_DIR), {cwd: directory});
-
-        const extMatchesAll = glob.sync(
-            this.getExtensionGlob(this.getExtensions(), ALL_DIRS), {cwd: directory}
-        );
-
-        const dirMatches = glob.sync(
-            this.getDirectoryGlob(this.getDirectories(), ROOT_OR_SINGLE_DIR), {cwd: directory}
-        );
-
-        const oldRoonMatches = glob.sync(this.getDirectoryGlob(['drafts', 'published', 'deleted'], ROOT_OR_SINGLE_DIR),
-            {cwd: directory});
-
-        // This is a temporary extra message for the old format roon export which doesn't work with Ghost
-        if (oldRoonMatches.length > 0) {
-            throw new errors.UnsupportedMediaTypeError({message: i18n.t('errors.data.importer.index.unsupportedRoonExport')});
-        }
-
-        // If this folder contains importable files or a content or images directory
-        if (extMatchesBase.length > 0 || (dirMatches.length > 0 && extMatchesAll.length > 0)) {
-            return true;
-        }
-
-        if (extMatchesAll.length < 1) {
-            throw new errors.UnsupportedMediaTypeError({message: i18n.t('errors.data.importer.index.noContentToImport')});
-        }
-
-        throw new errors.UnsupportedMediaTypeError({message: i18n.t('errors.data.importer.index.invalidZipStructure')});
-    },
-    /**
-     * Use the extract module to extract the given zip file to a temp directory & return the temp directory path
-     * @param {String} filePath
-     * @returns {Promise[]} Files
-     */
-    extractZip: function (filePath) {
-        const tmpDir = path.join(os.tmpdir(), uuid.v4());
-        this.fileToDelete = tmpDir;
-
-        return extract(filePath, tmpDir).then(function () {
-            return tmpDir;
-        });
-    },
-    /**
-     * Use the handler extensions to get a globbing pattern, then use that to fetch all the files from the zip which
-     * are relevant to the given handler, and return them as a name and path combo
-     * @param {Object} handler
-     * @param {String} directory
-     * @returns [] Files
-     */
-    getFilesFromZip: function (handler, directory) {
-        const globPattern = this.getExtensionGlob(handler.extensions, ALL_DIRS);
-        return _.map(glob.sync(globPattern, {cwd: directory}), function (file) {
-            return {name: file, path: path.join(directory, file)};
-        });
-    },
-    /**
-     * Get the name of the single base directory if there is one, else return an empty string
-     * @param {String} directory
-     * @returns {Promise (String)}
-     */
-    getBaseDirectory: function (directory) {
-        // Globs match root level only
-        const extMatches = glob.sync(this.getExtensionGlob(this.getExtensions(), ROOT_ONLY), {cwd: directory});
-
-        const dirMatches = glob.sync(this.getDirectoryGlob(this.getDirectories(), ROOT_ONLY), {cwd: directory});
-        let extMatchesAll;
-
-        // There is no base directory
-        if (extMatches.length > 0 || dirMatches.length > 0) {
-            return;
-        }
-        // There is a base directory, grab it from any ext match
-        extMatchesAll = glob.sync(
-            this.getExtensionGlob(this.getExtensions(), ALL_DIRS), {cwd: directory}
-        );
-        if (extMatchesAll.length < 1 || extMatchesAll[0].split('/') < 1) {
-            throw new errors.ValidationError({message: i18n.t('errors.data.importer.index.invalidZipFileBaseDirectory')});
-        }
-
-        return extMatchesAll[0].split('/')[0];
-    },
-    /**
-     * Process Zip
-     * Takes a reference to a zip file, extracts it, sends any relevant files from inside to the right handler, and
-     * returns an object in the importData format: {data: {}, images: []}
-     * The data key contains JSON representing any data that should be imported
-     * The image key contains references to images that will be stored (and where they will be stored)
-     * @param {File} file
-     * @returns {Promise(ImportData)}
-     */
-    processZip: function (file) {
-        const self = this;
-
-        return this.extractZip(file.path).then(function (zipDirectory) {
-            const ops = [];
-            const importData = {};
-            let baseDir;
-
-            self.isValidZip(zipDirectory);
-            baseDir = self.getBaseDirectory(zipDirectory);
-
-            _.each(self.handlers, function (handler) {
-                if (Object.prototype.hasOwnProperty.call(importData, handler.type)) {
-                    // This limitation is here to reduce the complexity of the importer for now
-                    return Promise.reject(new errors.UnsupportedMediaTypeError({
-                        message: i18n.t('errors.data.importer.index.zipContainsMultipleDataFormats')
-                    }));
-                }
-
-                const files = self.getFilesFromZip(handler, zipDirectory);
-
-                if (files.length > 0) {
-                    ops.push(function () {
-                        return handler.loadFile(files, baseDir).then(function (data) {
-                            importData[handler.type] = data;
-                        });
-                    });
-                }
-            });
-
-            if (ops.length === 0) {
-                return Promise.reject(new errors.UnsupportedMediaTypeError({
-                    message: i18n.t('errors.data.importer.index.noContentToImport')
-                }));
-            }
-
-            return sequence(ops).then(function () {
-                return importData;
-            });
-        });
-    },
-    /**
-     * Process File
-     * Takes a reference to a single file, sends it to the relevant handler to be loaded and returns an object in the
-     * importData format: {data: {}, images: []}
-     * The data key contains JSON representing any data that should be imported
-     * The image key contains references to images that will be stored (and where they will be stored)
-     * @param {File} file
-     * @returns {Promise(ImportData)}
-     */
-    processFile: function (file, ext) {
-        const fileHandler = _.find(this.handlers, function (handler) {
-            return _.includes(handler.extensions, ext);
-        });
-
-        return fileHandler.loadFile([_.pick(file, 'name', 'path')]).then(function (loadedData) {
-            // normalize the returned data
-            const importData = {};
-            importData[fileHandler.type] = loadedData;
-            return importData;
-        });
-    },
-    /**
-     * Import Step 1:
-     * Load the given file into usable importData in the format: {data: {}, images: []}, regardless of
-     * whether the file is a single importable file like a JSON file, or a zip file containing loads of files.
-     * @param {File} file
-     * @returns {Promise}
-     */
-    loadFile: function (file) {
-        const self = this;
-        const ext = path.extname(file.name).toLowerCase();
-        return this.isZip(ext) ? self.processZip(file) : self.processFile(file, ext);
-    },
-    /**
-     * Import Step 2:
-     * Pass the prepared importData through the preProcess function of the various importers, so that the importers can
-     * make any adjustments to the data based on relationships between it
-     * @param {ImportData} importData
-     * @returns {Promise(ImportData)}
-     */
-    preProcess: function (importData) {
-        const ops = [];
-        _.each(this.importers, function (importer) {
-            ops.push(function () {
-                return importer.preProcess(importData);
-            });
-        });
-
-        return pipeline(ops);
-    },
-    /**
-     * Import Step 3:
-     * Each importer gets passed the data from importData which has the key matching its type - i.e. it only gets the
-     * data that it should import. Each importer then handles actually importing that data into Ghost
-     * @param {ImportData} importData
-     * @param {importOptions} importOptions to allow override of certain import features such as locking a user
-     * @returns {Promise(ImportData)}
-     */
-    doImport: function (importData, importOptions) {
-        importOptions = importOptions || {};
-        const ops = [];
-        _.each(this.importers, function (importer) {
-            if (Object.prototype.hasOwnProperty.call(importData, importer.type)) {
-                ops.push(function () {
-                    return importer.doImport(importData[importer.type], importOptions);
-                });
-            }
-        });
-
-        return sequence(ops).then(function (importResult) {
-            return importResult;
-        });
-    },
-    /**
-     * Import Step 4:
-     * Report on what was imported, currently a no-op
-     * @param {ImportData} importData
-     * @returns {Promise(ImportData)}
-     */
-    generateReport: function (importData) {
-        return Promise.resolve(importData);
-    },
-    /**
-     * Import From File
-     * The main method of the ImportManager, call this to kick everything off!
-     * @param {File} file
-     * @param {importOptions} importOptions to allow override of certain import features such as locking a user
-     * @returns {Promise}
-     */
-    importFromFile: function (file, importOptions = {}) {
-        const self = this;
-
-        // Step 1: Handle converting the file to usable data
-        return this.loadFile(file).then(function (importData) {
-            // Step 2: Let the importers pre-process the data
-            return self.preProcess(importData);
-        }).then(function (importData) {
-            // Step 3: Actually do the import
-            // @TODO: It would be cool to have some sort of dry run flag here
-            return self.doImport(importData, importOptions);
-        }).then(function (importData) {
-            // Step 4: Report on the import
-            return self.generateReport(importData);
-        }).finally(() => self.cleanUp()); // Step 5: Cleanup any files
-    }
-});
-
-module.exports = new ImportManager();
+module.exports = require('./import-manager');

package/core/server/data/migrations/versions/4.14/01-fix-comped-member-statuses.js

@@ -0,0 +1,70 @@
+const {chunk} = require('lodash');
+const {createTransactionalMigration} = require('../../utils');
+const logging = require('@tryghost/logging');
+
+module.exports = createTransactionalMigration(async function up(knex) {
+    const compedMemberIds = (await knex('members')
+        .select('members.id')
+        .innerJoin(
+            'members_stripe_customers',
+            'members.id',
+            'members_stripe_customers.member_id'
+        ).innerJoin(
+            'members_stripe_customers_subscriptions',
+            function () {
+                this.on(
+                    'members_stripe_customers.customer_id',
+                    'members_stripe_customers_subscriptions.customer_id'
+                ).onIn(
+                    'members_stripe_customers_subscriptions.status',
+                    ['active', 'trialing', 'past_due', 'unpaid']
+                );
+            }
+        ).where(
+            'members_stripe_customers_subscriptions.plan_nickname',
+            '=',
+            'Complimentary'
+        ).andWhere(
+            'members.status',
+            '!=',
+            'comped'
+        )).map(({id}) => id);
+
+    if (!compedMemberIds.length) {
+        logging.warn('No Complimentary members found with incorrect status');
+        return;
+    } else {
+        logging.info(`Found ${compedMemberIds.length} Complimentary members with the incorrect status`);
+    }
+
+    // Umm? Well... The current version of SQLite3 bundled with Ghost supports
+    // a maximum of 999 variables, we use one variable for the SET value
+    // and so we're left with 998 for our WHERE IN clause values
+    const chunkSize = 998;
+
+    const compedMemberIdChunks = chunk(compedMemberIds, chunkSize);
+
+    for (const compedMemberIdsChunk of compedMemberIdChunks) {
+        await knex('members')
+            .update('status', 'comped')
+            .whereIn('id', compedMemberIdsChunk);
+    }
+
+    for (const memberId of compedMemberIds) {
+        const mostRecentStatusEvent = await knex('members_status_events')
+            .select('*')
+            .where('member_id', memberId)
+            .orderBy('created_at', 'desc')
+            .limit(1)
+            .first();
+
+        if (!mostRecentStatusEvent) {
+            logging.warn(`Could not find a status event for member ${memberId} - skipping this member`);
+        } else if (mostRecentStatusEvent.to_status !== 'comped') {
+            logging.info(`Updating members_status_event ${mostRecentStatusEvent.id}`);
+            await knex('members_status_events')
+                .update('to_status', 'comped')
+                .where('id', mostRecentStatusEvent.id);
+        }
+    }
+}, async function down() {});

package/core/server/data/migrations/versions/4.14/02-fix-free-members-status-events.js

@@ -0,0 +1,60 @@
+const _ = require('lodash');
+const logging = require('@tryghost/logging');
+const {createTransactionalMigration} = require('../../utils.js');
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        logging.info('Updating members_status_events for free members');
+        const freeMemberEvents = await knex('members')
+            .select(
+                'members.id as member_id',
+                'members_status_events.id',
+                'members_status_events.to_status',
+                'members_status_events.created_at'
+            )
+            .innerJoin(
+                'members_status_events',
+                'members.id',
+                'members_status_events.member_id'
+            )
+            .where('members.status', 'free');
+
+        if (freeMemberEvents.length === 0) {
+            logging.info('No free members found - skipping migration');
+            return;
+        }
+
+        const eventsByMember = _.groupBy(freeMemberEvents, 'member_id');
+
+        const eventsToUpdate = Object.keys(eventsByMember).reduce((incorrectEvents, memberId) => {
+            const events = eventsByMember[memberId];
+
+            events.sort((a, b) => {
+                return new Date(b.created_at) - new Date(a.created_at);
+            });
+
+            const mostRecentStatusEvent = events[0];
+
+            if (mostRecentStatusEvent && mostRecentStatusEvent.to_status !== 'free') {
+                return incorrectEvents.concat(mostRecentStatusEvent.id);
+            }
+
+            return incorrectEvents;
+        }, []);
+
+        logging.info(`Found ${eventsToUpdate.length} member status events that need updating`);
+
+        // Umm? Well... The current version of SQLite3 bundled with Ghost supports
+        // a maximum of 999 variables, we use one variable for the SET value
+        // and so we're left with 998 for our WHERE IN clause values
+        const chunkedEventsToUpdate = _.chunk(eventsToUpdate, 998);
+
+        for (const chunk of chunkedEventsToUpdate) {
+            logging.info(`Updating a chunk of ${chunk.length} member status events`);
+            await knex('members_status_events')
+                .update('to_status', 'free')
+                .whereIn('id', chunk);
+        }
+    },
+    async function down() {}
+);

package/core/server/data/migrations/versions/4.15/01-add-temp-members-analytic-events-table.js

@@ -0,0 +1,12 @@
+const utils = require('../../utils');
+
+module.exports = utils.addTable('temp_member_analytic_events', {
+    id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+    event_name: {type: 'string', maxlength: 50, nullable: false},
+    created_at: {type: 'dateTime', nullable: false},
+    member_id: {type: 'string', maxlength: 24, nullable: false},
+    member_status: {type: 'string', maxlength: 50, nullable: false},
+    entry_id: {type: 'string', maxlength: 24, nullable: true},
+    source_url: {type: 'string', maxlength: 2000, nullable: true},
+    metadata: {type: 'string', maxlength: 191, nullable: true}
+});