ghost 4.48.0 → 5.0.2

package/core/server/api/canary/utils/validators/input/{passwordreset.js → password_reset.js}

@@ -13,7 +13,7 @@ module.exports = {
     resetPassword(apiConfig, frame) {
         debug('resetPassword');
 
-        const data = frame.data.passwordreset[0];
+        const data = frame.data.password_reset[0];
 
         if (data.newPassword !== data.ne2Password) {
             return Promise.reject(new errors.ValidationError({
@@ -25,7 +25,7 @@ module.exports = {
     generateResetToken(apiConfig, frame) {
         debug('generateResetToken');
 
-        const email = frame.data.passwordreset[0].email;
+        const email = frame.data.password_reset[0].email;
 
         if (typeof email !== 'string' || !validator.isEmail(email)) {
             throw new errors.BadRequestError({

package/core/server/api/canary/utils/validators/input/settings.js

@@ -1,7 +1,6 @@
 const Promise = require('bluebird');
 const _ = require('lodash');
-const tpl = require('@tryghost/tpl');
-const {NotFoundError, ValidationError, BadRequestError} = require('@tryghost/errors');
+const {ValidationError, BadRequestError} = require('@tryghost/errors');
 const validator = require('@tryghost/validator');
 
 const messages = {
@@ -11,30 +10,10 @@ const messages = {
 };
 
 module.exports = {
-    read(apiConfig, frame) {
-        // @NOTE: was removed https://github.com/TryGhost/Ghost/issues/10373
-        if (frame.options.key === 'ghost_head' || frame.options.key === 'ghost_foot') {
-            return Promise.reject(new NotFoundError({
-                message: tpl(messages.problemFindingSetting, {
-                    key: frame.options.key
-                })
-            }));
-        }
-    },
-
     edit(apiConfig, frame) {
         const errors = [];
 
         _.each(frame.data.settings, (setting) => {
-            if (setting.key === 'ghost_head' || setting.key === 'ghost_foot') {
-                // @NOTE: was removed https://github.com/TryGhost/Ghost/issues/10373
-                errors.push(new NotFoundError({
-                    message: tpl(messages.problemFindingSetting, {
-                        key: setting.key
-                    })
-                }));
-            }
-
             // TODO: the below array is INCOMPLETE
             //       it should include all setting values that have array as a type
             const arrayTypeSettings = [
@@ -80,7 +59,7 @@ module.exports = {
             });
         }
 
-        if (!type || !['fromAddressUpdate', 'supportAddressUpdate'].includes(type)) {
+        if (!type || !['supportAddressUpdate'].includes(type)) {
             throw new BadRequestError({
                 message: messages.invalidEmailTypeReceived
             });

package/core/server/api/canary/utils/validators/utils/json-schema.js

@@ -10,8 +10,7 @@ const jsonSchema = require('@tryghost/admin-api-schema');
  */
 const validate = async (apiConfig, frame) => await jsonSchema.validate({
     data: frame.data,
-    schema: `${apiConfig.docName}-${apiConfig.method}`,
-    version: 'canary'
+    schema: `${apiConfig.docName}-${apiConfig.method}`
 });
 
 module.exports.validate = validate;

package/core/server/api/index.js

@@ -1,9 +1,3 @@
-const defaultAPI = require('./canary');
-
-module.exports = defaultAPI;
-module.exports.canary = defaultAPI;
-module.exports.v4 = defaultAPI;
-module.exports.v3 = require('./v3');
-module.exports.v2 = require('./v2');
+module.exports.endpoints = require('./canary');
 
 module.exports.shared = require('./shared');

package/core/server/api/shared/serializers/input/utils/settings-key-group-mapper.js

@@ -6,7 +6,7 @@ const keyGroupMapping = {
     db_hash: 'core',
     next_update_check: 'core',
     notifications: 'core',
-    session_secret: 'core',
+    admin_session_secret: 'core',
     theme_session_secret: 'core',
     ghost_public_key: 'core',
     ghost_private_key: 'core',
@@ -16,7 +16,7 @@ const keyGroupMapping = {
     cover_image: 'site',
     icon: 'site',
     accent_color: 'site',
-    lang: 'site',
+    locale: 'site',
     timezone: 'site',
     codeinjection_head: 'site',
     codeinjection_foot: 'site',

package/core/server/api/shared/serializers/input/utils/settings-key-type-mapper.js

@@ -1,7 +1,7 @@
 // NOTE: mapping is based on maping present in migration - 3.22/07-update-type-for-settings
 const keyTypeMapping = {
     db_hash: 'string',
-    session_secret: 'string',
+    admin_session_secret: 'string',
     theme_session_secret: 'string',
     ghost_public_key: 'string',
     ghost_private_key: 'string',
@@ -11,7 +11,7 @@ const keyTypeMapping = {
     cover_image: 'string',
     icon: 'string',
     accent_color: 'string',
-    lang: 'string',
+    locale: 'string',
     timezone: 'string',
     codeinjection_head: 'string',
     codeinjection_foot: 'string',
@@ -32,8 +32,6 @@ const keyTypeMapping = {
     members_private_key: 'string',
     members_email_auth_secret: 'string',
     default_content_visibility: 'string',
-    members_from_address: 'string',
-    stripe_product_name: 'string',
     stripe_secret_key: 'string',
     stripe_publishable_key: 'string',
     stripe_connect_publishable_key: 'string',

package/core/server/data/db/connection.js

@@ -36,19 +36,6 @@ function configure(dbConfig) {
         dbConfig.connection.timezone = 'Z';
         dbConfig.connection.charset = 'utf8mb4';
         dbConfig.connection.decimalNumbers = true;
-
-        // NOTE: disabled so that worker processes can use the db without
-        // requiring logging and causing file desriptor leaks.
-        // See https://github.com/TryGhost/Ghost/issues/12496
-        //
-        // const logging = require('@tryghost/logging');
-        // const errors = require('@tryghost/errors');
-        // dbConfig.connection.loggingHook = function loggingHook(err) {
-        //     logging.error(new errors.InternalServerError({
-        //         code: 'MYSQL_LOGGING_HOOK',
-        //         err: err
-        //     }));
-        // };
     }
 
     return dbConfig;

package/core/server/data/exporter/table-lists.js

@@ -14,7 +14,6 @@ const BACKUP_TABLES = [
     'members_stripe_customers_subscriptions',
     'migrations',
     'migrations_lock',
-    'oauth',
     'permissions',
     'permissions_roles',
     'permissions_users',
@@ -71,8 +70,6 @@ const SETTING_KEYS_BLOCKLIST = [
     'stripe_publishable_key',
     'members_stripe_webhook_id',
     'members_stripe_webhook_secret',
-    'oauth_client_id',
-    'oauth_client_secret',
     'email_verification_required'
 ];
 

package/core/server/data/importer/importers/data/base.js

@@ -304,6 +304,19 @@ class Base {
         });
     }
 
+    /**
+     * @returns {Object}
+     */
+    mapImportedData(originalObject, importedObject) {
+        return {
+            id: importedObject.id,
+            originalId: this.originalIdMap[importedObject.id],
+            slug: importedObject.get('slug'),
+            originalSlug: originalObject.slug,
+            email: importedObject.get('email')
+        };
+    }
+
     doImport(options, importOptions) {
         debug('doImport', this.modelName, this.dataToImport.length);
 
@@ -322,13 +335,9 @@ class Base {
                         }
 
                         // for identifier lookup
-                        this.importedData.push({
-                            id: importedModel.id,
-                            originalId: this.originalIdMap[importedModel.id],
-                            slug: importedModel.get('slug'),
-                            originalSlug: obj.slug,
-                            email: importedModel.get('email')
-                        });
+                        this.importedData.push(
+                            this.mapImportedData(obj, importedModel)
+                        );
 
                         importedModel = null;
                         this.dataToImport.splice(index, 1);

package/core/server/data/importer/importers/data/data-importer.js

@@ -9,6 +9,10 @@ const PostsImporter = require('./posts');
 const TagsImporter = require('./tags');
 const SettingsImporter = require('./settings');
 const UsersImporter = require('./users');
+const NewslettersImporter = require('./newsletters');
+const ProductsImporter = require('./products');
+const StripeProductsImporter = require('./stripe-products');
+const StripePricesImporter = require('./stripe-prices');
 const RolesImporter = require('./roles');
 let importers = {};
 let DataImporter;
@@ -25,8 +29,12 @@ DataImporter = {
         importers.users = new UsersImporter(importData.data);
         importers.roles = new RolesImporter(importData.data);
         importers.tags = new TagsImporter(importData.data);
-        importers.posts = new PostsImporter(importData.data);
+        importers.newsletters = new NewslettersImporter(importData.data);
         importers.settings = new SettingsImporter(importData.data);
+        importers.products = new ProductsImporter(importData.data);
+        importers.stripe_products = new StripeProductsImporter(importData.data);
+        importers.stripe_prices = new StripePricesImporter(importData.data);
+        importers.posts = new PostsImporter(importData.data);
 
         return importData;
     },
@@ -112,6 +120,35 @@ DataImporter = {
                 });
             });
 
+            /**
+             * @TODO: figure out how to fix this properly
+             * fixup the circular reference from
+             * stripe_prices -> stripe_products -> products -> stripe_prices
+             *
+             * Note: the product importer validates that all values are either
+             *   - being imported, or
+             *   - already exist in the db
+             * so we only need to map imported products
+             */
+            ops.push(() => {
+                const importedStripePrices = importers.stripe_prices.importedData;
+                const importedProducts = importers.products.importedData;
+                const productOps = [];
+
+                _.forEach(importedProducts, (importedProduct) => {
+                    return _.forEach(['monthly_price_id', 'yearly_price_id'], (field) => {
+                        const mappedPrice = _.find(importedStripePrices, {originalId: importedProduct[field]});
+                        if (mappedPrice) {
+                            productOps.push(() => {
+                                return models.Product.edit({[field]: mappedPrice.id}, {id: importedProduct.id, transacting});
+                            });
+                        }
+                    });
+                });
+
+                return sequence(productOps);
+            });
+
             return sequence(ops)
                 .then(function () {
                     results.forEach(function (promise) {

package/core/server/data/importer/importers/data/newsletters.js

@@ -0,0 +1,45 @@
+const debug = require('@tryghost/debug')('importer:newsletters');
+const _ = require('lodash');
+const BaseImporter = require('./base');
+const models = require('../../../../models');
+
+const ignoredColumns = ['sender_email'];
+
+class NewslettersImporter extends BaseImporter {
+    constructor(allDataFromFile) {
+        super(allDataFromFile, {
+            modelName: 'Newsletter',
+            dataKeyToImport: 'newsletters'
+        });
+    }
+
+    /**
+    * Remove ignored columns
+    */
+    sanitizeValues() {
+        _.each(this.dataToImport, (obj) => {
+            ignoredColumns.forEach((column) => {
+                delete obj[column];
+            });
+        });
+    }
+
+    fetchExisting(modelOptions) {
+        return models.Newsletter.findAll(_.merge({columns: ['id']}, modelOptions))
+            .then((existingData) => {
+                this.existingData = existingData.toJSON();
+            });
+    }
+
+    beforeImport() {
+        debug('beforeImport');
+        this.sanitizeValues();
+        return super.beforeImport();
+    }
+
+    doImport(options, importOptions) {
+        return super.doImport(options, importOptions);
+    }
+}
+
+module.exports = NewslettersImporter;

package/core/server/data/importer/importers/data/posts.js

@@ -6,16 +6,23 @@ const mobiledocLib = require('../../../../lib/mobiledoc');
 const validator = require('@tryghost/validator');
 const postsMetaSchema = require('../../../schema').tables.posts_meta;
 const metaAttrs = _.keys(_.omit(postsMetaSchema, ['id']));
-const ignoredColumns = ['newsletter_id'];
 
 class PostsImporter extends BaseImporter {
     constructor(allDataFromFile) {
         super(allDataFromFile, {
             modelName: 'Post',
             dataKeyToImport: 'posts',
-            requiredFromFile: ['posts', 'tags', 'posts_tags', 'posts_authors', 'posts_meta'],
-            requiredImportedData: ['tags'],
-            requiredExistingData: ['tags']
+            requiredFromFile: [
+                'posts',
+                'tags',
+                'posts_tags',
+                'posts_authors',
+                'posts_meta',
+                'products',
+                'posts_products'
+            ],
+            requiredImportedData: ['tags', 'products', 'newsletters'],
+            requiredExistingData: ['tags', 'products', 'newsletters']
         });
     }
 
@@ -37,16 +44,11 @@ class PostsImporter extends BaseImporter {
 
             if (_.has(obj, 'send_email_when_published')) {
                 if (obj.send_email_when_published) {
-                    obj.email_recipient_filter = obj.visibility === 'paid' ? 'paid' : 'all';
-                } else {
-                    obj.email_recipient_filter = 'none';
+                    obj.email_recipient_filter = obj.visibility === 'paid' ? 'status:-free' : 'all';
+                    // @TODO: we need to set the newsletter_id to the default newsletter here to have a proper fallback for older imports
                 }
                 delete obj.send_email_when_published;
             }
-
-            ignoredColumns.forEach((column) => {
-                delete obj[column];
-            });
         });
     }
 
@@ -65,15 +67,17 @@ class PostsImporter extends BaseImporter {
     }
 
     /**
-     * Naive function to attach related tags and authors.
+     * Naive function to attach related tags, authors, and products.
      */
     addNestedRelations() {
         this.requiredFromFile.posts_tags = _.orderBy(this.requiredFromFile.posts_tags, ['post_id', 'sort_order'], ['asc', 'asc']);
         this.requiredFromFile.posts_authors = _.orderBy(this.requiredFromFile.posts_authors, ['post_id', 'sort_order'], ['asc', 'asc']);
+        this.requiredFromFile.posts_products = _.orderBy(this.requiredFromFile.posts_products, ['post_id', 'sort_order'], ['asc', 'asc']);
 
         /**
          * from {post_id: 1, tag_id: 2} to post.tags=[{id:id}]
          * from {post_id: 1, author_id: 2} post.authors=[{id:id}]
+         * from {post_id: 1, product_id: 2} post.products=[{id:id}]
          */
         const run = (relations, target, fk) => {
             _.each(relations, (relation) => {
@@ -103,6 +107,7 @@ class PostsImporter extends BaseImporter {
 
         run(this.requiredFromFile.posts_tags, 'tags', 'tag_id');
         run(this.requiredFromFile.posts_authors, 'authors', 'author_id');
+        run(this.requiredFromFile.posts_products, 'tiers', 'product_id');
     }
 
     /**
@@ -180,6 +185,29 @@ class PostsImporter extends BaseImporter {
         _.each(this.dataToImport, (postToImport, postIndex) => {
             run(postToImport, postIndex, 'tags', 'tags');
             run(postToImport, postIndex, 'authors', 'users');
+            run(postToImport, postIndex, 'tiers', 'products');
+        });
+
+        // map newsletter_id -> newsletters.id
+        _.each(this.dataToImport, (objectInFile) => {
+            if (!objectInFile.newsletter_id) {
+                return;
+            }
+            const importedObject = _.find(this.requiredImportedData.newsletters, {originalId: objectInFile.newsletter_id});
+            // CASE: we've imported the newsletter
+            if (importedObject) {
+                debug(`replaced newsletter_id ${objectInFile.newsletter_id} with ${importedObject.id}`);
+                objectInFile.newsletter_id = importedObject.id;
+                return;
+            }
+            const existingObject = _.find(this.requiredExistingData.newsletters, {id: objectInFile.newsletter_id});
+            // CASE: newsletter already exists in the db
+            if (existingObject) {
+                return;
+            }
+            // CASE: newsletter doesn't exist; ignore it
+            debug(`newsletter ${objectInFile.newsletter_id} not found; ignoring`);
+            delete objectInFile.newsletter_id;
         });
 
         return super.replaceIdentifiers();

package/core/server/data/importer/importers/data/products.js

@@ -0,0 +1,68 @@
+const _ = require('lodash');
+const BaseImporter = require('./base');
+const models = require('../../../../models');
+
+class ProductsImporter extends BaseImporter {
+    constructor(allDataFromFile) {
+        super(allDataFromFile, {
+            modelName: 'Product',
+            dataKeyToImport: 'products',
+            requiredFromFile: ['stripe_prices'],
+            requiredExistingData: ['stripe_prices']
+        });
+    }
+
+    fetchExisting(modelOptions) {
+        return models.Product.findAll(_.merge({columns: ['products.id as id']}, modelOptions))
+            .then((existingData) => {
+                this.existingData = existingData.toJSON();
+            });
+    }
+
+    mapImportedData(originalObject, importedObject) {
+        return {
+            id: importedObject.id,
+            originalId: this.originalIdMap[importedObject.id],
+            monthly_price_id: originalObject.monthly_price_id,
+            yearly_price_id: originalObject.yearly_price_id
+        };
+    }
+
+    validateStripePrice() {
+        // the stripe price either needs to exist in the current db,
+        // or be imported as part of the same import
+        let invalidProducts = [];
+        _.each(['monthly_price_id', 'yearly_price_id'], (field) => {
+            _.each(this.dataToImport, (objectInFile) => {
+                const importedObject = _.find(
+                    this.requiredFromFile.stripe_prices,
+                    {id: objectInFile[field]}
+                );
+                // CASE: we'll import the stripe price later
+                if (importedObject) {
+                    return;
+                }
+                const existingObject = _.find(
+                    this.requiredExistingData.stripe_prices,
+                    {id: objectInFile[field]}
+                );
+                // CASE: stripe price already exists in the DB
+                if (existingObject) {
+                    return;
+                }
+                // CASE: we don't know what stripe price this is for
+                invalidProducts.push(objectInFile.id);
+            });
+        });
+        // ignore prices with invalid products
+        this.dataToImport = this.dataToImport.filter(item => !invalidProducts.includes(item.id));
+    }
+
+    replaceIdentifiers() {
+        // this has to be in replaceIdentifiers because it's after required* fields are set
+        this.validateStripePrice();
+        return super.replaceIdentifiers();
+    }
+}
+
+module.exports = ProductsImporter;

package/core/server/data/importer/importers/data/settings.js

@@ -10,15 +10,15 @@ const keyTypeMapper = require('../../../../api/shared/serializers/input/utils/se
 const {WRITABLE_KEYS_ALLOWLIST} = require('../../../../../shared/labs');
 
 const labsDefaults = JSON.parse(defaultSettings.labs.labs.defaultValue);
-const ignoredSettings = ['slack_url', 'members_from_address', 'members_support_address'];
-
-// NOTE: drop support in Ghost 5.0
-const deprecatedSupportedSettingsMap = {
-    default_locale: 'lang',
-    active_timezone: 'timezone',
-    ghost_head: 'codeinjection_head',
-    ghost_foot: 'codeinjection_foot'
+const ignoredSettings = ['slack_url', 'members_from_address', 'members_support_address', 'portal_products'];
+
+// Importer maintains as much backwards compatibility as possible
+const renamedSettingsMap = {
+    default_locale: 'locale',
+    lang: 'locale',
+    active_timezone: 'timezone'
 };
+
 const deprecatedSupportedSettingsOneToManyMap = {
     // NOTE: intentionally ignoring slack_url setting
     slack: [{
@@ -101,8 +101,8 @@ class SettingsImporter extends BaseImporter {
 
         // NOTE: import settings removed in v3 and move them to ignored once Ghost v4 changes are done
         this.dataToImport = this.dataToImport.map((data) => {
-            if (deprecatedSupportedSettingsMap[data.key]) {
-                data.key = deprecatedSupportedSettingsMap[data.key];
+            if (renamedSettingsMap[data.key]) {
+                data.key = renamedSettingsMap[data.key];
             }
 
             return data;

package/core/server/data/importer/importers/data/stripe-prices.js

@@ -0,0 +1,59 @@
+const _ = require('lodash');
+const debug = require('@tryghost/debug')('importer:stripeprices');
+const BaseImporter = require('./base');
+const models = require('../../../../models');
+
+class StripePricesImporter extends BaseImporter {
+    constructor(allDataFromFile) {
+        super(allDataFromFile, {
+            modelName: 'StripePrice',
+            dataKeyToImport: 'stripe_prices',
+            requiredImportedData: ['stripe_products'],
+            requiredExistingData: ['stripe_products']
+        });
+    }
+
+    fetchExisting(modelOptions) {
+        return models.StripePrice.findAll(_.merge({columns: ['id', 'stripe_product_id']}, modelOptions))
+            .then((existingData) => {
+                this.existingData = existingData.toJSON();
+            });
+    }
+
+    validateStripeProduct() {
+        // ensure we have a valid stripe_product_id in the stripe_products table
+        let invalidPrices = [];
+        _.each(this.dataToImport, (objectInFile) => {
+            const importedObject = _.find(
+                this.requiredImportedData.stripe_products,
+                {stripe_product_id: objectInFile.stripe_product_id}
+            );
+            // CASE: we've imported the stripe_product
+            if (importedObject) {
+                return;
+            }
+            const existingObject = _.find(
+                this.requiredExistingData.stripe_products,
+                {stripe_product_id: objectInFile.stripe_product_id}
+            );
+            // CASE: stripe product already exists in the DB
+            if (existingObject) {
+                return;
+            }
+            // CASE: we don't know what stripe product this is for
+            debug(`ignoring invalid product ${objectInFile.stripe_product_id}`);
+            invalidPrices.push(objectInFile.id);
+        });
+        // ignore prices with invalid products
+        debug(`ignoring ${invalidPrices.length} products`);
+        this.dataToImport = this.dataToImport.filter(item => !invalidPrices.includes(item.id));
+    }
+
+    replaceIdentifiers() {
+        // this has to be in replaceIdentifiers because it's after required* fields are set
+        this.validateStripeProduct();
+        return super.replaceIdentifiers();
+    }
+}
+
+module.exports = StripePricesImporter;

package/core/server/data/importer/importers/data/stripe-products.js

@@ -0,0 +1,61 @@
+const _ = require('lodash');
+const debug = require('@tryghost/debug')('importer:stripeproducts');
+const BaseImporter = require('./base');
+const models = require('../../../../models');
+
+class StripeProductsImporter extends BaseImporter {
+    constructor(allDataFromFile) {
+        super(allDataFromFile, {
+            modelName: 'StripeProduct',
+            dataKeyToImport: 'stripe_products',
+            requiredImportedData: ['products'],
+            requiredExistingData: ['products']
+        });
+    }
+
+    fetchExisting(modelOptions) {
+        return models.StripeProduct.findAll(_.merge({columns: ['id', 'stripe_product_id']}, modelOptions))
+            .then((existingData) => {
+                this.existingData = existingData.toJSON();
+            });
+    }
+
+    mapImportedData(originalObject, importedObject) {
+        return {
+            id: importedObject.id,
+            originalId: this.originalIdMap[importedObject.id],
+            stripe_product_id: originalObject.stripe_product_id
+        };
+    }
+
+    replaceIdentifiers() {
+        debug('replaceIdentifiers');
+
+        // map product_id -> product.id
+        let invalidProducts = [];
+        _.each(this.dataToImport, (objectInFile) => {
+            const importedObject = _.find(this.requiredImportedData.products, {originalId: objectInFile.product_id});
+            // CASE: we've imported the product and need to map the ID
+            if (importedObject) {
+                debug(`replaced product_id ${objectInFile.product_id} with ${importedObject.id}`);
+                objectInFile.product_id = importedObject.id;
+                return;
+            }
+            const existingObject = _.find(this.requiredExistingData.products, {id: objectInFile.product_id});
+            // CASE: the product exists in the db already
+            if (existingObject) {
+                return;
+            }
+            // CASE: we don't know what product this is for
+            debug(`ignoring stripe product ${objectInFile.stripe_product_id}`);
+            invalidProducts.push(objectInFile.id);
+        });
+        // ignore Stripe products without Ghost products
+        debug(`ignoring ${invalidProducts.length} products`);
+        this.dataToImport = this.dataToImport.filter(item => !invalidProducts.includes(item.id));
+
+        return super.replaceIdentifiers();
+    }
+}
+
+module.exports = StripeProductsImporter;

package/core/server/data/migrations/utils/constants.js

@@ -0,0 +1,3 @@
+module.exports = {
+    MIGRATION_USER: 1
+};

package/core/server/data/migrations/utils/index.js

@@ -0,0 +1,10 @@
+module.exports = {
+    ...require('./migrations'),
+    ...require('./permissions'),
+    ...require('./schema'),
+    ...require('./settings'),
+    ...require('./tables'),
+    meta: {
+        MIGRATION_USER: require('./constants').MIGRATION_USER
+    }
+};