ghost 4.48.0 → 5.0.2

package/core/server/data/migrations/utils/tables.js

@@ -0,0 +1,94 @@
+const logging = require('@tryghost/logging');
+const {commands} = require('../../schema');
+const {createIrreversibleMigration, createNonTransactionalMigration} = require('./migrations');
+
+/**
+ * Creates a migrations which will add a new table from schema.js to the database
+ * @param {string} name - table name
+ * @param {Object} tableSpec - copy of table schema definition as defined in schema.js at the moment of writing the migration,
+ * this parameter MUST be present, otherwise @daniellockyer will hunt you down
+ *
+ * @returns {Object} migration object returning config/up/down properties
+ */
+function addTable(name, tableSpec) {
+    return createNonTransactionalMigration(
+        async function up(connection) {
+            const tableExists = await connection.schema.hasTable(name);
+            if (tableExists) {
+                logging.warn(`Skipping adding table: ${name} - table already exists`);
+                return;
+            }
+
+            logging.info(`Adding table: ${name}`);
+            return commands.createTable(name, connection, tableSpec);
+        },
+        async function down(connection) {
+            const tableExists = await connection.schema.hasTable(name);
+            if (!tableExists) {
+                logging.warn(`Skipping dropping table: ${name} - table does not exist`);
+                return;
+            }
+
+            logging.info(`Dropping table: ${name}`);
+            return commands.deleteTable(name, connection);
+        }
+    );
+}
+
+/**
+ * Creates migration which will drop a table
+ *
+ * @param {[string]} names  - names of the tables to drop
+ */
+function dropTables(names) {
+    return createIrreversibleMigration(
+        async function up(connection) {
+            for (const name of names) {
+                const exists = await connection.schema.hasTable(name);
+
+                if (!exists) {
+                    logging.warn(`Skipping dropping table: ${name} - table does not exist`);
+                } else {
+                    logging.info(`Dropping table: ${name}`);
+                    await commands.deleteTable(name, connection);
+                }
+            }
+        }
+    );
+}
+
+/**
+ * Creates a migration which will drop an existing table and then re-add a new table based on provided spec
+ * @param {string} name - table name
+ * @param {Object} tableSpec - copy of table schema definition as defined in schema.js at the moment of writing the migration,
+ * this parameter MUST be present, otherwise @daniellockyer will hunt you down
+ *
+ * @returns {Object} migration object returning config/up/down properties
+ */
+function recreateTable(name, tableSpec) {
+    return createNonTransactionalMigration(
+        async function up(connection) {
+            const exists = await connection.schema.hasTable(name);
+
+            if (!exists) {
+                logging.warn(`Skipping dropping table: ${name} - table does not exist`);
+            } else {
+                logging.info(`Dropping table: ${name}`);
+                await commands.deleteTable(name, connection);
+                logging.info(`Re-adding table: ${name}`);
+                await commands.createTable(name, connection, tableSpec);
+            }
+        },
+        async function down() {
+            // noop: we cannot go back to old table schema
+            logging.warn(`Ignoring rollback for table recreate: ${name}`);
+            return Promise.resolve();
+        }
+    );
+}
+
+module.exports = {
+    addTable,
+    dropTables,
+    recreateTable
+};

package/core/server/data/migrations/versions/1.25/01-final-v1.js

@@ -0,0 +1,2 @@
+const {createFinalMigration} = require('../../utils');
+module.exports = createFinalMigration(1);

package/core/server/data/migrations/versions/1.25/02-noop.js

@@ -0,0 +1,6 @@
+const {createTransactionalMigration} = require('../../utils');
+
+module.exports = createTransactionalMigration(
+    async function up() {},
+    async function down() {}
+);

package/core/server/data/migrations/versions/2.37/01-final-v2.js

@@ -0,0 +1,2 @@
+const {createFinalMigration} = require('../../utils');
+module.exports = createFinalMigration(2);

package/core/server/data/migrations/versions/3.41/01-final-v3.js

@@ -0,0 +1,2 @@
+const {createFinalMigration} = require('../../utils');
+module.exports = createFinalMigration(3);

package/core/server/data/migrations/versions/4.0/22-solve-orphaned-webhooks.js

@@ -1,13 +1,7 @@
 const {createIrreversibleMigration} = require('../../utils');
 const logging = require('@tryghost/logging');
 const ObjectID = require('bson-objectid');
-const crypto = require('crypto');
-
-// createSecret is copied from core/server/models/api-key.js
-const createSecret = (type) => {
-    const bytes = type === 'content' ? 13 : 32;
-    return crypto.randomBytes(bytes).toString('hex');
-};
+const security = require('@tryghost/security');
 
 module.exports = createIrreversibleMigration(
     async function up(knex) {
@@ -54,7 +48,7 @@ module.exports = createIrreversibleMigration(
         const contentKey = {
             id: ObjectID().toHexString(),
             type: 'content',
-            secret: createSecret('content'),
+            secret: security.secret.create('content'),
             role_id: null,
             integration_id: integration.id,
             created_at: now,
@@ -67,7 +61,7 @@ module.exports = createIrreversibleMigration(
         const adminKey = {
             id: ObjectID().toHexString(),
             type: 'admin',
-            secret: createSecret('admin'),
+            secret: security.secret.create('admin'),
             role_id: role.id,
             integration_id: integration.id,
             created_at: now,

package/core/server/data/migrations/versions/4.0/23-regenerate-posts-html.js

@@ -1,7 +1,7 @@
 const logging = require('@tryghost/logging');
 const {createIrreversibleMigration} = require('../../utils');
 const mobiledocLib = require('../../../../lib/mobiledoc');
-const htmlToText = require('html-to-text');
+const htmlToPlaintext = require('../../../../../shared/html-to-plaintext');
 
 module.exports = createIrreversibleMigration(async (knex) => {
     logging.info('Starting re-generation of posts html.');
@@ -45,17 +45,8 @@ module.exports = createIrreversibleMigration(async (knex) => {
                 html: html
             };
 
-            // NOTE: block comes straight from the Post model
-            // https://github.com/TryGhost/Ghost/blob/4.0.0-alpha.2/core/server/models/post.js#L484
             if (html !== post.html || !post.plaintext) {
-                const plaintext = htmlToText.fromString(html, {
-                    wordwrap: 80,
-                    ignoreImage: true,
-                    hideLinkHrefIfSameAsText: true,
-                    preserveNewlines: true,
-                    returnDomByDefault: true,
-                    uppercaseHeadings: false
-                });
+                const plaintext = htmlToPlaintext.excerpt(html);
 
                 if (plaintext !== post.plaintext) {
                     updatedAttrs.plaintext = plaintext;

package/core/server/data/migrations/versions/4.11/02-add-email-verification-required-setting.js

@@ -1,6 +1,6 @@
 const ObjectID = require('bson-objectid');
 const logging = require('@tryghost/logging');
-const {createTransactionalMigration} = require('../../utils.js');
+const {createTransactionalMigration} = require('../../utils');
 
 const MIGRATION_USER = 1;
 

package/core/server/data/migrations/versions/4.12/02-fix-member-statuses.js

@@ -1,6 +1,6 @@
 const logging = require('@tryghost/logging');
 const {chunk: chunkArray} = require('lodash');
-const {createTransactionalMigration} = require('../../utils.js');
+const {createTransactionalMigration} = require('../../utils');
 
 module.exports = createTransactionalMigration(
     async function up(knex) {

package/core/server/data/migrations/versions/4.14/02-fix-free-members-status-events.js

@@ -1,6 +1,6 @@
 const _ = require('lodash');
 const logging = require('@tryghost/logging');
-const {createTransactionalMigration} = require('../../utils.js');
+const {createTransactionalMigration} = require('../../utils');
 
 module.exports = createTransactionalMigration(
     async function up(knex) {

package/core/server/data/migrations/versions/4.22/01-add-is-launch-complete-setting.js

@@ -1,4 +1,4 @@
-const {addSetting} = require('../../utils.js');
+const {addSetting} = require('../../utils');
 
 module.exports = addSetting({
     key: 'editor_is_launch_complete',

package/core/server/data/migrations/versions/4.22/02-update-launch-complete-setting-from-user-data.js

@@ -1,5 +1,5 @@
 const logging = require('@tryghost/logging');
-const {createTransactionalMigration} = require('../../utils.js');
+const {createTransactionalMigration} = require('../../utils');
 
 module.exports = createTransactionalMigration(
     async function up(knex) {

package/core/server/data/migrations/versions/4.35/2022-02-02-10-38-add-default-content-visibility-tiers-setting.js

@@ -1,4 +1,4 @@
-const {addSetting} = require('../../utils.js');
+const {addSetting} = require('../../utils');
 
 module.exports = addSetting({
     key: 'default_content_visibility_tiers',

package/core/server/data/migrations/versions/4.45/2022-04-21-02-55-add-notifications-key-entry-to-settings-table.js

@@ -1,4 +1,4 @@
-const {addSetting} = require('../../utils.js');
+const {addSetting} = require('../../utils');
 
 module.exports = addSetting({
     key: 'version_notifications',

package/core/server/data/migrations/versions/4.6/01-remove-comped-status.js

@@ -1,5 +1,5 @@
 const logging = require('@tryghost/logging');
-const {createTransactionalMigration} = require('../../utils.js');
+const {createTransactionalMigration} = require('../../utils');
 
 module.exports = createTransactionalMigration(
     async function up(knex) {

package/core/server/data/migrations/versions/4.7/03-add-labs-setting.js

@@ -1,6 +1,6 @@
 const ObjectID = require('bson-objectid');
 const logging = require('@tryghost/logging');
-const {createTransactionalMigration} = require('../../utils.js');
+const {createTransactionalMigration} = require('../../utils');
 
 const MIGRATION_USER = 1;
 

package/core/server/data/migrations/versions/4.9/05-fix-missed-mobiledoc-url-transforms.js

@@ -65,7 +65,7 @@ module.exports = createTransactionalMigration(
                     continue;
                 }
 
-                const plaintext = htmlToPlaintext(html);
+                const plaintext = htmlToPlaintext.excerpt(html);
 
                 await knex('posts')
                     .transacting(trx)

package/core/server/data/migrations/versions/4.9/06-add-comped-status.js

@@ -1,5 +1,5 @@
 const logging = require('@tryghost/logging');
-const {createTransactionalMigration} = require('../../utils.js');
+const {createTransactionalMigration} = require('../../utils');
 
 module.exports = createTransactionalMigration(
     async function up(knex) {

package/core/server/data/migrations/versions/4.9/07-update-comped-members-status-events.js

@@ -1,5 +1,5 @@
 const logging = require('@tryghost/logging');
-const {createTransactionalMigration} = require('../../utils.js');
+const {createTransactionalMigration} = require('../../utils');
 
 module.exports = createTransactionalMigration(
     async function up(knex) {

package/core/server/data/migrations/versions/5.0/2022-03-14-12-33-delete-duplicate-offer-redemptions.js

@@ -0,0 +1,36 @@
+const DatabaseInfo = require('@tryghost/database-info/lib/database-info');
+const logging = require('@tryghost/logging');
+
+const {createTransactionalMigration} = require('../../utils');
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        if (DatabaseInfo.isSQLite(knex)) {
+            const duplicates = await knex('offer_redemptions')
+                .select('subscription_id')
+                .count('subscription_id as count')
+                .groupBy('subscription_id')
+                .having('count', '>', 1);
+
+            logging.info(`Deleting all offer redemptions which have duplicates`);
+            await knex('offer_redemptions')
+                .whereIn('subscription_id', duplicates.map(row => row.subscription_id))
+                .del();
+            return;
+        }
+        const result = await knex.raw(`
+            DELETE
+                duplicate_redemptions
+            FROM
+                offer_redemptions AS duplicate_redemptions,
+                offer_redemptions
+            WHERE
+                duplicate_redemptions.subscription_id = offer_redemptions.subscription_id
+            AND
+                duplicate_redemptions.created_at < offer_redemptions.created_at
+        `);
+
+        logging.info(`Deleted ${result[0].affectedRows} duplicate offer redemptions`);
+    },
+    async function down() {}
+);

package/core/server/data/migrations/versions/5.0/2022-03-28-15-25-backfill-mrr-adjustments-for-offers.js

@@ -0,0 +1,108 @@
+const logging = require('@tryghost/logging');
+const {uniq} = require('lodash');
+
+const {createTransactionalMigration} = require('../../utils');
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        logging.info('Adjusting MRR based on Offer Redemptions');
+        const offerRedemptions = await knex
+            .select('or.*', 'o.discount_type', 'o.discount_amount', 'o.interval AS discount_interval', 's.mrr AS mrr', 's.id AS subscription_id', 'p.amount AS amount', 'p.interval AS interval')
+            .from('offer_redemptions AS or')
+            .join('offers AS o', 'or.offer_id', '=', 'o.id')
+            .join('members_stripe_customers_subscriptions AS s', 'or.subscription_id', '=', 's.id')
+            .join('stripe_prices AS p', 'p.stripe_price_id', '=', 's.stripe_price_id')
+            .where('o.duration', '=', 'forever')
+            .whereNotNull('s.offer_id')
+            .where('s.mrr', '!=', 0)
+            .orderBy('or.created_at', 'asc');
+
+        if (offerRedemptions.length === 0) {
+            logging.info('No Offers redeemed, skipping migration');
+            return;
+        } else {
+            logging.info(`Adjusting MRR for ${offerRedemptions.length} Offer Redemptions`);
+        }
+
+        const memberIds = uniq(offerRedemptions.map(redemption => redemption.member_id));
+
+        const mrrCreatedEvents = await knex
+            .select('*')
+            .from('members_paid_subscription_events')
+            .where('type', 'created')
+            .whereIn('member_id', memberIds);
+
+        function storeEventOnMemberId(storage, event) {
+            return {
+                ...storage,
+                [event.member_id]: storage[event.member_id] ? storage[event.member_id].concat(event) : [event]
+            };
+        }
+
+        const mrrCreatedEventsByMemberId = mrrCreatedEvents.reduce(storeEventOnMemberId, {});
+
+        const updatedEvents = [];
+
+        function calculateMRR(subscription, redemption) {
+            if (redemption && subscription.interval !== redemption.discount_interval) {
+                logging.error('Found invalid price & redemption pair');
+                return calculateMRR(subscription);
+            }
+
+            if (!redemption) {
+                return subscription.interval === 'year' ? subscription.amount / 12 : subscription.amount;
+            }
+
+            if (redemption.discount_type === 'percent') {
+                return calculateMRR({
+                    interval: subscription.interval,
+                    amount: subscription.amount * (100 - redemption.discount_amount) / 100
+                });
+            }
+
+            return calculateMRR({
+                interval: subscription.interval,
+                amount: subscription.amount - redemption.discount_amount
+            });
+        }
+
+        // eslint-disable-next-line no-restricted-syntax
+        offerRedemptions.forEach((redemption) => {
+            const memberEvents = mrrCreatedEventsByMemberId[redemption.member_id];
+
+            // If a member has had multiple subscriptions we ignore because we cannot easily work out which event is correct.
+            if (memberEvents.length !== 1) {
+                return;
+            }
+
+            const firstEvent = memberEvents[0];
+
+            const mrr = calculateMRR({
+                interval: redemption.interval,
+                amount: redemption.amount
+            }, redemption);
+            updatedEvents.push({
+                id: firstEvent.id,
+                type: 'created',
+                subscription_id: redemption.subscription_id,
+                member_id: firstEvent.member_id,
+                from_plan: firstEvent.from_plan,
+                to_plan: firstEvent.to_plan,
+                currency: firstEvent.currency,
+                source: firstEvent.source,
+                created_at: firstEvent.created_at,
+                mrr_delta: mrr
+            });
+        });
+
+        if (updatedEvents.length === 0) {
+            return;
+        }
+
+        const idsToDelete = updatedEvents.map(event => event.id);
+
+        await knex('members_paid_subscription_events').whereIn('id', idsToDelete).del();
+        await knex.batchInsert('members_paid_subscription_events', updatedEvents);
+    },
+    async function down() {}
+);

package/core/server/data/migrations/versions/5.0/2022-04-25-10-32-backfill-mrr-for-discounted-subscriptions.js

@@ -0,0 +1,44 @@
+const logging = require('@tryghost/logging');
+
+const {createTransactionalMigration} = require('../../utils');
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        const subscriptionsToUpdate = await knex('members_stripe_customers_subscriptions AS s')
+            .join('offers AS o', 's.offer_id', '=', 'o.id')
+            .where('o.duration', '=', 'forever')
+            .andWhere('s.mrr', '!=', 0)
+            .select('s.*', 'o.discount_type AS offer_type', 'o.discount_amount AS offer_amount');
+
+        if (!subscriptionsToUpdate.length) {
+            logging.info('No subscriptions found needing updating');
+            return;
+        }
+
+        const toInsert = subscriptionsToUpdate.map((subscription) => {
+            let discountedAmount;
+            if (subscription.offer_type === 'percent') {
+                discountedAmount = subscription.plan_amount * (100 - subscription.offer_amount) / 100;
+            } else {
+                discountedAmount = subscription.plan_amount - subscription.offer_amount;
+            }
+
+            const newSubscription = {
+                ...subscription,
+                mrr: subscription.plan_interval === 'year' ? discountedAmount / 12 : discountedAmount
+            };
+
+            delete newSubscription.offer_type;
+            delete newSubscription.offer_amount;
+
+            return newSubscription;
+        });
+
+        const toDelete = toInsert.map(sub => sub.id);
+
+        logging.info(`Replacing ${toDelete.length} subscriptions with updated MRR based on Offers`);
+        await knex('members_stripe_customers_subscriptions').whereIn('id', toDelete).del();
+        await knex.batchInsert('members_stripe_customers_subscriptions', toInsert);
+    },
+    async function down() {}
+);

package/core/server/data/migrations/versions/5.0/2022-04-26-15-44-backfill-mrr-events-for-canceled-subscriptions.js

@@ -0,0 +1,73 @@
+const ObjectID = require('bson-objectid').default;
+const logging = require('@tryghost/logging');
+
+const {createTransactionalMigration} = require('../../utils');
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        // eslint-disable-next-line no-restricted-syntax
+        const canceledSubscriptions = await knex('members_stripe_customers_subscriptions')
+            .select(
+                'members_stripe_customers_subscriptions.id',
+                'members_stripe_customers_subscriptions.updated_at',
+                'members_stripe_customers_subscriptions.mrr',
+                'members_stripe_customers_subscriptions.stripe_price_id',
+                'members_stripe_customers_subscriptions.plan_currency',
+                'members_stripe_customers.member_id'
+            )
+            .join('members_stripe_customers', 'members_stripe_customers_subscriptions.customer_id', '=', 'members_stripe_customers.customer_id')
+            .join('members', 'members_stripe_customers.member_id', '=', 'members.id')
+            .where('cancel_at_period_end', '=', true)
+            .where('mrr', '!=', 0);
+
+        if (canceledSubscriptions.length === 0) {
+            logging.info('No canceled subscriptions found, skipping migration.');
+            return;
+        } else {
+            logging.info(`Found ${canceledSubscriptions.length} canceled subscriptions, updating MRR events`);
+        }
+
+        const canceledEvents = await knex('members_paid_subscription_events')
+            .select('*')
+            .where('type', '=', 'canceled')
+            .whereIn('subscription_id', canceledSubscriptions.map(x => x.id))
+            .orderBy('created_at', 'desc');
+
+        const toUpdate = [];
+        const toInsert = [];
+
+        // eslint-disable-next-line no-restricted-syntax
+        for (const subscription of canceledSubscriptions) {
+            const event = canceledEvents.find(e => e.subscription_id === subscription.id);
+            if (event) {
+                // if an event exists, update it
+                // we always update the latest event for a subscription due to the orderBy DESC
+                toUpdate.push({
+                    ...event,
+                    mrr_delta: -subscription.mrr
+                });
+            } else {
+                toInsert.push({
+                    id: ObjectID().toHexString(),
+                    type: 'canceled',
+                    source: 'migration',
+                    created_at: subscription.updated_at,
+                    from_plan: subscription.stripe_price_id,
+                    to_plan: subscription.stripe_price_id,
+                    subscription_id: subscription.id,
+                    member_id: subscription.member_id,
+                    currency: subscription.plan_currency,
+                    mrr_delta: -subscription.mrr
+                });
+            }
+        }
+
+        logging.info(`Inserting ${toInsert.length} MRR events for canceled subscriptions`);
+        await knex.batchInsert('members_paid_subscription_events', toInsert);
+
+        logging.info(`Updating ${toUpdate.length} MRR events for canceled subscriptions`);
+        await knex('members_paid_subscription_events').whereIn('id', toUpdate.map(row => row.id)).del();
+        await knex.batchInsert('members_paid_subscription_events', toUpdate);
+    },
+    async function down() {}
+);

package/core/server/data/migrations/versions/5.0/2022-04-27-11-26-backfill-mrr-for-canceled-subscriptions.js

@@ -0,0 +1,31 @@
+const logging = require('@tryghost/logging');
+
+const {createTransactionalMigration} = require('../../utils');
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        logging.info('Setting "mrr" to 0 for all canceled subscriptions in "members_stripe_customers_subscriptions"');
+
+        await knex('members_stripe_customers_subscriptions')
+            .update('mrr', 0)
+            .where('cancel_at_period_end', true);
+    },
+    async function down(knex) {
+        logging.info('Setting "mrr" for all canceled and not yet expired subscriptions in "members_stripe_customers_subscriptions"');
+
+        await knex('members_stripe_customers_subscriptions')
+            .update('mrr', knex.raw(`
+                CASE WHEN plan_interval = 'year' THEN
+                    FLOOR(plan_amount / 12)
+                WHEN plan_interval = 'week' THEN
+                    plan_amount * 4
+                WHEN plan_interval = 'day' THEN
+                    plan_amount * 30
+                ELSE 
+                    plan_amount
+                END
+            `))
+            .where('cancel_at_period_end', true)
+            .whereNotIn('status', ['trialing', 'incomplete', 'incomplete_expired', 'canceled']);
+    }
+);

package/core/server/data/migrations/versions/5.0/2022-04-28-03-26-remove-author-id-column-from-posts-table.js

@@ -0,0 +1,7 @@
+const {createDropColumnMigration} = require('../../utils');
+
+module.exports = createDropColumnMigration('posts', 'author_id', {});
+
+module.exports.down = async () => {
+    // noop - major version migrations are not backwards compatible
+};

package/core/server/data/migrations/versions/5.0/2022-05-03-09-39-drop-nullable-subscribe-event-newsletter-id.js

@@ -0,0 +1,4 @@
+const {createDropNullableMigration} = require('../../utils');
+
+// We need to disable foreign key checks because if MySQL is missing the STRICT_TRANS_TABLES mode, we cannot drop nullable from a foreign key
+module.exports = createDropNullableMigration('members_subscribe_events', 'newsletter_id', {disableForeignKeyChecks: true});

package/core/server/data/migrations/versions/5.0/2022-05-04-15-24-map-existing-emails-to-default-newsletter.js

@@ -0,0 +1,44 @@
+const logging = require('@tryghost/logging');
+const {createTransactionalMigration} = require('../../utils');
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        logging.info('Linking existing emails and related posts to the default newsletter');
+
+        // Get the default newsletter
+        // Note we intentionally use the default newsletter slug instead of the usual orderBy logic
+        let newsletter = await knex('newsletters')
+            .where('slug', 'default-newsletter')
+            .first('id', 'slug');
+
+        if (!newsletter) {
+            // Fall back to orderBy - just in case
+            logging.warn(`Original default newsletter not found - using first in sort order`);
+            newsletter = await knex('newsletters')
+                .where('status', 'active')
+                .orderBy('sort_order', 'asc')
+                .orderBy('created_at', 'asc')
+                .orderBy('id', 'asc')
+                .first('id', 'slug');
+        }
+
+        if (!newsletter) {
+            logging.error(`Newsletter not found - skipping`);
+            return;
+        }
+
+        logging.info(`Assigning existing emails to newsletter ID ${newsletter.id} (${newsletter.slug})`);
+
+        // Set newsletter ID only on posts with related email records without a newsletter assigned
+        await knex('posts')
+            .update('newsletter_id', newsletter.id)
+            .whereIn('id', knex.raw('SELECT post_id FROM emails WHERE emails.newsletter_id IS NULL'));
+
+        await knex('emails')
+            .update('newsletter_id', newsletter.id)
+            .whereNull('newsletter_id');
+    },
+    async function down() {
+        // Not required
+    }
+);