ghost 4.42.1 → 4.44.0

package/core/server/models/post.js

@@ -557,7 +557,7 @@ Post = ghostBookshelf.Model.extend({
                 if (!tag.id && !tag.tag_id && tag.slug) {
                     // Clean up the provided slugs before we do any matching with existing tags
                     tag.slug = await ghostBookshelf.Model.generateSlug(
-                        Tag, 
+                        Tag,
                         tag.slug,
                         {skipDuplicateChecks: true}
                     );
@@ -674,6 +674,13 @@ Post = ghostBookshelf.Model.extend({
             }
         }
 
+        // newsletter_id is read-only and should only be set using a query param when publishing/scheduling
+        if (options.newsletter_id
+            && this.hasChanged('status')
+            && (newStatus === 'published' || newStatus === 'scheduled')) {
+            this.set('newsletter_id', options.newsletter_id);
+        }
+
         // email_recipient_filter is read-only and should only be set using a query param when publishing/scheduling
         if (options.email_recipient_filter
             && (options.email_recipient_filter !== 'none')
@@ -878,6 +885,9 @@ Post = ghostBookshelf.Model.extend({
         // CASE: never expose the revisions
         delete attrs.mobiledoc_revisions;
 
+        // CASE: hide the newsletter_id for now
+        delete attrs.newsletter_id;
+
         // If the current column settings allow it...
         if (!options.columns || (options.columns && options.columns.indexOf('primary_tag') > -1)) {
             // ... attach a computed property of primary_tag which is the first tag if it is public, else null
@@ -1016,7 +1026,7 @@ Post = ghostBookshelf.Model.extend({
             findPage: ['status'],
             findAll: ['columns', 'filter'],
             destroy: ['destroyAll', 'destroyBy'],
-            edit: ['filter', 'email_recipient_filter', 'force_rerender']
+            edit: ['filter', 'email_recipient_filter', 'force_rerender', 'newsletter_id']
         };
 
         // The post model additionally supports having a formats option

package/core/server/models/stripe-customer-subscription.js

@@ -4,6 +4,10 @@ const _ = require('lodash');
 const StripeCustomerSubscription = ghostBookshelf.Model.extend({
     tableName: 'members_stripe_customers_subscriptions',
 
+    defaults: {
+        mrr: 0
+    },
+
     customer() {
         return this.belongsTo('MemberStripeCustomer', 'customer_id', 'customer_id');
     },

package/core/server/services/auth/setup.js

@@ -107,7 +107,9 @@ async function doSettings(data, settingsAPI) {
     return user;
 }
 
-async function doProduct(data, productsAPI) {
+// Update names for default product and newsletter to site title
+async function doProductAndNewsletter(data, api) {
+    const {products: productsAPI, newsletters: newslettersAPI} = api;
     const context = {context: {user: data.user.id}};
     const user = data.user;
     const blogTitle = data.userData.blogTitle;
@@ -116,15 +118,23 @@ async function doProduct(data, productsAPI) {
         return user;
     }
     try {
-        const page = await productsAPI.browse({limit: 'all'});
+        const productPage = await productsAPI.browse({limit: 'all'});
+        const newsletterPage = await newslettersAPI.browse({limit: 'all'});
 
-        const product = page.products.find(p => p.slug === 'default-product');
+        const defaultProduct = productPage.products.find(p => p.slug === 'default-product');
+        const defaultNewsletter = newsletterPage.newsletters.find(p => p.slug === 'default-newsletter');
 
-        if (!product) {
-            return data;
+        if (defaultProduct) {
+            await productsAPI.edit({products: [{
+                name: blogTitle.trim()
+            }]}, {context: context.context, id: defaultProduct.id});
         }
 
-        await productsAPI.edit({products: [{name: blogTitle.trim()}]}, {context: context.context, id: product.id});
+        if (defaultNewsletter) {
+            await newslettersAPI.edit({newsletters: [{
+                name: blogTitle.trim()
+            }]}, {context: context.context, id: defaultNewsletter.id});
+        }
     } catch (e) {
         return data;
     }
@@ -142,7 +152,10 @@ async function doFixtures(data) {
         mobiledoc = mobiledoc.replace(/{{date}}/, date);
 
         const post = await models.Post.findOne({slug: key});
-        await models.Post.edit({mobiledoc}, {id: post.id});
+
+        if (post) {
+            await models.Post.edit({mobiledoc}, {id: post.id});
+        }
     });
 
     return data;
@@ -218,7 +231,7 @@ module.exports = {
     assertSetupCompleted: assertSetupCompleted,
     setupUser: setupUser,
     doSettings: doSettings,
-    doProduct: doProduct,
+    doProductAndNewsletter: doProductAndNewsletter,
     installTheme: installTheme,
     doFixtures: doFixtures,
     sendWelcomeEmail: sendWelcomeEmail

package/core/server/services/mega/mega.js

@@ -159,6 +159,7 @@ const transformEmailRecipientFilter = (emailRecipientFilter, {errorProperty = 'e
  * @param {object} postModel Post Model Object
  * @param {object} options
  * @param {ValidAPIVersion} options.apiVersion - api version to be used when serializing email data
+ * @param {string} options.newsletter_id - the newsletter_id to send the email to
  */
 
 const addEmail = async (postModel, options) => {
@@ -211,7 +212,8 @@ const addEmail = async (postModel, options) => {
             plaintext: emailData.plaintext,
             submitted_at: moment().toDate(),
             track_opens: !!settingsCache.get('email_track_opens'),
-            recipient_filter: emailRecipientFilter
+            recipient_filter: emailRecipientFilter,
+            newsletter_id: options.newsletter_id
         }, knexOptions);
     } else {
         return existing;

package/core/server/services/members/api.js

@@ -13,6 +13,7 @@ const SingleUseTokenProvider = require('./SingleUseTokenProvider');
 const urlUtils = require('../../../shared/url-utils');
 const labsService = require('../../../shared/labs');
 const offersService = require('../offers');
+const getNewslettersServiceInstance = require('../newsletters');
 
 const MAGIC_LINK_TOKEN_VALIDITY = 24 * 60 * 60 * 1000;
 
@@ -195,7 +196,8 @@ function createApiInstance(config) {
         },
         stripeAPIService: stripeService.api,
         offersAPI: offersService.api,
-        labsService: labsService
+        labsService: labsService,
+        newslettersService: getNewslettersServiceInstance({NewsletterModel: models.Newsletter})
     });
 
     return membersApiInstance;

package/core/server/services/members/middleware.js

@@ -70,12 +70,12 @@ const getOfferData = async function (req, res) {
 
 const updateMemberData = async function (req, res) {
     try {
-        const data = _.pick(req.body, 'name', 'subscribed');
+        const data = _.pick(req.body, 'name', 'subscribed', 'newsletters');
         const member = await membersService.ssr.getMemberDataFromSession(req, res);
         if (member) {
             const options = {
                 id: member.id,
-                withRelated: ['stripeSubscriptions', 'stripeSubscriptions.customer', 'stripeSubscriptions.stripePrice']
+                withRelated: ['stripeSubscriptions', 'stripeSubscriptions.customer', 'stripeSubscriptions.stripePrice', 'newsletters']
             };
             const updatedMember = await membersService.api.members.update(data, options);
 
@@ -133,6 +133,11 @@ const getPortalProductPrices = async function () {
     };
 };
 
+const getSiteNewsletters = async function () {
+    const newsletters = await models.Newsletter.findAll();
+    return newsletters.toJSON();
+};
+
 const getMemberSiteData = async function (req, res) {
     const isStripeConfigured = membersService.config.isStripeConnected();
     const domain = urlUtils.urlFor('home', true).match(new RegExp('^https?://([^/:?#]+)(?:[/:?#]|$)', 'i'));
@@ -144,7 +149,7 @@ const getMemberSiteData = async function (req, res) {
     }
     const {products = [], prices = []} = await getPortalProductPrices() || {};
     const portalVersion = config.get('portal:version');
-
+    const newsletters = await getSiteNewsletters();
     const response = {
         title: settingsCache.get('title'),
         description: settingsCache.get('description'),
@@ -170,6 +175,11 @@ const getMemberSiteData = async function (req, res) {
         prices,
         products
     };
+
+    if (labsService.isSet('multipleNewsletters')) {
+        response.newsletters = newsletters;
+    }
+
     if (labsService.isSet('multipleProducts')) {
         response.portal_products = settingsCache.get('portal_products');
     }

package/core/server/services/members/service.js

@@ -58,17 +58,9 @@ const membersImporter = new MembersCSVImporter({
 
 const processImport = async (options) => {
     const result = await membersImporter.process(options);
-    const importSize = result.meta.originalImportSize;
-    delete result.meta.originalImportSize;
-
-    const importThreshold = await verificationTrigger.getImportThreshold();
-    if (importSize > importThreshold) {
-        await verificationTrigger.startVerificationProcess({
-            amountImported: importSize,
-            throwOnTrigger: true,
-            source: 'import'
-        });
-    }
+
+    // Check whether all imports in last 30 days > threshold
+    await verificationTrigger.testImportThreshold();
 
     return result;
 };

package/core/server/services/members/utils.js

@@ -1,8 +1,16 @@
+const labsService = require('../../../shared/labs');
+
+function formatNewsletterResponse(newsletters) {
+    return newsletters.map(({id, name, description, sort_order: sortOrder}) => {
+        return {id, name, description, sort_order: sortOrder};
+    });
+}
+
 module.exports.formattedMemberResponse = function formattedMemberResponse(member) {
     if (!member) {
         return null;
     }
-    return {
+    const data = {
         uuid: member.uuid,
         email: member.email,
         name: member.name,
@@ -12,4 +20,8 @@ module.exports.formattedMemberResponse = function formattedMemberResponse(member
         subscriptions: member.subscriptions || [],
         paid: member.status !== 'free'
     };
+    if (member.newsletters && labsService.isSet('multipleNewsletters')) {
+        data.newsletters = formatNewsletterResponse(member.newsletters);
+    }
+    return data;
 };

package/core/server/services/newsletters/index.js

@@ -0,0 +1,10 @@
+const NewslettersService = require('./service.js');
+
+/**
+ * @returns {NewslettersService} instance of the NewslettersService
+ */
+const getNewslettersServiceInstance = ({NewsletterModel}) => {
+    return new NewslettersService({NewsletterModel});
+};
+
+module.exports = getNewslettersServiceInstance;

package/core/server/services/newsletters/service.js

@@ -0,0 +1,24 @@
+class NewslettersService {
+    /**
+     *
+     * @param {Object} options
+     * @param {Object} options.NewsletterModel
+     */
+    constructor({NewsletterModel}) {
+        this.NewsletterModel = NewsletterModel;
+    }
+
+    /**
+     *
+     * @param {Object} options browse options
+     * @returns
+     */
+    async browse(options) {
+        let newsletters = await this.NewsletterModel.findAll(options);
+
+        return newsletters.toJSON();
+    }
+}
+
+module.exports = NewslettersService;
+

package/core/server/services/posts/posts-service.js

@@ -4,7 +4,8 @@ const tpl = require('@tryghost/tpl');
 
 const messages = {
     invalidEmailRecipientFilter: 'Invalid filter in email_recipient_filter param.',
-    invalidVisibilityFilter: 'Invalid visibility filter.'
+    invalidVisibilityFilter: 'Invalid visibility filter.',
+    invalidNewsletterId: 'The newsletter_id parameter doesn\'t match any active newsletter.'
 };
 
 class PostsService {
@@ -19,6 +20,16 @@ class PostsService {
     async editPost(frame) {
         let model;
 
+        // Make sure the newsletter_id is matching an active newsletter
+        if (frame.options.newsletter_id) {
+            const newsletter = await this.models.Newsletter.findOne({id: frame.options.newsletter_id, status: 'active'}, {transacting: frame.options.transacting});
+            if (!newsletter) {
+                throw new BadRequestError({
+                    message: messages.invalidNewsletterId
+                });
+            }
+        }
+
         if (!frame.options.email_recipient_filter && frame.options.send_email_when_published) {
             await this.models.Base.transaction(async (transacting) => {
                 const options = {
@@ -65,6 +76,14 @@ class PostsService {
             const sendEmail = model.wasChanged() && this.shouldSendEmail(model.get('status'), model.previous('status'));
 
             if (sendEmail) {
+                // Set the newsletter_id if it isn't passed to the API
+                if (!frame.options.newsletter_id) {
+                    const newsletters = await this.models.Newsletter.findPage({status: 'active', limit: 1, columns: ['id']}, {transacting: frame.options.transacting});
+                    if (newsletters.data.length > 0) {
+                        frame.options.newsletter_id = newsletters.data[0].id;
+                    }
+                }
+
                 let postEmail = model.relations.email;
 
                 if (!postEmail) {

package/core/server/services/slack.js

@@ -6,7 +6,6 @@ const {blogIcon} = require('../lib/image');
 const urlUtils = require('../../shared/url-utils');
 const urlService = require('./url');
 const settingsCache = require('../../shared/settings-cache');
-const schema = require('../data/schema').checks;
 const moment = require('moment');
 
 // Used to receive post.published model event, but also the slack.test event from the API which iirc this was done to avoid circular deps a long time ago
@@ -37,6 +36,15 @@ function getSlackSettings() {
     };
 }
 
+/**
+ * @TODO: change this function to check for the properties we depend on
+ * @param {Object} data
+ * @returns {boolean}
+ */
+function hasPostProperties(data) {
+    return Object.prototype.hasOwnProperty.call(data, 'html') && Object.prototype.hasOwnProperty.call(data, 'title') && Object.prototype.hasOwnProperty.call(data, 'slug');
+}
+
 function ping(post) {
     let message;
     let title;
@@ -47,7 +55,7 @@ function ping(post) {
     let blogTitle = settingsCache.get('title');
 
     // If this is a post, we want to send the link of the post
-    if (schema.isPost(post)) {
+    if (hasPostProperties(post)) {
         message = urlService.getUrlByResourceId(post.id, {absolute: true});
         title = post.title ? post.title : null;
         author = post.authors ? post.authors[0] : null;
@@ -79,7 +87,7 @@ function ping(post) {
             return;
         }
 
-        if (schema.isPost(post)) {
+        if (hasPostProperties(post)) {
             slackData = {
                 // We are handling the case of test notification here by checking
                 // if it is a post or a test message to check webhook working.

package/core/server/services/stats/lib/members-stats-service.js

@@ -1,4 +1,4 @@
-const {DateTime} = require('luxon');
+const moment = require('moment');
 
 class MembersStatsService {
     constructor({db}) {
@@ -28,8 +28,8 @@ class MembersStatsService {
     }
 
     /**
-     * Get the member deltas by status for all days (from new to old)
-     * @returns {Promise<MemberStatusDelta[]>} The deltas of paid, free and comped users per day, sorted from new to old
+     * Get the member deltas by status for all days, sorted ascending
+     * @returns {Promise<MemberStatusDelta[]>} The deltas of paid, free and comped users per day, sorted ascending
      */
     async fetchAllStatusDeltas() {
         const knex = this.db.knex;
@@ -54,7 +54,7 @@ class MembersStatsService {
                 ELSE 0 END
             ) as free_delta`))
             .groupByRaw('DATE(created_at)')
-            .orderByRaw('DATE(created_at) DESC');
+            .orderByRaw('DATE(created_at)');
         return rows;
     }
 
@@ -69,22 +69,26 @@ class MembersStatsService {
         const totals = await this.getCount();
         let {paid, free, comped} = totals;
 
-        // Get today in UTC (default timezone for Luxon)
-        const today = DateTime.local().toISODate();
+        // Get today in UTC (default timezone)
+        const today = moment().format('YYYY-MM-DD');
 
         const cumulativeResults = [];
-        for (const row of rows) {
+
+        // Loop in reverse order (needed to have correct sorted result)
+        for (let i = rows.length - 1; i >= 0; i -= 1) {
+            const row = rows[i];
+
             // Convert JSDates to YYYY-MM-DD (in UTC)
-            const date = DateTime.fromJSDate(row.date).toISODate();
+            const date = moment(row.date).format('YYYY-MM-DD');
             if (date > today) {
                 // Skip results that are in the future (fix for invalid events)
                 continue;
             }
             cumulativeResults.unshift({
                 date,
-                paid,
-                free,
-                comped,
+                paid: Math.max(0, paid),
+                free: Math.max(0, free),
+                comped: Math.max(0, comped),
 
                 // Deltas
                 paid_subscribed: row.paid_subscribed,
@@ -92,36 +96,28 @@ class MembersStatsService {
             });
 
             // Update current counts
-            paid = Math.max(0, paid - row.paid_subscribed + row.paid_canceled);
-            free = Math.max(0, free - row.free_delta);
-            comped = Math.max(0, comped - row.comped_delta);
+            paid -= row.paid_subscribed - row.paid_canceled;
+            free -= row.free_delta;
+            comped -= row.comped_delta;
         }
 
-        // Always make sure we have at least one result
-        if (cumulativeResults.length === 0) {
-            cumulativeResults.push({
-                date: today,
-                paid,
-                free,
-                comped,
+        // Now also add the oldest day we have left over (this one will be zero, which is also needed as a data point for graphs)
+        const oldestDate = rows.length > 0 ? moment(rows[0].date).add(-1, 'days').format('YYYY-MM-DD') : today;
 
-                // Deltas
-                paid_subscribed: 0,
-                paid_canceled: 0
-            });
-        }
+        cumulativeResults.unshift({
+            date: oldestDate,
+            paid: Math.max(0, paid),
+            free: Math.max(0, free),
+            comped: Math.max(0, comped),
+
+            // Deltas
+            paid_subscribed: 0,
+            paid_canceled: 0
+        });
 
         return {
             data: cumulativeResults,
             meta: {
-                pagination: {
-                    page: 1,
-                    limit: 'all',
-                    pages: 1,
-                    total: cumulativeResults.length,
-                    next: null,
-                    prev: null
-                },
                 totals
             }
         };

package/core/server/services/stats/lib/mrr-stats-service.js

@@ -0,0 +1,154 @@
+const moment = require('moment');
+
+class MrrStatsService {
+    constructor({db}) {
+        this.db = db;
+    }
+
+    /**
+     * Get the current total MRR, grouped by currency (ascending order)
+     * @returns {Promise<MrrByCurrency[]>}
+     */
+    async getCurrentMrr() {
+        const knex = this.db.knex;
+        const rows = await knex('members_stripe_customers_subscriptions')
+            .select(knex.raw(`plan_currency as currency`))
+            .select(knex.raw(`SUM(
+                CASE WHEN plan_interval = 'year' THEN
+                    FLOOR(plan_amount / 12)
+                ELSE 
+                    plan_amount
+                END
+            ) AS mrr`))
+            .whereIn('status', ['active', 'unpaid', 'past_due'])
+            .where('cancel_at_period_end', 0)
+            .groupBy('plan_currency')
+            .orderBy('currency');
+
+        if (rows.length === 0) {
+            // Add a USD placeholder to always have at least one currency
+            rows.push({
+                currency: 'usd',
+                mrr: 0
+            });
+        }
+
+        return rows;
+    }
+
+    /**
+     * Get the MRR deltas for all days (from old to new), grouped by currency (ascending alphabetically)
+     * @returns {Promise<MrrDelta[]>} The deltas sorted from new to old
+     */
+    async fetchAllDeltas() {
+        const knex = this.db.knex;
+        const rows = await knex('members_paid_subscription_events')
+            .select('currency')
+            // In SQLite, DATE(created_at) would map to a string value, while DATE(created_at) would map to a JSDate object in MySQL
+            // That is why we need the cast here (to have some consistency)
+            .select(knex.raw('CAST(DATE(created_at) as CHAR) as date'))
+            .select(knex.raw(`SUM(mrr_delta) as delta`))
+            .groupByRaw('CAST(DATE(created_at) as CHAR), currency')
+            .orderByRaw('CAST(DATE(created_at) as CHAR), currency');
+        return rows;
+    }
+
+    /**
+     * Returns a list of the MRR history for each day and currency, including the current MRR per currency as meta data. 
+     * The respons is in ascending date order, and currencies for the same date are always in ascending order.
+     * @returns {Promise<MrrHistory>}
+     */
+    async getHistory() {
+        // Fetch current total amounts and start counting from there
+        const totals = await this.getCurrentMrr();
+
+        const rows = await this.fetchAllDeltas();
+
+        // Get today in UTC (default timezone)
+        const today = moment().format('YYYY-MM-DD');
+
+        const results = [];
+
+        // Create a map of the totals by currency for fast lookup and editing
+        const currentTotals = {};
+        for (const total of totals) {
+            currentTotals[total.currency] = total.mrr;
+        }
+
+        // Loop in reverse order (needed to have correct sorted result)
+        for (let i = rows.length - 1; i >= 0; i -= 1) {
+            const row = rows[i];
+
+            if (currentTotals[row.currency] === undefined) {
+                // Skip unexpected currencies that are not in the totals
+                continue;
+            }
+
+            // Convert JSDates to YYYY-MM-DD (in UTC)
+            const date = moment(row.date).format('YYYY-MM-DD');
+            
+            if (date > today) {
+                // Skip results that are in the future for some reason
+                continue;
+            }
+
+            results.unshift({
+                date,
+                mrr: Math.max(0, currentTotals[row.currency]),
+                currency: row.currency
+            });
+
+            currentTotals[row.currency] -= row.delta;
+        }
+
+        // Now also add the oldest days we have left over and do not have deltas
+        const oldestDate = rows.length > 0 ? moment(rows[0].date).add(-1, 'days').format('YYYY-MM-DD') : today;
+        
+        // Note that we also need to loop the totals in reverse order because we need to unshift
+        for (let i = totals.length - 1; i >= 0; i -= 1) {
+            const total = totals[i];
+            results.unshift({
+                date: oldestDate,
+                mrr: Math.max(0, currentTotals[total.currency]),
+                currency: total.currency
+            });
+        }
+        
+        return {
+            data: results,
+            meta: {
+                totals
+            }
+        };
+    }
+}
+
+module.exports = MrrStatsService;
+
+/**
+ * @typedef MrrByCurrency
+ * @type {Object}
+ * @property {number} mrr
+ * @property {string} currency
+ */
+
+/**
+ * @typedef MrrDelta
+ * @type {Object}
+ * @property {Date} date
+ * @property {string} currency
+ * @property {number} delta MRR change on this day
+ */
+
+/**
+ * @typedef {Object} MrrRecord
+ * @property {string} date In YYYY-MM-DD format
+ * @property {string} currency
+ * @property {number} mrr MRR on this day
+ */
+
+/**
+ * @typedef {Object} MrrHistory
+ * @property {MrrRecord[]} data List of the total members by status for each day, including the paid deltas paid_subscribed and paid_canceled
+ * @property {Object} meta
+ */

package/core/server/services/stats/service.js

@@ -1,6 +1,8 @@
 const db = require('../../data/db');
 const MemberStatsService = require('./lib/members-stats-service');
+const MrrStatsService = require('./lib/mrr-stats-service');
 
 module.exports = {
-    members: new MemberStatsService({db})
+    members: new MemberStatsService({db}),
+    mrr: new MrrStatsService({db})
 };

package/core/server/services/stripe/service.js

@@ -12,6 +12,7 @@ const {getConfig} = require('./config');
 async function configureApi() {
     const cfg = getConfig(settings, config, urlUtils);
     if (cfg) {
+        cfg.testEnv = process.env.NODE_ENV.startsWith('test');
         await module.exports.configure(cfg);
         return true;
     }