ghost 5.111.0 → 5.113.0

package/core/server/services/stats/ReferrersStatsService.js

@@ -0,0 +1,164 @@
+const moment = require('moment');
+
+class ReferrersStatsService {
+    /**
+     * @param {object} deps
+     * @param {import('knex').Knex} deps.knex
+     **/
+    constructor({knex}) {
+        this.knex = knex;
+    }
+
+    /**
+     * Return a list of all the attribution sources for a given post, with their signup and conversion counts
+     * @param {string} postId
+     * @returns {Promise<AttributionCountStat[]>}
+     */
+    async getForPost(postId) {
+        const knex = this.knex;
+        const signupRows = await knex('members_created_events')
+            .select('referrer_source')
+            .select(knex.raw('COUNT(id) AS total'))
+            .where('attribution_id', postId)
+            .where('attribution_type', 'post')
+            .groupBy('referrer_source');
+
+        const conversionRows = await knex('members_subscription_created_events')
+            .select('referrer_source')
+            .select(knex.raw('COUNT(id) AS total'))
+            .where('attribution_id', postId)
+            .where('attribution_type', 'post')
+            .groupBy('referrer_source');
+
+        // Stitch them toghether, grouping them by source
+
+        const map = new Map();
+        for (const row of signupRows) {
+            map.set(row.referrer_source, {
+                source: row.referrer_source,
+                signups: row.total,
+                paid_conversions: 0
+            });
+        }
+
+        for (const row of conversionRows) {
+            const existing = map.get(row.referrer_source) ?? {
+                source: row.referrer_source,
+                signups: 0,
+                paid_conversions: 0
+            };
+            existing.paid_conversions = row.total;
+            map.set(row.referrer_source, existing);
+        }
+
+        return [...map.values()].sort((a, b) => b.paid_conversions - a.paid_conversions);
+    }
+
+    /**
+     * Return a list of all the attribution sources, with their signup and conversion counts on each date
+     * @returns {Promise<{data: AttributionCountStat[], meta: {}}>}
+     */
+    async getReferrersHistory() {
+        const paidConversionEntries = await this.fetchAllPaidConversionSources();
+        const signupEntries = await this.fetchAllSignupSources();
+
+        const allEntries = signupEntries.map((entry) => {
+            return {
+                ...entry,
+                paid_conversions: 0,
+                date: moment(entry.date).format('YYYY-MM-DD')
+            };
+        });
+
+        paidConversionEntries.forEach((entry) => {
+            const entryDate = moment(entry.date).format('YYYY-MM-DD');
+            const existingEntry = allEntries.find(e => e.source === entry.source && e.date === entryDate);
+
+            if (existingEntry) {
+                existingEntry.paid_conversions = entry.paid_conversions;
+            } else {
+                allEntries.push({
+                    ...entry,
+                    signups: 0,
+                    date: entryDate
+                });
+            }
+        });
+
+        // sort allEntries in date ascending format
+        allEntries.sort((a, b) => {
+            return moment(a.date).diff(moment(b.date));
+        });
+
+        return {
+            data: allEntries,
+            meta: {}
+        };
+    }
+
+    /**
+     * @returns {Promise<PaidConversionsCountStatDate[]>}
+     **/
+    async fetchAllPaidConversionSources() {
+        const knex = this.knex;
+        const ninetyDaysAgo = moment.utc().subtract(90, 'days').startOf('day').utc().format('YYYY-MM-DD HH:mm:ss');
+        const rows = await knex('members_subscription_created_events')
+            .select(knex.raw(`DATE(created_at) as date`))
+            .select(knex.raw(`COUNT(*) as paid_conversions`))
+            .select(knex.raw(`referrer_source as source`))
+            .where('created_at', '>=', ninetyDaysAgo)
+            .groupBy('date', 'referrer_source')
+            .orderBy('date');
+
+        return rows;
+    }
+
+    /**
+     * @returns {Promise<SignupCountStatDate[]>}
+     **/
+    async fetchAllSignupSources() {
+        const knex = this.knex;
+        const ninetyDaysAgo = moment.utc().subtract(90, 'days').startOf('day').utc().format('YYYY-MM-DD HH:mm:ss');
+        const rows = await knex('members_created_events')
+            .select(knex.raw(`DATE(created_at) as date`))
+            .select(knex.raw(`COUNT(*) as signups`))
+            .select(knex.raw(`referrer_source as source`))
+            .where('created_at', '>=', ninetyDaysAgo)
+            .groupBy('date', 'referrer_source')
+            .orderBy('date');
+
+        return rows;
+    }
+}
+
+module.exports = ReferrersStatsService;
+
+/**
+ * @typedef AttributionCountStat
+ * @type {Object}
+ * @property {string} source Attribution Source
+ * @property {number} signups Total free members signed up for this source
+ * @property {number} paid_conversions Total paid conversions for this source
+ */
+
+/**
+ * @typedef AttributionCountStatDate
+ * @type {AttributionCountStat}
+ * @property {string} date The date (YYYY-MM-DD) on which these counts were recorded
+ */
+
+/**
+ * @typedef {object} SignupCountStatDate
+ * @type {Object}
+ * @property {string} source Attribution Source
+ * @property {number} signups Total free members signed up for this source
+ * @property {string} date The date (YYYY-MM-DD) on which these counts were recorded
+ **/
+
+/**
+ * @typedef {object} PaidConversionsCountStatDate
+ * @type {Object}
+ * @property {string} source Attribution Source
+ * @property {number} paid_conversions Total paid conversions for this source
+ * @property {string} date The date (YYYY-MM-DD) on which these counts were recorded
+ **/

package/core/server/services/stats/StatsService.js

@@ -0,0 +1,63 @@
+const MRRService = require('./MrrStatsService');
+const MembersService = require('./MembersStatsService');
+const SubscriptionStatsService = require('./SubscriptionStatsService');
+const ReferrersStatsService = require('./ReferrersStatsService');
+
+class StatsService {
+    /**
+     * @param {object} deps
+     * @param {MRRService} deps.mrr
+     * @param {MembersService} deps.members
+     * @param {SubscriptionStatsService} deps.subscriptions
+     * @param {ReferrersStatsService} deps.referrers
+     **/
+    constructor(deps) {
+        this.mrr = deps.mrr;
+        this.members = deps.members;
+        this.subscriptions = deps.subscriptions;
+        this.referrers = deps.referrers;
+    }
+
+    async getMRRHistory() {
+        return this.mrr.getHistory();
+    }
+
+    async getMemberCountHistory() {
+        return this.members.getCountHistory();
+    }
+
+    async getSubscriptionCountHistory() {
+        return this.subscriptions.getSubscriptionHistory();
+    }
+
+    async getReferrersHistory() {
+        return this.referrers.getReferrersHistory();
+    }
+
+    /**
+     * @param {string} postId
+     */
+    async getPostReferrers(postId) {
+        return {
+            data: await this.referrers.getForPost(postId),
+            meta: {}
+        };
+    }
+
+    /**
+     * @param {object} deps
+     * @param {import('knex').Knex} deps.knex
+     *
+     * @returns {StatsService}
+     **/
+    static create(deps) {
+        return new StatsService({
+            mrr: new MRRService(deps),
+            members: new MembersService(deps),
+            subscriptions: new SubscriptionStatsService(deps),
+            referrers: new ReferrersStatsService(deps)
+        });
+    }
+}
+
+module.exports = StatsService;

package/core/server/services/stats/SubscriptionStatsService.js

@@ -0,0 +1,180 @@
+const moment = require('moment');
+
+class SubscriptionStatsService {
+    /**
+     * @param {object} deps
+     * @param {import('knex').Knex} deps.knex*/
+    constructor({knex}) {
+        this.knex = knex;
+    }
+
+    /**
+     * @returns {Promise<{data: SubscriptionHistoryEntry[]}>}
+     **/
+    async getSubscriptionHistory() {
+        const subscriptionDeltaEntries = await this.fetchAllSubscriptionDeltas();
+        const counts = await this.fetchSubscriptionCounts();
+
+        /** @type {Object.<string, Object.<string, number>>} */
+        const countData = {};
+        counts.forEach((count) => {
+            if (!countData[count.tier]) {
+                countData[count.tier] = {};
+            }
+            countData[count.tier][count.cadence] = count.count;
+        });
+
+        /** @type {SubscriptionHistoryEntry[]} */
+        let subscriptionHistoryEntries = [];
+
+        /** @type {string[]} */
+        let cadences = [];
+        /** @type {string[]} */
+        let tiers = [];
+
+        for (let index = subscriptionDeltaEntries.length - 1; index >= 0; index -= 1) {
+            const entry = subscriptionDeltaEntries[index];
+            if (!countData[entry.tier]) {
+                countData[entry.tier] = {};
+            }
+            if (!countData[entry.tier][entry.cadence]) {
+                countData[entry.tier][entry.cadence] = 0;
+            }
+
+            subscriptionHistoryEntries.unshift({
+                ...entry,
+                date: moment(entry.date).format('YYYY-MM-DD'),
+                count: countData[entry.tier][entry.cadence]
+            });
+
+            countData[entry.tier][entry.cadence] += entry.negative_delta;
+            countData[entry.tier][entry.cadence] -= entry.positive_delta;
+
+            if (!cadences.includes(entry.cadence)) {
+                cadences.push(entry.cadence);
+            }
+            if (!tiers.includes(entry.tier)) {
+                tiers.push(entry.tier);
+            }
+        }
+
+        return {
+            data: subscriptionHistoryEntries,
+            meta: {
+                cadences,
+                tiers,
+                totals: counts
+            }
+        };
+    }
+
+    /**
+     * @returns {Promise<SubscriptionDelta[]>}
+     **/
+    async fetchAllSubscriptionDeltas() {
+        const knex = this.knex;
+        const rows = await knex('members_paid_subscription_events')
+            .join('stripe_prices AS price', function () {
+                this.on('price.stripe_price_id', '=', 'members_paid_subscription_events.from_plan')
+                    .orOn('price.stripe_price_id', '=', 'members_paid_subscription_events.to_plan');
+            })
+            .join('stripe_products AS product', 'product.stripe_product_id', '=', 'price.stripe_product_id')
+            .join('products AS tier', 'tier.id', '=', 'product.product_id')
+            .leftJoin('stripe_prices AS from_price', 'from_price.stripe_price_id', '=', 'members_paid_subscription_events.from_plan')
+            .leftJoin('stripe_prices AS to_price', 'to_price.stripe_price_id', '=', 'members_paid_subscription_events.to_plan')
+            .select(knex.raw(`
+                DATE(members_paid_subscription_events.created_at) as date
+            `))
+            .select(knex.raw(`
+                tier.id as tier
+            `))
+            .select(knex.raw(`
+                price.interval as cadence
+            `))
+            .select(knex.raw(`SUM(
+                CASE
+                    WHEN members_paid_subscription_events.type IN ('created','reactivated','active') AND members_paid_subscription_events.mrr_delta != 0 THEN 1
+                    WHEN members_paid_subscription_events.type='updated' AND price.id = to_price.id THEN 1
+                    WHEN members_paid_subscription_events.type='updated' AND members_paid_subscription_events.from_plan = members_paid_subscription_events.to_plan AND members_paid_subscription_events.mrr_delta > 0 THEN 1
+                    ELSE 0
+                END
+            ) as positive_delta`))
+            .select(knex.raw(`SUM(
+                CASE
+                    WHEN members_paid_subscription_events.type IN ('canceled', 'expired','inactive') AND members_paid_subscription_events.mrr_delta != 0 THEN 1
+                    WHEN members_paid_subscription_events.type='updated' AND price.id = from_price.id THEN 1
+                    ELSE 0
+                END
+            ) as negative_delta`))
+            .select(knex.raw(`SUM(
+                CASE
+                    WHEN members_paid_subscription_events.type IN ('created','reactivated','active') AND members_paid_subscription_events.mrr_delta != 0 THEN 1
+                    WHEN members_paid_subscription_events.type='updated' AND members_paid_subscription_events.from_plan = members_paid_subscription_events.to_plan AND members_paid_subscription_events.mrr_delta > 0 THEN 1
+                    ELSE 0
+                END
+            ) as signups`))
+            .select(knex.raw(`SUM(
+                CASE
+                    WHEN members_paid_subscription_events.type IN ('canceled', 'expired','inactive') AND members_paid_subscription_events.mrr_delta != 0 THEN 1
+                    ELSE 0
+                END
+            ) as cancellations`))
+            .groupBy('date', 'tier', 'cadence')
+            .orderBy('date');
+
+        return rows;
+    }
+
+    /**
+      * Get the current total subscriptions grouped by Cadence and Tier
+      * @returns {Promise<SubscriptionCount[]>}
+      **/
+    async fetchSubscriptionCounts() {
+        const knex = this.knex;
+
+        const data = await knex('members_stripe_customers_subscriptions')
+            .select(knex.raw(`
+                  COUNT(members_stripe_customers_subscriptions.id) AS count,
+                  products.id AS tier,
+                  stripe_prices.interval AS cadence
+             `))
+            .join('stripe_prices', 'stripe_prices.stripe_price_id', '=', 'members_stripe_customers_subscriptions.stripe_price_id')
+            .join('stripe_products', 'stripe_products.stripe_product_id', '=', 'stripe_prices.stripe_product_id')
+            .join('products', 'products.id', '=', 'stripe_products.product_id')
+            .whereNot('members_stripe_customers_subscriptions.mrr', 0)
+            .groupBy('tier', 'cadence');
+
+        return data;
+    }
+}
+
+/** @typedef {object} SubscriptionCount
+  * @prop {string} tier
+  * @prop {string} cadence
+  * @prop {number} count
+  **/
+
+/**
+ * @typedef {object} SubscriptionDelta
+ * @prop {string} tier
+ * @prop {string} cadence
+ * @prop {string} date
+ * @prop {number} positive_delta
+ * @prop {number} negative_delta
+ * @prop {number} signups
+ * @prop {number} cancellations
+ **/
+
+/**
+ * @typedef {object} SubscriptionHistoryEntry
+ * @prop {string} tier
+ * @prop {string} cadence
+ * @prop {string} date
+ * @prop {number} positive_delta
+ * @prop {number} negative_delta
+ * @prop {number} signups
+ * @prop {number} cancellations
+ * @prop {number} count
+ **/
+
+module.exports = SubscriptionStatsService;

package/core/server/services/stats/service.js

@@ -10,7 +10,7 @@ class StatsServiceWrapper {
             return;
         }
 
-        const StatsService = require('@tryghost/stats-service');
+        const StatsService = require('./StatsService');
         const db = require('../../data/db');
 
         this.api = StatsService.create({knex: db.knex});

package/core/server/services/url/Resources.js

@@ -9,6 +9,8 @@ const models = require('../../models');
 // This listens to all manner of model events to find new content that needs a URL...
 const events = require('../../lib/common/events');
 
+/** @typedef {'posts' | 'pages' | 'tags' | 'authors'} ResourceType */
+
 /**
  * @description At the moment the resources class is directly responsible for data population
  * for URLs...but because it's actually a storage cache of all published
@@ -173,7 +175,7 @@ class Resources {
      * If we resolve (https://github.com/TryGhost/Ghost/issues/10360) and talk to the Content API,
      * we could pass on e.g. `?include=authors&fields=authors.id,authors.slug`, but the API has to support it.
      *
-     * @param {Bookshelf-Model} model
+     * @param {import('bookshelf').Model} model
      * @param {Object} resourceConfig
      * @private
      */
@@ -227,8 +229,8 @@ class Resources {
      * all subscribers sequentially. The first generator, where the conditions match the resource, will
      * own the resource and it's url.
      *
-     * @param {String} type (post,user...)
-     * @param {Bookshelf-Model} model
+     * @param {ResourceType} type
+     * @param {import('bookshelf').Model} model
      * @private
      */
     async _onResourceAdded(type, model) {
@@ -306,8 +308,8 @@ class Resources {
      *   - but the data changed and is maybe no longer owned?
      *   - e.g. featured:false changes and your filter requires featured posts
      *
-     * @param {String} type (post,user...)
-     * @param {Bookshelf-Model} model
+     * @param {ResourceType} type
+     * @param {import('bookshelf').Model} model
      * @private
      */
     async _onResourceUpdated(type, model) {
@@ -389,37 +391,25 @@ class Resources {
 
     /**
      * @description Listener for "model removed" event.
-     * @param {String} type (post,user...)
-     * @param {Bookshelf-Model} model
+     * @param {ResourceType} type
+     * @param {import('bookshelf').Model} model
      * @private
      */
     _onResourceRemoved(type, model) {
         debug('_onResourceRemoved', type);
 
-        let index = null;
-        let resource;
-
-        // CASE: search for the cached resource and stop if it was found
-        this.data[type].every((_resource, _index) => {
-            if (_resource.data.id === model._previousAttributes.id) {
-                resource = _resource;
-                index = _index;
-                // break!
-                return false;
-            }
-
-            return true;
-        });
+        const resourceId = model._previousAttributes.id;
+        const index = this.data[type].findIndex(resource => resource.data.id === resourceId);
 
-        // CASE: there are possible cases that the resource was not fetched e.g. visibility is internal
-        if (index === null) {
-            debug('can\'t find resource', model._previousAttributes.id);
+        // Resource might not be in cache if e.g. visibility was internal
+        if (index === -1) {
+            debug('Resource not found in cache', resourceId);
             return;
         }
 
         // remove the resource from cache
-        this.data[type].splice(index, 1);
-        resource.remove();
+        const [removedResource] = this.data[type].splice(index, 1);
+        removedResource.remove();
     }
 
     /**
@@ -432,7 +422,7 @@ class Resources {
 
     /**
      * @description Get all cached resourced by type.
-     * @param {String} type (post, user...)
+     * @param {ResourceType} type
      * @returns {Object}
      */
     getAllByType(type) {
@@ -441,12 +431,12 @@ class Resources {
 
     /**
      * @description Get all cached resourced by resource id and type.
-     * @param {String} type (post, user...)
-     * @param {String} id
+     * @param {ResourceType} type
+     * @param {string} id
      * @returns {Object}
      */
     getByIdAndType(type, id) {
-        return _.find(this.data[type], {data: {id: id}});
+        return this.data[type]?.find(r => r.data.id === id);
     }
 
     /**

package/core/server/services/url/UrlService.js

@@ -54,7 +54,7 @@ class UrlService {
         this.queue.addListener('started', this._onQueueStartedListener);
 
         this._onQueueEndedListener = this._onQueueEnded.bind(this);
-        this.queue.addListener('ended', this._onQueueEnded.bind(this));
+        this.queue.addListener('ended', this._onQueueEndedListener);
     }
 
     /**
@@ -263,17 +263,7 @@ class UrlService {
     owns(routerId, id) {
         debug('owns', routerId, id);
 
-        let urlGenerator;
-
-        this.urlGenerators.every((_urlGenerator) => {
-            if (_urlGenerator.identifier === routerId) {
-                urlGenerator = _urlGenerator;
-                return false;
-            }
-
-            return true;
-        });
-
+        const urlGenerator = this.urlGenerators.find(g => g.identifier === routerId);
         if (!urlGenerator) {
             return false;
         }

package/core/server/services/url/Urls.js

@@ -1,4 +1,3 @@
-const _ = require('lodash');
 const debug = require('@tryghost/debug')('services:url:urls');
 const urlUtils = require('../../../shared/url-utils');
 const logging = require('@tryghost/logging');
@@ -7,6 +6,10 @@ const errors = require('@tryghost/errors');
 // This emits its own url added/removed events
 const events = require('../../lib/common/events');
 
+/**
+ * @typedef {{url: string, generatorId: string, resource: import('./Resource')}} Url
+ */
+
 /**
  * This class keeps track of all urls in the system.
  * Each resource has exactly one url. Each url is owned by exactly one url generator id.
@@ -20,24 +23,14 @@ const events = require('../../lib/common/events');
  * You can easily ask `this.urls[resourceId]`.
  */
 class Urls {
-    /**
-     *
-     * @param {Object} [options]
-     * @param {Object} [options.urls] map of available URLs with their resources
-     */
-    constructor({urls = {}} = {}) {
-        this.urls = urls;
-    }
+    /** @type {Object<string, Url>} */
+    urls = {};
 
     /**
      * @description Add a url to the system.
-     * @param {Object} options
-     * @param {import('./Resource')} options.resource - instance of the Resource class
-     * @param {string} options.generatorId
-     * @param {string} options.url
+     * @param {Url} options
      */
-    add(options) {
-        const {url, generatorId, resource} = options;
+    add({url, generatorId, resource}) {
         debug('add', resource.data.id, url);
 
         if (this.urls[resource.data.id]) {
@@ -76,7 +69,7 @@ class Urls {
     /**
      * @description Get url by resource id.
      * @param {String} id
-     * @returns {Object}
+     * @returns {Url}
      */
     getByResourceId(id) {
         return this.urls[id];
@@ -85,16 +78,10 @@ class Urls {
     /**
      * @description Get all urls by generator id.
      * @param {String} generatorId
-     * @returns {Array}
+     * @returns {Url[]}
      */
     getByGeneratorId(generatorId) {
-        return _.reduce(Object.keys(this.urls), (toReturn, resourceId) => {
-            if (this.urls[resourceId].generatorId === generatorId) {
-                toReturn.push(this.urls[resourceId]);
-            }
-
-            return toReturn;
-        }, []);
+        return Object.values(this.urls).filter(url => url.generatorId === generatorId);
     }
 
     /**
@@ -108,20 +95,17 @@ class Urls {
      *
      *  But depending on the routing registration, you will always serve e.g. resource1,
      *  because the router it belongs to was registered first.
+     *
+     *  @param {string} urlToLookup
+     *  @returns {Url[]}
      */
-    getByUrl(url) {
-        return _.reduce(Object.keys(this.urls), (toReturn, resourceId) => {
-            if (this.urls[resourceId].url === url) {
-                toReturn.push(this.urls[resourceId]);
-            }
-
-            return toReturn;
-        }, []);
+    getByUrl(urlToLookup) {
+        return Object.values(this.urls).filter(url => url.url === urlToLookup);
     }
 
     /**
      * @description Remove url.
-     * @param id
+     * @param {string} id
      */
     removeResourceId(id) {
         if (!this.urls[id]) {

package/core/shared/config/defaults.json

@@ -214,7 +214,7 @@
     },
     "comments": {
         "url": "https://cdn.jsdelivr.net/ghost/comments-ui@~{version}/umd/comments-ui.min.js",
-        "version": "1.0"
+        "version": "1.1"
     },
     "signupForm": {
         "url": "https://cdn.jsdelivr.net/ghost/signup-form@~{version}/umd/signup-form.min.js",

package/core/shared/labs.js

@@ -38,7 +38,8 @@ const BETA_FEATURES = [
     'ActivityPub',
     'importMemberTier',
     'staff2fa',
-    'contentVisibility'
+    'contentVisibility',
+    'superEditors'
 ];
 
 const ALPHA_FEATURES = [