ghost 5.111.0 → 5.113.0

package/core/server/services/media-inliner/ExternalMediaInliner.js

@@ -0,0 +1,346 @@
+const mime = require('mime-types');
+const FileType = require('file-type');
+const request = require('@tryghost/request');
+const errors = require('@tryghost/errors');
+const logging = require('@tryghost/logging');
+const string = require('@tryghost/string');
+const path = require('path');
+
+class ExternalMediaInliner {
+    /** @type {object} */
+    #PostModel;
+
+    /** @type {object} */
+    #PostMetaModel;
+
+    /** @type {object} */
+    #TagModel;
+
+    /** @type {object} */
+    #UserModel;
+
+    /**
+     *
+     * @param {Object} deps
+     * @param {Object} deps.PostModel - Post model
+     * @param {Object} deps.PostMetaModel - PostMeta model
+     * @param {Object} deps.TagModel - Tag model
+     * @param {Object} deps.UserModel - User model
+     * @param {(extension) => import('ghost-storage-base')} deps.getMediaStorage - getMediaStorage
+     */
+    constructor(deps) {
+        this.#PostModel = deps.PostModel;
+        this.#PostMetaModel = deps.PostMetaModel;
+        this.#TagModel = deps.TagModel;
+        this.#UserModel = deps.UserModel;
+        this.getMediaStorage = deps.getMediaStorage;
+    }
+
+    /**
+     *
+     * @param {string} requestURL - url of remote media
+     * @returns {Promise<Object>}
+     */
+    async getRemoteMedia(requestURL) {
+        // @NOTE: this is the most expensive operation in the whole inlining process
+        //        we should consider caching the results to improve performance
+
+        // Enforce http - http > https redirects are commonplace
+        requestURL = requestURL.replace(/^\/\//g, 'http://');
+
+        // Encode to handle special characters in URLs
+        requestURL = encodeURI(requestURL);
+        try {
+            const response = await request(requestURL, {
+                followRedirect: true,
+                responseType: 'buffer'
+            });
+
+            return response;
+        } catch (error) {
+            // NOTE: add special case for 404s
+            logging.error(`Error downloading remote media: ${requestURL}`);
+            logging.error(new errors.DataImportError({
+                err: error
+            }));
+
+            return null;
+        }
+    }
+
+    /**
+     *
+     * @param {Object} response - response from request
+     * @returns {Object}
+     */
+    async extractFileDataFromResponse(requestURL, response) {
+        let extension;
+
+        // Attempt to get the file extension from the file itself
+        // If that fails, or if `.ext` is undefined, get the extension from the file path in the catch
+        try {
+            const fileInfo = await FileType.fromBuffer(response.body);
+            extension = fileInfo.ext;
+        } catch {
+            const headers = response.headers;
+            const contentType = headers['content-type'];
+            const extensionFromPath = path.parse(requestURL).ext.split(/[^a-z]/i).filter(Boolean)[0];
+            extension = mime.extension(contentType) || extensionFromPath;
+        }
+
+        const removeExtRegExp = new RegExp(`.${extension}`, '');
+        const fileNameNoExt = path.parse(requestURL).base.replace(removeExtRegExp, '');
+
+        // CASE: Query strings _can_ form part of the unique image URL, so rather that strip them include the in the file name
+        // Then trim to last 248 chars (this will be more unique than the first 248), and trim leading & trailing dashes.
+        // 248 is on the lower end of limits from various OSes and file systems
+        const fileName = string.slugify(path.parse(fileNameNoExt).base, {
+            requiredChangesOnly: true
+        }).slice(-248).replace(/^-|-$/, '');
+
+        return {
+            fileBuffer: response.body,
+            filename: `${fileName}.${extension}`,
+            extension: `.${extension}`
+        };
+    }
+
+    /**
+     *
+     * @param {Object} media - media to store locally
+     * @returns {Promise<string>} - path to stored media
+     */
+    async storeMediaLocally(media) {
+        const storage = this.getMediaStorage(media.extension);
+
+        if (!storage) {
+            logging.warn(`No storage adapter found for file extension: ${media.extension}`);
+            return null;
+        } else {
+            // @NOTE: this is extremely convoluted and should live on a
+            //        storage adapter level
+            const targetDir = storage.getTargetDir(storage.storagePath);
+            const uniqueFileName = await storage.getUniqueFileName({
+                name: media.filename
+            }, targetDir);
+            const targetPath = path.relative(storage.storagePath, uniqueFileName);
+            const filePath = await storage.saveRaw(media.fileBuffer, targetPath);
+            return filePath;
+        }
+    }
+
+    static findMatches(content, domain) {
+        // NOTE: the src could end with a quote, bracket, apostrophe, double-backslash, or encoded quote.
+        //     Backlashes are added to content as an escape character
+        const srcTerminationSymbols = `("|\\)|'|(?=(?:,https?))| |<|\\\\|&quot;|$)`;
+        const regex = new RegExp(`(${domain}.*?)(${srcTerminationSymbols})`, 'igm');
+        const matches = content.matchAll(regex);
+
+        // Simplify the matches so we only get the result needed
+        let matchesArray = Array.from(matches, m => m[1]);
+
+        // Trim trailing commas from each match
+        matchesArray = matchesArray.map((item) => {
+            return item.replace(/,$/, '');
+        });
+
+        return matchesArray;
+    }
+
+    /**
+     * Find & inline external media from a JSON sting.
+     * This works with both Lexical & Mobiledoc, so no separate methods are needed here.
+     *
+     * @param {string} content - stringified JSON of post Lexical or Mobiledoc content
+     * @param {String[]} domains - domains to inline media from
+     * @returns {Promise<string>} - updated stringified JSON of post content
+     */
+    async inlineContent(content, domains) {
+        for (const domain of domains) {
+            const matches = this.constructor.findMatches(content, domain);
+
+            for (const src of matches) {
+                const response = await this.getRemoteMedia(src);
+
+                let media;
+                if (response) {
+                    media = await this.extractFileDataFromResponse(src, response);
+                }
+
+                if (media) {
+                    const filePath = await this.storeMediaLocally(media);
+
+                    if (filePath) {
+                        const inlinedSrc = `__GHOST_URL__${filePath}`;
+
+                        // NOTE: does not account for duplicate images in content
+                        //       in those cases would be processed twice
+                        content = content.replace(src, inlinedSrc);
+                        logging.info(`Inlined media: ${src} -> ${inlinedSrc}`);
+                    }
+                }
+            }
+        }
+
+        return content;
+    }
+
+    /**
+     *
+     * @param {Object} resourceModel - one of PostModel, TagModel, UserModel instances
+     * @param {String[]} fields - fields to inline
+     * @param {String[]} domains - domains to inline media from
+     * @returns Promise<Object> - updated fields map with local media paths
+     */
+    async inlineFields(resourceModel, fields, domains) {
+        const updatedFields = {};
+
+        for (const field of fields) {
+            for (const domain of domains) {
+                const src = resourceModel.get(field);
+
+                if (src && src.startsWith(domain)) {
+                    const response = await this.getRemoteMedia(src);
+
+                    let media;
+                    if (response) {
+                        media = await this.extractFileDataFromResponse(src, response);
+                    }
+
+                    if (media) {
+                        const filePath = await this.storeMediaLocally(media);
+
+                        if (filePath) {
+                            const inlinedSrc = `__GHOST_URL__${filePath}`;
+
+                            updatedFields[field] = inlinedSrc;
+                            logging.info(`Added media to inline: ${src} -> ${inlinedSrc}`);
+                        }
+                    }
+                }
+            }
+        }
+
+        return updatedFields;
+    }
+
+    /**
+     *
+     * @param {Object[]} resources - array of model instances
+     * @param {Object} model - resource model
+     * @param {string[]} fields - fields to inline
+     * @param {string[]} domains - domains to inline media from
+     */
+    async inlineSimpleFields(resources, model, fields, domains) {
+        logging.info(`Starting inlining external media for ${resources?.length} resources and with ${fields.join(', ')} fields`);
+
+        for (const resource of resources) {
+            try {
+                const updatedFields = await this.inlineFields(resource, fields, domains);
+
+                if (Object.keys(updatedFields).length > 0) {
+                    await model.edit(updatedFields, {
+                        id: resource.id,
+                        context: {
+                            internal: true
+                        }
+                    });
+                }
+            } catch (err) {
+                logging.error(`Error inlining media for: ${resource.id}`);
+                logging.error(new errors.DataImportError({
+                    err
+                }));
+            }
+        }
+    }
+
+    /**
+     *
+     * @param {string[]} domains domains to inline media from
+     */
+    async inline(domains) {
+        const posts = await this.#PostModel.findAll({context: {internal: true}});
+        const postsInilingFields = [
+            'feature_image'
+        ];
+
+        logging.info(`Starting inlining external media for posts: ${posts?.length}`);
+
+        for (const post of posts) {
+            try {
+                const mobiledocContent = post.get('mobiledoc');
+                const lexicalContent = post.get('lexical');
+
+                const updatedFields = await this.inlineFields(post, postsInilingFields, domains);
+
+                if (mobiledocContent) {
+                    const inlinedContent = await this.inlineContent(mobiledocContent, domains);
+
+                    // If content has changed, update the post
+                    if (inlinedContent !== mobiledocContent) {
+                        updatedFields.mobiledoc = inlinedContent;
+                    }
+                }
+
+                if (lexicalContent) {
+                    const inlinedContent = await this.inlineContent(lexicalContent, domains);
+
+                    // If content has changed, update the post
+                    if (inlinedContent !== lexicalContent) {
+                        updatedFields.lexical = inlinedContent;
+                    }
+                }
+
+                if (Object.keys(updatedFields).length > 0) {
+                    await this.#PostModel.edit(updatedFields, {
+                        id: post.id,
+                        context: {
+                            internal: true
+                        }
+                    });
+                }
+            } catch (err) {
+                logging.error(`Error inlining media for post: ${post.id}`);
+                logging.error(new errors.DataImportError({
+                    err
+                }));
+            }
+        }
+
+        const {data: postsMetas} = await this.#PostMetaModel.findPage({
+            limit: 'all'
+        });
+        const postsMetaInilingFields = [
+            'og_image',
+            'twitter_image'
+        ];
+
+        await this.inlineSimpleFields(postsMetas, this.#PostMetaModel, postsMetaInilingFields, domains);
+
+        const {data: tags} = await this.#TagModel.findPage({
+            limit: 'all'
+        });
+        const tagInliningFields = [
+            'feature_image',
+            'og_image',
+            'twitter_image'
+        ];
+
+        await this.inlineSimpleFields(tags, this.#TagModel, tagInliningFields, domains);
+
+        const {data: users} = await this.#UserModel.findPage({
+            limit: 'all'
+        });
+        const userInliningFields = [
+            'profile_image',
+            'cover_image'
+        ];
+
+        await this.inlineSimpleFields(users, this.#UserModel, userInliningFields, domains);
+
+        logging.info('Finished inlining external media for posts, tags, and users');
+    }
+}
+
+module.exports = ExternalMediaInliner;

package/core/server/services/media-inliner/service.js

@@ -1,7 +1,7 @@
 module.exports = {
     async init() {
         const debug = require('@tryghost/debug')('mediaInliner');
-        const MediaInliner = require('@tryghost/external-media-inliner');
+        const MediaInliner = require('./ExternalMediaInliner');
         const models = require('../../models');
         const jobsService = require('../jobs');
 

package/core/server/services/permissions/can-this.js

@@ -5,6 +5,7 @@ const tpl = require('@tryghost/tpl');
 const providers = require('./providers');
 const parseContext = require('./parse-context');
 const actionsMap = require('./actions-map-cache');
+const {setIsRoles} = require('../../models/role-utils');
 
 const messages = {
     noPermissionToAction: 'You do not have permission to perform this action',
@@ -66,8 +67,8 @@ class CanThisResult {
 
                         return true;
                     };
-
-                    if (loadedPermissions.user && _.some(loadedPermissions.user.roles, {name: 'Owner'})) {
+                    const {isOwner} = setIsRoles(loadedPermissions);
+                    if (isOwner) {
                         hasUserPermission = true;
                     } else if (!_.isEmpty(userPermissions)) {
                         hasUserPermission = _.some(userPermissions, checkPermission);

package/core/server/services/stats/MembersStatsService.js

@@ -0,0 +1,167 @@
+const moment = require('moment');
+
+class MembersStatsService {
+    /**
+     * @param {object} deps
+     * @param {import('knex').Knex} deps.knex*/
+    constructor({knex}) {
+        this.knex = knex;
+    }
+
+    /**
+     * Get the current total members grouped by status
+     * @returns {Promise<TotalMembersByStatus>}
+     */
+    async getCount() {
+        const knex = this.knex;
+        const rows = await knex('members')
+            .select('status')
+            .select(knex.raw('COUNT(id) AS total'))
+            .groupBy('status');
+
+        const paidEvent = rows.find(c => c.status === 'paid');
+        const freeEvent = rows.find(c => c.status === 'free');
+        const compedEvent = rows.find(c => c.status === 'comped');
+
+        return {
+            paid: paidEvent ? paidEvent.total : 0,
+            free: freeEvent ? freeEvent.total : 0,
+            comped: compedEvent ? compedEvent.total : 0
+        };
+    }
+
+    /**
+     * Get the member deltas by status for all days, sorted ascending
+     * @returns {Promise<MemberStatusDelta[]>} The deltas of paid, free and comped users per day, sorted ascending
+     */
+    async fetchAllStatusDeltas() {
+        const knex = this.knex;
+        const ninetyDaysAgo = moment.utc().subtract(91, 'days').startOf('day').utc().format('YYYY-MM-DD HH:mm:ss');
+        const rows = await knex('members_status_events')
+            .select(knex.raw('DATE(created_at) as date'))
+            .select(knex.raw(`SUM(
+                CASE WHEN to_status='paid' THEN 1
+                ELSE 0 END
+            ) as paid_subscribed`))
+            .select(knex.raw(`SUM(
+                CASE WHEN from_status='paid' THEN 1
+                ELSE 0 END
+            ) as paid_canceled`))
+            .select(knex.raw(`SUM(
+                CASE WHEN to_status='comped' THEN 1
+                WHEN from_status='comped' THEN -1
+                ELSE 0 END
+            ) as comped_delta`))
+            .select(knex.raw(`SUM(
+                CASE WHEN to_status='free' THEN 1
+                WHEN from_status='free' THEN -1
+                ELSE 0 END
+            ) as free_delta`))
+            .where('created_at', '>=', ninetyDaysAgo)
+            .groupByRaw('DATE(created_at)');
+        return rows;
+    }
+
+    /**
+     * Returns a list of the total members by status for each day, including the paid deltas paid_subscribed and paid_canceled
+     * @returns {Promise<CountHistory>}
+     */
+    async getCountHistory() {
+        const rows = await this.fetchAllStatusDeltas();
+
+        // Fetch current total amounts and start counting from there
+        const totals = await this.getCount();
+        let {paid, free, comped} = totals;
+
+        // Get today in UTC (default timezone)
+        const today = moment().format('YYYY-MM-DD');
+
+        const cumulativeResults = [];
+
+        rows.sort((a, b) => new Date(a.date) - new Date(b.date));
+        // Loop in reverse order (needed to have correct sorted result)
+        for (let i = rows.length - 1; i >= 0; i -= 1) {
+            const row = rows[i];
+
+            // Convert JSDates to YYYY-MM-DD (in UTC)
+            const date = moment(row.date).format('YYYY-MM-DD');
+            if (date > today) {
+                // Skip results that are in the future (fix for invalid events)
+                continue;
+            }
+            cumulativeResults.unshift({
+                date,
+                paid: Math.max(0, paid),
+                free: Math.max(0, free),
+                comped: Math.max(0, comped),
+
+                // Deltas
+                paid_subscribed: row.paid_subscribed,
+                paid_canceled: row.paid_canceled
+            });
+
+            // Update current counts
+            paid -= row.paid_subscribed - row.paid_canceled;
+            free -= row.free_delta;
+            comped -= row.comped_delta;
+        }
+
+        // Now also add the oldest day we have left over (this one will be zero, which is also needed as a data point for graphs)
+        const oldestDate = rows.length > 0 ? moment(rows[0].date).add(-1, 'days').format('YYYY-MM-DD') : today;
+
+        cumulativeResults.unshift({
+            date: oldestDate,
+            paid: Math.max(0, paid),
+            free: Math.max(0, free),
+            comped: Math.max(0, comped),
+
+            // Deltas
+            paid_subscribed: 0,
+            paid_canceled: 0
+        });
+
+        return {
+            data: cumulativeResults,
+            meta: {
+                totals
+            }
+        };
+    }
+}
+
+module.exports = MembersStatsService;
+
+/**
+ * @typedef MemberStatusDelta
+ * @type {Object}
+ * @property {Date} date
+ * @property {number} paid_subscribed Paid members that subscribed on this day
+ * @property {number} paid_canceled Paid members that canceled on this day
+ * @property {number} comped_delta Total net comped members on this day
+ * @property {number} free_delta Total net members on this day
+ */
+
+/**
+ * @typedef TotalMembersByStatus
+ * @type {Object}
+ * @property {number} paid Total paid members
+ * @property {number} free Total free members
+ * @property {number} comped Total comped members
+ */
+
+/**
+ * @typedef {Object} TotalMembersByStatusItem
+ * @property {string} date In YYYY-MM-DD format
+ * @property {number} paid Total paid members
+ * @property {number} free Total free members
+ * @property {number} comped Total comped members
+ * @property {number} paid_subscribed Paid members that subscribed on this day
+ * @property {number} paid_canceled Paid members that canceled on this day
+ */
+
+/**
+ * @typedef {Object} CountHistory
+ * @property {TotalMembersByStatusItem[]} data List of the total members by status for each day, including the paid deltas paid_subscribed and paid_canceled
+ * @property {Object} meta
+ * @property {TotalMembersByStatus} meta.totals
+ */

package/core/server/services/stats/MrrStatsService.js

@@ -0,0 +1,161 @@
+const moment = require('moment');
+
+class MrrStatsService {
+    /**
+     * @param {object} deps
+     * @param {import('knex').Knex} deps.knex
+     **/
+    constructor({knex}) {
+        this.knex = knex;
+    }
+
+    /**
+     * Get the current total MRR, grouped by currency (ascending order)
+     * @returns {Promise<MrrByCurrency[]>}
+     */
+    async getCurrentMrr() {
+        const knex = this.knex;
+        const rows = await knex('members_stripe_customers_subscriptions')
+            .select(knex.raw(`plan_currency as currency`))
+            .select(knex.raw(`SUM(mrr) AS mrr`))
+            .groupBy('plan_currency')
+            .orderBy('currency');
+
+        if (rows.length === 0) {
+            // Add a USD placeholder to always have at least one currency
+            rows.push({
+                currency: 'usd',
+                mrr: 0
+            });
+        }
+
+        return rows;
+    }
+
+    /**
+     * Get the MRR deltas for all days (from old to new), grouped by currency (ascending alphabetically)
+     * @returns {Promise<MrrDelta[]>} The deltas sorted from new to old
+     */
+    async fetchAllDeltas() {
+        const knex = this.knex;
+        const ninetyDaysAgo = moment.utc().subtract(90, 'days').startOf('day').utc().format('YYYY-MM-DD HH:mm:ss');
+        const rows = await knex('members_paid_subscription_events')
+            .select('currency')
+            // In SQLite, DATE(created_at) would map to a string value, while DATE(created_at) would map to a JSDate object in MySQL
+            // That is why we need the cast here (to have some consistency)
+            .select(knex.raw('CAST(DATE(created_at) as CHAR) as date'))
+            .select(knex.raw(`SUM(mrr_delta) as delta`))
+            .where('created_at', '>=', ninetyDaysAgo)
+            .groupByRaw('CAST(DATE(created_at) as CHAR), currency');
+        return rows;
+    }
+
+    /**
+     * Returns a list of the MRR history for each day and currency, including the current MRR per currency as meta data.
+     * The respons is in ascending date order, and currencies for the same date are always in ascending order.
+     * @returns {Promise<MrrHistory>}
+     */
+    async getHistory() {
+        // Fetch current total amounts and start counting from there
+        const totals = await this.getCurrentMrr();
+
+        const rows = await this.fetchAllDeltas();
+
+        rows.sort((rowA, rowB) => {
+            const dateA = new Date(rowA.date);
+            const dateB = new Date(rowB.date);
+        
+            return dateA - dateB || rowA.currency.localeCompare(rowB.currency);
+        });
+
+        // Get today in UTC (default timezone)
+        const today = moment().format('YYYY-MM-DD');
+
+        const results = [];
+
+        // Create a map of the totals by currency for fast lookup and editing
+
+        /** @type {Object.<string, number>}*/
+        const currentTotals = {};
+        for (const total of totals) {
+            currentTotals[total.currency] = total.mrr;
+        }
+
+        // Loop in reverse order (needed to have correct sorted result)
+        for (let i = rows.length - 1; i >= 0; i -= 1) {
+            const row = rows[i];
+
+            if (currentTotals[row.currency] === undefined) {
+                // Skip unexpected currencies that are not in the totals
+                continue;
+            }
+
+            // Convert JSDates to YYYY-MM-DD (in UTC)
+            const date = moment(row.date).format('YYYY-MM-DD');
+
+            if (date > today) {
+                // Skip results that are in the future for some reason
+                continue;
+            }
+
+            results.unshift({
+                date,
+                mrr: Math.max(0, currentTotals[row.currency]),
+                currency: row.currency
+            });
+
+            currentTotals[row.currency] -= row.delta;
+        }
+
+        // Now also add the oldest days we have left over and do not have deltas
+        const oldestDate = rows.length > 0 ? moment(rows[0].date).add(-1, 'days').format('YYYY-MM-DD') : today;
+
+        // Note that we also need to loop the totals in reverse order because we need to unshift
+        for (let i = totals.length - 1; i >= 0; i -= 1) {
+            const total = totals[i];
+            results.unshift({
+                date: oldestDate,
+                mrr: Math.max(0, currentTotals[total.currency]),
+                currency: total.currency
+            });
+        }
+
+        return {
+            data: results,
+            meta: {
+                totals
+            }
+        };
+    }
+}
+
+module.exports = MrrStatsService;
+
+/**
+ * @typedef MrrByCurrency
+ * @type {Object}
+ * @property {number} mrr
+ * @property {string} currency
+ */
+
+/**
+ * @typedef MrrDelta
+ * @type {Object}
+ * @property {Date} date
+ * @property {string} currency
+ * @property {number} delta MRR change on this day
+ */
+
+/**
+ * @typedef {Object} MrrRecord
+ * @property {string} date In YYYY-MM-DD format
+ * @property {string} currency
+ * @property {number} mrr MRR on this day
+ */
+
+/**
+ * @typedef {Object} MrrHistory
+ * @property {MrrRecord[]} data List of the total members by status for each day, including the paid deltas paid_subscribed and paid_canceled
+ * @property {Object} meta
+ * @property {MrrByCurrency[]} meta.totals
+ */