ghost 5.36.1 → 5.38.0

package/core/server/data/schema/commands.js

@@ -6,7 +6,6 @@ const tpl = require('@tryghost/tpl');
 const db = require('../db');
 const DatabaseInfo = require('@tryghost/database-info');
 const schema = require('./schema');
-const clients = require('./clients');
 
 const messages = {
     hasPrimaryKeySQLiteError: 'Must use hasPrimaryKeySQLite on an SQLite3 database',
@@ -432,11 +431,15 @@ function deleteTable(table, transaction = db.knex) {
 /**
  * @param {import('knex').Knex} [transaction] - connection to the DB
  */
-function getTables(transaction = db.knex) {
+async function getTables(transaction = db.knex) {
     const client = transaction.client.config.client;
 
-    if (_.includes(_.keys(clients), client)) {
-        return clients[client].getTables(transaction);
+    if (client === 'sqlite3') {
+        const response = await transaction.raw('select * from sqlite_master where type = "table"');
+        return _.reject(_.map(response, 'tbl_name'), name => name === 'sqlite_sequence');
+    } else if (client === 'mysql2') {
+        const response = await transaction.raw('show tables');
+        return _.flatten(_.map(response[0], entry => _.values(entry)));
     }
 
     return Promise.reject(tpl(messages.noSupportForDatabase, {client: client}));
@@ -446,11 +449,15 @@ function getTables(transaction = db.knex) {
  * @param {string} table
  * @param {import('knex').Knex} [transaction] - connection to the DB
  */
-function getIndexes(table, transaction = db.knex) {
+async function getIndexes(table, transaction = db.knex) {
     const client = transaction.client.config.client;
 
-    if (_.includes(_.keys(clients), client)) {
-        return clients[client].getIndexes(table, transaction);
+    if (client === 'sqlite3') {
+        const response = await transaction.raw(`pragma index_list("${table}")`);
+        return _.flatten(_.map(response, 'name'));
+    } else if (client === 'mysql2') {
+        const response = await transaction.raw(`SHOW INDEXES from ${table}`);
+        return _.flatten(_.map(response[0], 'Key_name'));
     }
 
     return Promise.reject(tpl(messages.noSupportForDatabase, {client: client}));
@@ -460,11 +467,15 @@ function getIndexes(table, transaction = db.knex) {
  * @param {string} table
  * @param {import('knex').Knex} [transaction] - connection to the DB
  */
-function getColumns(table, transaction = db.knex) {
+async function getColumns(table, transaction = db.knex) {
     const client = transaction.client.config.client;
 
-    if (_.includes(_.keys(clients), client)) {
-        return clients[client].getColumns(table);
+    if (client === 'sqlite3') {
+        const response = await transaction.raw(`pragma table_info("${table}")`);
+        return _.flatten(_.map(response, 'name'));
+    } else if (client === 'mysql2') {
+        const response = await transaction.raw(`SHOW COLUMNS from ${table}`);
+        return _.flatten(_.map(response[0], 'Field'));
     }
 
     return Promise.reject(tpl(messages.noSupportForDatabase, {client: client}));

package/core/server/lib/common/events.js

@@ -12,29 +12,22 @@
  */
 
 const events = require('events');
-const util = require('util');
-let EventRegistry;
-let EventRegistryInstance;
 
-EventRegistry = function () {
-    events.EventEmitter.call(this);
-};
+class EventRegistry extends events.EventEmitter {
+    /**
+     * This is method is semi-hack to make sure listeners are only registered once
+     * during the lifetime of the process. And example problem it solves is
+     * registering duplicate listeners between Ghost instance reboots when running tests.
+     * @param {String} eventName
+     * @param {String} listenerName named function name registered as a listener for the event
+     * @returns {Boolean}
+     */
+    hasRegisteredListener(eventName, listenerName) {
+        return !!(this.listeners(eventName).find(listener => (listener.name === listenerName)));
+    }
+}
 
-util.inherits(EventRegistry, events.EventEmitter);
+const eventRegistryInstance = new EventRegistry();
+eventRegistryInstance.setMaxListeners(100);
 
-/**
- * This is method is semi-hack to make sure listeners are only registered once
- * during the lifetime of the process. And example problem it solves is
- * registering duplicate listeners between Ghost instance reboots when running tests.
- * @param {String} eventName
- * @param {String} listenerName named function name registered as a listener for the event
- * @returns {Boolean}
- */
-EventRegistry.prototype.hasRegisteredListener = function (eventName, listenerName) {
-    return !!(this.listeners(eventName).find(listener => (listener.name === listenerName)));
-};
-
-EventRegistryInstance = new EventRegistry();
-EventRegistryInstance.setMaxListeners(100);
-
-module.exports = EventRegistryInstance;
+module.exports = eventRegistryInstance;

package/core/server/models/base/plugins/relations.js

@@ -7,7 +7,7 @@ module.exports = function (Bookshelf) {
          * Return a relation, and load it if it hasn't been loaded already (or force a refresh with the forceRefresh option).
          * refs https://github.com/TryGhost/Team/issues/1626
          * @param {string} name Name of the relation to load
-         * @param {Object} [options] Options to pass to the fetch when not yet loaded (or when force refreshing) 
+         * @param {Object} [options] Options to pass to the fetch when not yet loaded (or when force refreshing)
          * @param {boolean} [options.forceRefresh] If true, the relation will be fetched again even if it has already been loaded.
          * @returns {Promise<import('bookshelf').Model|import('bookshelf').Collection|null>}
          */
@@ -23,8 +23,10 @@ module.exports = function (Bookshelf) {
             // Not yet loaded, or force refresh
             // Note that we don't use .refresh on the relation on options.forceRefresh
             // Because the relation can also be a collection, which doesn't have a refresh method
-            this.relations[name] = this[name]();
-            return this.relations[name].fetch(options);
+            const instance = this[name]();
+            await instance.fetch(options);
+            this.relations[name] = instance;
+            return instance;
         }
     });
 };

package/core/server/models/index.js

@@ -3,6 +3,7 @@
  */
 
 const _ = require('lodash');
+const debug = require('@tryghost/debug')('models');
 const glob = require('glob');
 
 // enable event listeners
@@ -14,12 +15,16 @@ require('./base/listeners');
 exports = module.exports;
 
 function init() {
+    const baseNow = Date.now();
     exports.Base = require('./base');
+    debug(`${Date.now() - baseNow}ms - Base.js require`);
 
     let modelsFiles = glob.sync('!(index).js', {cwd: __dirname});
     modelsFiles.forEach((model) => {
         const name = model.replace(/.js$/, '');
+        const modelNow = Date.now();
         _.extend(exports, require('./' + name));
+        debug(`${Date.now() - modelNow}ms - ${model} require`);
     });
 }
 

package/core/server/models/mention.js

@@ -9,6 +9,19 @@ const Mention = ghostBookshelf.Model.extend({
     enforcedFilters() {
         return 'deleted:false';
     }
+}, {
+    permittedOptions(methodName) {
+        let options = ghostBookshelf.Model.permittedOptions.call(this, methodName);
+        const validOptions = {
+            findPage: ['selectRaw', 'whereRaw']
+        };
+
+        if (validOptions[methodName]) {
+            options = options.concat(validOptions[methodName]);
+        }
+
+        return options;
+    }
 });
 
 module.exports = {

package/core/server/run-update-check.js

@@ -42,7 +42,9 @@ if (parentPort) {
     await settings.init();
     // Finished INIT
 
-    await updateCheck();
+    await updateCheck({
+        rethrowErrors: true
+    });
 
     postParentPortMessage(`Ran update check`);
 

package/core/server/services/comments/emails.js

@@ -2,7 +2,7 @@ const {promises: fs} = require('fs');
 const path = require('path');
 const moment = require('moment');
 const htmlToPlaintext = require('@tryghost/html-to-plaintext');
-const postEmailSerializer = require('../mega/post-email-serializer');
+const emailService = require('../email-service');
 
 class CommentsServiceEmails {
     constructor({config, logging, models, mailer, settingsCache, settingsHelpers, urlService, urlUtils}) {
@@ -95,7 +95,7 @@ class CommentsServiceEmails {
             accentColor: this.settingsCache.get('accent_color'),
             fromEmail: this.notificationFromAddress,
             toEmail: to,
-            profileUrl: postEmailSerializer.createUnsubscribeUrl(member.get('uuid'), {comments: true})
+            profileUrl: emailService.renderer.createUnsubscribeUrl(member.get('uuid'), {comments: true})
         };
 
         const {html, text} = await this.renderEmailTemplate('new-comment-reply', templateData);

package/core/server/services/email-service/wrapper.js

@@ -94,6 +94,8 @@ class EmailServiceWrapper {
             sentry
         });
 
+        this.renderer = emailRenderer;
+
         this.service = new EmailService({
             batchSendingService,
             sendingService,

package/core/server/services/link-tracking/LinkClickRepository.js

@@ -56,7 +56,7 @@ module.exports = class LinkClickRepository {
         }
 
         const model = await this.#MemberLinkClickEventModel.add({
-            // Only store the parthname (no support for variable query strings)
+            // Only store the pathname (no support for variable query strings)
             redirect_id: linkClick.link_id.toHexString(),
             member_id: member.id
         }, {});

package/core/server/services/media-inliner/index.js

@@ -0,0 +1 @@
+module.exports = require('./service');

package/core/server/services/media-inliner/service.js

@@ -0,0 +1,62 @@
+module.exports = {
+    async init() {
+        const debug = require('@tryghost/debug')('mediaInliner');
+        const MediaInliner = require('@tryghost/external-media-inliner');
+        const models = require('../../models');
+        const jobsService = require('../jobs');
+
+        const mediaStorage = require('../../adapters/storage').getStorage('media');
+        const imageStorage = require('../../adapters/storage').getStorage('images');
+        const fileStorage = require('../../adapters/storage').getStorage('files');
+
+        const config = require('../../../shared/config');
+
+        const mediaInliner = new MediaInliner({
+            PostModel: models.Post,
+            TagModel: models.Tag,
+            UserModel: models.User,
+            PostMetaModel: models.PostsMeta,
+            getMediaStorage: (extension) => {
+                if (config.get('uploads').images.extensions.includes(extension)) {
+                    return imageStorage;
+                } else if (config.get('uploads').media.extensions.includes(extension)) {
+                    return mediaStorage;
+                } else if (config.get('uploads').files.extensions.includes(extension)) {
+                    return fileStorage;
+                } else {
+                    return null;
+                }
+            }
+        });
+
+        this.api = {
+
+            startMediaInliner: async (domains) => {
+                if (!domains || !domains.length) {
+                    // default domains to inline from if none are provided
+                    domains = [
+                        'https://s3.amazonaws.com/revue',
+                        'https://substackcdn.com'
+                    ];
+                }
+
+                debug('[Inliner] Starting media inlining job for domains: ', domains);
+
+                // @NOTE: the job is "inline" (aka non-offloaded into a thread), because usecases are currently
+                //        limited to migrational, so there is no expectations for site's availability etc.
+                await jobsService.addJob({
+                    name: 'external-media-inliner',
+                    job: (data) => {
+                        return mediaInliner.inline(data.domains);
+                    },
+                    data: {domains},
+                    offloaded: false
+                });
+
+                return {
+                    status: 'success'
+                };
+            }
+        };
+    }
+};

package/core/server/services/members/stats/members-stats.js

@@ -1,5 +1,4 @@
 const moment = require('moment-timezone');
-const Promise = require('bluebird');
 
 const dateFormat = 'YYYY-MM-DD HH:mm:ss';
 class MembersStats {
@@ -131,14 +130,19 @@ class MembersStats {
         const totalMembers = await this.getTotalMembers();
 
         // perform final calculations in parallel
-        const results = await Promise.props({
-            total: totalMembers,
-            total_in_range: this.getTotalMembersInRange({days, totalMembers, siteTimezone}),
-            total_on_date: this.getTotalMembersOnDatesInRange({days, totalMembers, siteTimezone}),
-            new_today: this.getNewMembersToday({siteTimezone})
-        });
-
-        return results;
+        const [total, totalInRange, totalOnDate, newToday] = await Promise.all([
+            totalMembers,
+            this.getTotalMembersInRange({days, totalMembers, siteTimezone}),
+            this.getTotalMembersOnDatesInRange({days, totalMembers, siteTimezone}),
+            this.getNewMembersToday({siteTimezone})
+        ]);
+
+        return {
+            total,
+            total_in_range: totalInRange,
+            total_on_date: totalOnDate,
+            new_today: newToday
+        };
     }
 }
 

package/core/server/services/mentions/BookshelfMentionRepository.js

@@ -65,7 +65,18 @@ module.exports = class BookshelfMentionRepository {
      * @returns {Promise<Page<import('@tryghost/webmentions/lib/Mention')>>}
      */
     async getPage(options) {
-        const page = await this.#MentionModel.findPage(options);
+        /**
+         * @type {GetPageOptions & {whereRaw?: string}}
+         */
+        const _options = {
+            ...options
+        };
+        delete _options.unique;
+        if (options.unique) {
+            _options.whereRaw = 'NOT EXISTS (select id from mentions as m where m.id > mentions.id and m.source = mentions.source)';
+        }
+
+        const page = await this.#MentionModel.findPage(_options);
 
         return {
             data: await Promise.all(page.data.map(model => this.#modelToMention(model))),

package/core/server/services/mentions/MentionController.js

@@ -80,11 +80,17 @@ module.exports = class MentionController {
             order = 'created_at asc';
         }
 
+        let unique;
+        if (frame.options.unique && (frame.options.unique === 'true' || frame.options.unique === true)) {
+            unique = true;
+        }
+
         const mentions = await this.#api.listMentions({
             filter: frame.options.filter,
             order,
             limit,
-            page
+            page,
+            unique
         });
 
         const resources = await Promise.all(mentions.data.map((mention) => {

package/core/server/services/mentions/WebmentionMetadata.js

@@ -6,14 +6,15 @@ module.exports = class WebmentionMetadata {
      * @returns {Promise<import('@tryghost/webmentions/lib/MentionsAPI').WebmentionMetadata>}
      */
     async fetch(url) {
-        const data = await oembedService.fetchOembedDataFromUrl(url.href, 'bookmark');
+        const data = await oembedService.fetchOembedDataFromUrl(url.href, 'mention');
         const result = {
             siteTitle: data.metadata.publisher,
             title: data.metadata.title,
             excerpt: data.metadata.description,
             author: data.metadata.author,
             image: data.metadata.thumbnail ? new URL(data.metadata.thumbnail) : null,
-            favicon: data.metadata.icon ? new URL(data.metadata.icon) : null
+            favicon: data.metadata.icon ? new URL(data.metadata.icon) : null,
+            body: data.body
         };
         return result;
     }

package/core/server/services/mentions/service.js

@@ -1,6 +1,5 @@
 const MentionController = require('./MentionController');
 const WebmentionMetadata = require('./WebmentionMetadata');
-const WebmentionRequest = require('./WebmentionRequest');
 const {
     MentionsAPI,
     MentionSendingService,
@@ -27,13 +26,17 @@ function getPostUrl(post) {
 
 module.exports = {
     controller: new MentionController(),
+    didInit: false,
     async init() {
+        if (this.didInit) {
+            return;
+        }
+        this.didInit = true;
         const repository = new BookshelfMentionRepository({
             MentionModel: models.Mention,
             DomainEvents
         });
         const webmentionMetadata = new WebmentionMetadata();
-        const webmentionRequest = new WebmentionRequest();
         const discoveryService = new MentionDiscoveryService({externalRequest});
         const resourceService = new ResourceService({
             urlUtils,
@@ -49,7 +52,6 @@ module.exports = {
         const api = new MentionsAPI({
             repository,
             webmentionMetadata,
-            webmentionRequest,
             resourceService,
             routingService
         });

package/core/server/services/posts/posts-service.js

@@ -8,8 +8,7 @@ const messages = {
 };
 
 class PostsService {
-    constructor({mega, urlUtils, models, isSet, stats, emailService}) {
-        this.mega = mega;
+    constructor({urlUtils, models, isSet, stats, emailService}) {
         this.urlUtils = urlUtils;
         this.models = models;
         this.isSet = isSet;
@@ -45,17 +44,9 @@ class PostsService {
                 let email;
 
                 if (!postEmail) {
-                    if (this.isSet('emailStability')) {
-                        email = await this.emailService.createEmail(model);
-                    } else {
-                        email = await this.mega.addEmail(model, frame.options);
-                    }
+                    email = await this.emailService.createEmail(model);
                 } else if (postEmail && postEmail.get('status') === 'failed') {
-                    if (this.isSet('emailStability')) {
-                        email = await this.emailService.retryEmail(postEmail);
-                    } else {
-                        email = await this.mega.retryFailedEmail(postEmail);
-                    }
+                    email = await this.emailService.retryEmail(postEmail);
                 }
                 if (email) {
                     model.set('email', email);
@@ -130,7 +121,6 @@ class PostsService {
  */
 const getPostServiceInstance = () => {
     const urlUtils = require('../../../shared/url-utils');
-    const {mega} = require('../mega');
     const labs = require('../../../shared/labs');
     const models = require('../../models');
     const PostStats = require('./stats/post-stats');
@@ -139,7 +129,6 @@ const getPostServiceInstance = () => {
     const postStats = new PostStats();
 
     return new PostsService({
-        mega: mega,
         urlUtils: urlUtils,
         models: models,
         isSet: flag => labs.isSet(flag), // don't use bind, that breaks test subbing of labs

package/core/server/{analytics-events.js → services/segment/index.js}

@@ -1,11 +1,12 @@
 const _ = require('lodash');
 const Analytics = require('analytics-node');
-const config = require('../shared/config');
 const logging = require('@tryghost/logging');
-const sentry = require('../shared/sentry');
+
+const config = require('../../../shared/config');
+const sentry = require('../../../shared/sentry');
 
 // Listens to model events to layer on analytics - also uses the "fake" theme.uploaded event from the theme API
-const events = require('./lib/common/events');
+const events = require('../../lib/common/events');
 
 module.exports.init = function () {
     const analytics = new Analytics(config.get('segment:key'));

package/core/server/services/staff/index.js

@@ -12,6 +12,7 @@ class StaffServiceWrapper {
 
         const logging = require('@tryghost/logging');
         const models = require('../../models');
+        const memberAttribution = require('../member-attribution');
         const {GhostMailer} = require('../mail');
         const mailer = new GhostMailer();
         const settingsCache = require('../../../shared/settings-cache');
@@ -26,6 +27,7 @@ class StaffServiceWrapper {
             settingsCache,
             urlUtils,
             DomainEvents,
+            memberAttributionService: memberAttribution.service,
             labs
         });
 

package/core/server/services/stripe/config.js

@@ -1,5 +1,6 @@
 const logging = require('@tryghost/logging');
 const tpl = require('@tryghost/tpl');
+const labs = require('../../../shared/labs');
 
 const messages = {
     remoteWebhooksInDevelopment: 'Cannot use remote webhooks in development. See https://ghost.org/docs/webhooks/#stripe-webhooks for developing with Stripe.'
@@ -64,6 +65,9 @@ module.exports = {
             ...keys,
             ...urls,
             enablePromoCodes: config.get('enableStripePromoCodes'),
+            get enableAutomaticTax() {
+                return labs.isSet('stripeAutomaticTax');
+            },
             webhookSecret: webhookSecret,
             webhookHandlerUrl: webhookHandlerUrl.href
         };

package/core/server/services/url/Urls.js

@@ -44,10 +44,18 @@ class Urls {
         debug('cache', url);
 
         if (this.urls[resource.data.id]) {
-            logging.error(new errors.InternalServerError({
+            const error = new errors.InternalServerError({
                 message: 'This should not happen.',
                 code: 'URLSERVICE_RESOURCE_DUPLICATE'
-            }));
+            });
+            if (process.env.NODE_ENV.startsWith('test')) {
+                logging.warn({
+                    message: 'Duplicate URL',
+                    err: error
+                });
+            } else {
+                logging.error(error);
+            }
 
             this.removeResourceId(resource.data.id);
         }

package/core/server/update-check.js

@@ -12,10 +12,11 @@ const UpdateCheckService = require('@tryghost/update-check-service');
 
 /**
  * Initializes and triggers update check
- *
+ * @param {Object} [options]
+ * @param {Boolean} [options.rethrowErrors] - if true, errors will be thrown instead of logged
  * @returns {Promise<any>}
  */
-module.exports = async () => {
+module.exports = async ({rethrowErrors = false} = {}) => {
     const allowedCheckEnvironments = ['development', 'production'];
 
     // CASE: The check will not happen if your NODE_ENV is not in the allowed defined environments.
@@ -51,7 +52,8 @@ module.exports = async () => {
             notificationGroups: config.get('notificationGroups'),
             siteUrl: urlUtils.urlFor('home', true),
             forceUpdate: config.get('updateCheck:forceUpdate'),
-            ghostVersion: ghostVersion.original
+            ghostVersion: ghostVersion.original,
+            rethrowErrors
         },
         request,
         sendEmail: ghostMailer.send.bind(ghostMailer)

package/core/server/web/api/endpoints/admin/app.js

@@ -1,10 +1,11 @@
 const debug = require('@tryghost/debug')('web:endpoints:admin:app');
 const boolParser = require('express-query-boolean');
-const express = require('../../../../../shared/express');
 const bodyParser = require('body-parser');
-const shared = require('../../../shared');
-const apiMw = require('../../middleware');
 const errorHandler = require('@tryghost/mw-error-handler');
+const versionMatch = require('@tryghost/mw-version-match');
+
+const shared = require('../../../shared');
+const express = require('../../../../../shared/express');
 const sentry = require('../../../../../shared/sentry');
 const routes = require('./routes');
 const APIVersionCompatibilityService = require('../../../../services/api-version-compatibility');
@@ -24,7 +25,7 @@ module.exports = function setupApiApp() {
 
     // Check version matches for API requests, depends on res.locals.safeVersion being set
     // Therefore must come after themeHandler.ghostLocals, for now
-    apiApp.use(apiMw.versionMatch);
+    apiApp.use(versionMatch);
 
     // Admin API shouldn't be cached
     apiApp.use(shared.middleware.cacheControl('private'));

package/core/server/web/api/endpoints/admin/routes.js

@@ -205,6 +205,12 @@ module.exports = function apiRoutes() {
         http(api.db.backupContent)
     );
 
+    router.post('/db/media/inline',
+        mw.authAdminApi,
+        labs.enabledMiddleware('mediaInliner'),
+        http(api.db.inlineMedia)
+    );
+
     // ## Slack
     router.post('/slack/test', mw.authAdminApi, http(api.slack.sendTest));
 

package/core/server/web/api/middleware/index.js

@@ -1,6 +1,5 @@
 module.exports = {
     cors: require('./cors'),
     updateUserLastSeen: require('./update-user-last-seen'),
-    upload: require('./upload'),
-    versionMatch: require('./version-match')
+    upload: require('./upload')
 };