ghost 4.22.1 → 4.23.0

package/core/server/services/oembed.js

@@ -1,6 +1,6 @@
-const Promise = require('bluebird');
 const errors = require('@tryghost/errors');
 const tpl = require('@tryghost/tpl');
+const logging = require('@tryghost/logging');
 const {extract, hasProvider} = require('oembed-parser');
 const cheerio = require('cheerio');
 const _ = require('lodash');
@@ -11,6 +11,10 @@ const messages = {
     insufficientMetadata: 'URL contains insufficient metadata.'
 };
 
+/**
+ * @param {string} url
+ * @returns {{url: string, provider: boolean}}
+ */
 const findUrlWithProvider = (url) => {
     let provider;
 
@@ -44,6 +48,12 @@ const findUrlWithProvider = (url) => {
  * @typedef {(url: string, config: Object) => Promise} IExternalRequest
  */
 
+/**
+ * @typedef {object} ICustomProvider
+ * @prop {(url: URL) => Promise<boolean>} canSupportRequest
+ * @prop {(url: URL, externalRequest: IExternalRequest) => Promise<import('oembed-parser').OembedData>} getOEmbedData
+ */
+
 class OEmbed {
     /**
      *
@@ -53,34 +63,51 @@ class OEmbed {
      */
     constructor({config, externalRequest}) {
         this.config = config;
-        this.externalRequest = externalRequest;
+
+        /** @type {IExternalRequest} */
+        this.externalRequest = async (url, requestConfig) => {
+            if (this.isIpOrLocalhost(url)) {
+                return this.unknownProvider(url);
+            }
+            const response = await externalRequest(url, requestConfig);
+            if (this.isIpOrLocalhost(response.url)) {
+                return this.unknownProvider(url);
+            }
+            return response;
+        };
+
+        /** @type {ICustomProvider[]} */
+        this.customProviders = [];
     }
 
-    unknownProvider(url) {
-        return Promise.reject(new errors.ValidationError({
-            message: tpl(messages.unknownProvider),
-            context: url
-        }));
+    /**
+     * @param {ICustomProvider} provider
+     */
+    registerProvider(provider) {
+        this.customProviders.push(provider);
     }
 
-    knownProvider(url) {
-        return extract(url).catch((err) => {
-            return Promise.reject(new errors.InternalServerError({
-                message: err.message
-            }));
+    /**
+     * @param {string} url
+     */
+    async unknownProvider(url) {
+        throw new errors.ValidationError({
+            message: tpl(messages.unknownProvider),
+            context: url
         });
     }
 
-    errorHandler(url) {
-        return (err) => {
-            // allow specific validation errors through for better error messages
-            if (errors.utils.isIgnitionError(err) && err.errorType === 'ValidationError') {
-                return Promise.reject(err);
-            }
-
-            // default to unknown provider to avoid leaking any app specifics
-            return this.unknownProvider(url);
-        };
+    /**
+     * @param {string} url
+     */
+    async knownProvider(url) {
+        try {
+            return await extract(url);
+        } catch (err) {
+            throw new errors.InternalServerError({
+                message: err.message
+            });
+        }
     }
 
     async fetchBookmarkData(url) {
@@ -97,19 +124,11 @@ class OEmbed {
 
         let scraperResponse;
 
-        try {
-            const cookieJar = new CookieJar();
-            const response = await this.externalRequest(url, {cookieJar});
+        const cookieJar = new CookieJar();
+        const response = await this.externalRequest(url, {cookieJar});
 
-            if (this.isIpOrLocalhost(response.url)) {
-                scraperResponse = {};
-            } else {
-                const html = response.body;
-                scraperResponse = await metascraper({html, url});
-            }
-        } catch (err) {
-            return Promise.reject(err);
-        }
+        const html = response.body;
+        scraperResponse = await metascraper({html, url});
 
         const metadata = Object.assign({}, scraperResponse, {
             thumbnail: scraperResponse.image,
@@ -119,20 +138,25 @@ class OEmbed {
         delete metadata.image;
         delete metadata.logo;
 
-        if (metadata.title) {
-            return Promise.resolve({
-                type: 'bookmark',
-                url,
-                metadata
+        if (!metadata.title) {
+            throw new errors.ValidationError({
+                message: tpl(messages.insufficientMetadata),
+                context: url
             });
         }
 
-        return Promise.reject(new errors.ValidationError({
-            message: tpl(messages.insufficientMetadata),
-            context: url
-        }));
+        return {
+            version: '1.0',
+            type: 'bookmark',
+            url,
+            metadata
+        };
     }
 
+    /**
+     * @param {string} url
+     * @returns {boolean}
+     */
     isIpOrLocalhost(url) {
         try {
             const IPV4_REGEX = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
@@ -163,14 +187,7 @@ class OEmbed {
      *
      * @returns {Promise<Object>}
      */
-    fetchOembedData(_url, cardType) {
-        // parse the url then validate the protocol and host to make sure it's
-        // http(s) and not an IP address or localhost to avoid potential access to
-        // internal network endpoints
-        if (this.isIpOrLocalhost(_url)) {
-            return this.unknownProvider();
-        }
-
+    async fetchOembedData(_url, cardType) {
         // check against known oembed list
         let {url, provider} = findUrlWithProvider(_url);
         if (provider) {
@@ -180,88 +197,81 @@ class OEmbed {
         // url not in oembed list so fetch it in case it's a redirect or has a
         // <link rel="alternate" type="application/json+oembed"> element
         const cookieJar = new CookieJar();
-        return this.externalRequest(url, {
+        const pageResponse = await this.externalRequest(url, {
             method: 'GET',
             timeout: 2 * 1000,
             followRedirect: true,
             cookieJar
-        }).then((pageResponse) => {
-            // url changed after fetch, see if we were redirected to a known oembed
-            if (pageResponse.url !== url) {
-                ({url, provider} = findUrlWithProvider(pageResponse.url));
-                if (provider) {
-                    return this.knownProvider(url);
-                }
+        });
+        // url changed after fetch, see if we were redirected to a known oembed
+        if (pageResponse.url !== url) {
+            ({url, provider} = findUrlWithProvider(pageResponse.url));
+            if (provider) {
+                return this.knownProvider(url);
             }
+        }
 
-            // check for <link rel="alternate" type="application/json+oembed"> element
-            let oembedUrl;
-            try {
-                oembedUrl = cheerio('link[type="application/json+oembed"]', pageResponse.body).attr('href');
-            } catch (e) {
-                return this.unknownProvider(url);
+        // check for <link rel="alternate" type="application/json+oembed"> element
+        let oembedUrl;
+        try {
+            oembedUrl = cheerio('link[type="application/json+oembed"]', pageResponse.body).attr('href');
+        } catch (e) {
+            return this.unknownProvider(url);
+        }
+
+        if (oembedUrl) {
+            // for standard WP oembed's we want to insert a bookmark card rather than their blockquote+script
+            // which breaks in the editor and most Ghost themes. Only fallback if card type was not explicitly chosen
+            if (!cardType && oembedUrl.match(/wp-json\/oembed/)) {
+                return;
             }
 
-            if (oembedUrl) {
-                // make sure the linked url is not an ip address or localhost
-                if (this.isIpOrLocalhost(oembedUrl)) {
-                    return this.unknownProvider(oembedUrl);
+            // fetch oembed response from embedded rel="alternate" url
+            const oembedResponse = await this.externalRequest(oembedUrl, {
+                method: 'GET',
+                json: true,
+                timeout: 2 * 1000,
+                followRedirect: true,
+                cookieJar
+            });
+            // validate the fetched json against the oembed spec to avoid
+            // leaking non-oembed responses
+            const body = oembedResponse.body;
+            const hasRequiredFields = body.type && body.version;
+            const hasValidType = ['photo', 'video', 'link', 'rich'].includes(body.type);
+
+            if (hasRequiredFields && hasValidType) {
+                // extract known oembed fields from the response to limit leaking of unrecognised data
+                const knownFields = [
+                    'type',
+                    'version',
+                    'html',
+                    'url',
+                    'title',
+                    'width',
+                    'height',
+                    'author_name',
+                    'author_url',
+                    'provider_name',
+                    'provider_url',
+                    'thumbnail_url',
+                    'thumbnail_width',
+                    'thumbnail_height'
+                ];
+                const oembed = _.pick(body, knownFields);
+
+                // ensure we have required data for certain types
+                if (oembed.type === 'photo' && !oembed.url) {
+                    return;
                 }
-
-                // for standard WP oembed's we want to insert a bookmark card rather than their blockquote+script
-                // which breaks in the editor and most Ghost themes. Only fallback if card type was not explicitly chosen
-                if (!cardType && oembedUrl.match(/wp-json\/oembed/)) {
+                if ((oembed.type === 'video' || oembed.type === 'rich') && (!oembed.html || !oembed.width || !oembed.height)) {
                     return;
                 }
 
-                // fetch oembed response from embedded rel="alternate" url
-                return this.externalRequest(oembedUrl, {
-                    method: 'GET',
-                    json: true,
-                    timeout: 2 * 1000,
-                    followRedirect: true,
-                    cookieJar
-                }).then((oembedResponse) => {
-                    // validate the fetched json against the oembed spec to avoid
-                    // leaking non-oembed responses
-                    const body = oembedResponse.body;
-                    const hasRequiredFields = body.type && body.version;
-                    const hasValidType = ['photo', 'video', 'link', 'rich'].includes(body.type);
-
-                    if (hasRequiredFields && hasValidType) {
-                        // extract known oembed fields from the response to limit leaking of unrecognised data
-                        const knownFields = [
-                            'type',
-                            'version',
-                            'html',
-                            'url',
-                            'title',
-                            'width',
-                            'height',
-                            'author_name',
-                            'author_url',
-                            'provider_name',
-                            'provider_url',
-                            'thumbnail_url',
-                            'thumbnail_width',
-                            'thumbnail_height'
-                        ];
-                        const oembed = _.pick(body, knownFields);
-
-                        // ensure we have required data for certain types
-                        if (oembed.type === 'photo' && !oembed.url) {
-                            return;
-                        }
-                        if ((oembed.type === 'video' || oembed.type === 'rich') && (!oembed.html || !oembed.width || !oembed.height)) {
-                            return;
-                        }
-
-                        // return the extracted object, don't pass through the response body
-                        return oembed;
-                    }
-                }).catch(() => {});
+                // return the extracted object, don't pass through the response body
+                return oembed;
             }
-        });
+        }
     }
 
     /**
@@ -271,26 +281,51 @@ class OEmbed {
      * @returns {Promise<Object>}
      */
     async fetchOembedDataFromUrl(url, type) {
-        let data;
-
         try {
+            const urlObject = new URL(url);
+
+            for (const provider of this.customProviders) {
+                if (await provider.canSupportRequest(urlObject)) {
+                    const result = await provider.getOEmbedData(urlObject, this.externalRequest);
+                    if (result !== null) {
+                        return result;
+                    }
+                }
+            }
+
+            // fetch only bookmark when explicitly requested
             if (type === 'bookmark') {
                 return this.fetchBookmarkData(url);
             }
 
-            data = await this.fetchOembedData(url);
+            // attempt to fetch oembed
+            let data = await this.fetchOembedData(url);
 
+            // fallback to bookmark when we can't get oembed
             if (!data && !type) {
                 data = await this.fetchBookmarkData(url);
             }
 
+            // couldn't get anything, throw a validation error
             if (!data) {
-                data = await this.unknownProvider(url);
+                return this.unknownProvider(url);
             }
 
             return data;
-        } catch (e) {
-            return this.errorHandler(url);
+        } catch (err) {
+            // allow specific validation errors through for better error messages
+            if (errors.utils.isIgnitionError(err) && err.errorType === 'ValidationError') {
+                throw err;
+            }
+
+            // log the real error because we're going to throw a generic "Unknown provider" error
+            logging.error(new errors.GhostError({
+                message: 'Encountered error when fetching oembed',
+                err
+            }));
+
+            // default to unknown provider to avoid leaking any app specifics
+            return this.unknownProvider(url);
         }
     }
 }

package/core/server/services/public-config/config.js

@@ -16,7 +16,8 @@ module.exports = function getConfigProperties() {
         stripeDirect: config.get('stripeDirect'),
         mailgunIsConfigured: config.get('bulkEmail') && config.get('bulkEmail').mailgun,
         emailAnalytics: config.get('emailAnalytics'),
-        hostSettings: config.get('hostSettings')
+        hostSettings: config.get('hostSettings'),
+        tenor: config.get('tenor')
     };
 
     const billingUrl = config.get('hostSettings:billing:enabled') ? config.get('hostSettings:billing:url') : '';

package/core/server/services/stripe/index.js

@@ -27,7 +27,10 @@ function configureApi() {
     }
 }
 
-const debouncedConfigureApi = _.debounce(configureApi, 600);
+const debouncedConfigureApi = _.debounce(() => {
+    configureApi();
+    events.emit('services.stripe.reconfigured');
+}, 600);
 
 module.exports = {
     async init() {
@@ -37,7 +40,6 @@ module.exports = {
                 return;
             }
             debouncedConfigureApi();
-            events.emit('services.stripe.reconfigured');
         });
     },
 

package/core/server/services/url/Resource.js

@@ -8,7 +8,7 @@ const errors = require('@tryghost/errors');
 class Resource extends EventEmitter {
     /**
      * @param {('posts'|'pages'|'tags'|'authors')} type - of the resource
-     * @param {Object} obj - object data to sotre
+     * @param {Object} obj - object data to store
      */
     constructor(type, obj) {
         super();

package/core/server/services/url/Resources.js

@@ -17,10 +17,16 @@ const events = require('../../lib/common/events');
  * Each entry in the database will be represented by a "Resource" (see /Resource.js).
  */
 class Resources {
-    constructor(queue) {
+    /**
+     *
+     * @param {Object} options
+     * @param {Object} [options.resources] - resources to initialize with instead of fetching them from the database
+     * @param {Object} [options.queue] - instance of the Queue class
+     */
+    constructor({resources = {}, queue} = {}) {
         this.queue = queue;
         this.resourcesConfig = [];
-        this.data = {};
+        this.data = resources;
 
         this.listeners = [];
     }
@@ -43,38 +49,57 @@ class Resources {
     }
 
     /**
-     * @description Initialise the resource config. We currently fetch the data straight via the the model layer,
+     * @description Initialize the resource config. We currently fetch the data straight via the the model layer,
      *              but because Ghost supports multiple API versions, we have to ensure we load the correct data.
      *
      * @TODO: https://github.com/TryGhost/Ghost/issues/10360
-     * @private
      */
-    _initResourceConfig() {
+    initResourceConfig() {
         if (!_.isEmpty(this.resourcesConfig)) {
             return;
         }
 
         const bridge = require('../../../bridge');
-        this.resourcesAPIVersion = bridge.getFrontendApiVersion();
-        this.resourcesConfig = require(`./configs/${this.resourcesAPIVersion}`);
+        const resourcesAPIVersion = bridge.getFrontendApiVersion();
+        this.resourcesConfig = require(`./configs/${resourcesAPIVersion}`);
     }
 
     /**
-     * @description Helper function to initialise data fetching. Each resource type needs to register resource/model
-     *              events to get notified about updates/deletions/inserts.
+     * @description Helper function to initialize data fetching.
      */
     fetchResources() {
         const ops = [];
         debug('fetchResources');
 
-        this._initResourceConfig();
-
         // NOTE: Iterate over all resource types (posts, users etc..) and call `_fetch`.
         _.each(this.resourcesConfig, (resourceConfig) => {
             this.data[resourceConfig.type] = [];
 
             // NOTE: We are querying knex directly, because the Bookshelf ORM overhead is too slow.
             ops.push(this._fetch(resourceConfig));
+        });
+
+        return Promise.all(ops);
+    }
+
+    /**
+     * @description Each resource type needs to register resource/model events to get notified
+     * about updates/deletions/inserts.
+     *
+     * For example for a "tag" resource type with following configuration:
+     *  events: {
+     *     add: 'tag.added',
+     *     update: ['tag.edited', 'tag.attached', 'tag.detached'],
+     *     remove: 'tag.deleted'
+     *  }
+     * there would be:
+     * 1 event listener connected to  "_onResourceAdded"   handler and it's 'tag.added' event
+     * 3 event listeners connected to "_onResourceUpdated" handler and it's 'tag.edited', 'tag.attached', 'tag.detached' events
+     * 1 event listener connected to  "_onResourceRemoved" handler and it's 'tag.deleted' event
+     */
+    initEvenListeners() {
+        _.each(this.resourcesConfig, (resourceConfig) => {
+            this.data[resourceConfig.type] = [];
 
             this._listenOn(resourceConfig.events.add, (model) => {
                 return this._onResourceAdded.bind(this)(resourceConfig.type, model);
@@ -96,16 +121,6 @@ class Resources {
                 return this._onResourceRemoved.bind(this)(resourceConfig.type, model);
             });
         });
-
-        Promise.all(ops)
-            .then(() => {
-                // CASE: all resources are fetched, start the queue
-                this.queue.start({
-                    event: 'init',
-                    tolerance: 100,
-                    requiredSubscriberCount: 1
-                });
-            });
     }
 
     /**
@@ -430,8 +445,6 @@ class Resources {
      * @description Reset this class instance.
      *
      * Is triggered if you switch API versions.
-     *
-     * @param {Object} options
      */
     reset() {
         _.each(this.listeners, (obj) => {

package/core/server/services/url/UrlGenerator.js

@@ -33,18 +33,29 @@ const EXPANSIONS = [{
  * Each router is represented by a url generator.
  */
 class UrlGenerator {
-    constructor(router, queue, resources, urls, position) {
-        this.router = router;
+    /**
+     * @param {Object} options
+     * @param {String} options.identifier frontend router ID reference
+     * @param {String} options.filter NQL filter string
+     * @param {String} options.resourceType resource type (e.g. 'posts', 'tags')
+     * @param {String} options.permalink permalink string
+     * @param {Object} options.queue instance of the backend Queue
+     * @param {Object} options.resources instance of the backend Resources
+     * @param {Object} options.urls instance of the backend URLs (used to store the urls)
+     * @param {Number} options.position an ID of the generator
+     */
+    constructor({identifier, filter, resourceType, permalink, queue, resources, urls, position}) {
+        this.identifier = identifier;
+        this.resourceType = resourceType;
+        this.permalink = permalink;
         this.queue = queue;
         this.urls = urls;
         this.resources = resources;
         this.uid = position;
 
-        debug('constructor', this.toString());
-
         // CASE: routers can define custom filters, but not required.
-        if (this.router.getFilter()) {
-            this.filter = this.router.getFilter();
+        if (filter) {
+            this.filter = filter;
             this.nql = nql(this.filter, {
                 expansions: EXPANSIONS,
                 transformer: nql.utils.mapKeyValues({
@@ -110,10 +121,10 @@ class UrlGenerator {
      * @private
      */
     _onInit() {
-        debug('_onInit', this.router.getResourceType());
+        debug('_onInit', this.resourceType);
 
         // @NOTE: get the resources of my type e.g. posts.
-        const resources = this.resources.getAllByType(this.router.getResourceType());
+        const resources = this.resources.getAllByType(this.resourceType);
 
         debug(resources.length);
 
@@ -131,7 +142,7 @@ class UrlGenerator {
         debug('onAdded', this.toString());
 
         // CASE: you are type "pages", but the incoming type is "users"
-        if (event.type !== this.router.getResourceType()) {
+        if (event.type !== this.resourceType) {
             return;
         }
 
@@ -182,8 +193,7 @@ class UrlGenerator {
      * @NOTE We currently generate relative urls (https://github.com/TryGhost/Ghost/commit/7b0d5d465ba41073db0c3c72006da625fa11df32).
      */
     _generateUrl(resource) {
-        const permalink = this.router.getPermalinks().getValue();
-        return localUtils.replacePermalink(permalink, resource.data);
+        return localUtils.replacePermalink(this.permalink, resource.data);
     }
 
     /**
@@ -214,7 +224,7 @@ class UrlGenerator {
                 action: 'added:' + resource.data.id,
                 eventData: {
                     id: resource.data.id,
-                    type: this.router.getResourceType()
+                    type: this.resourceType
                 }
             });
         };
@@ -246,19 +256,12 @@ class UrlGenerator {
 
     /**
      * @description Get all urls of this url generator.
+     * NOTE: the method is only used for testing purposes at the moment.
      * @returns {Array}
      */
     getUrls() {
         return this.urls.getByGeneratorId(this.uid);
     }
-
-    /**
-     * @description Override of `toString`
-     * @returns {string}
-     */
-    toString() {
-        return this.router.toString();
-    }
 }
 
 module.exports = UrlGenerator;