ghost 4.21.0 → 4.22.3

package/core/server/services/oembed.js

@@ -1,4 +1,3 @@
-const Promise = require('bluebird');
 const errors = require('@tryghost/errors');
 const tpl = require('@tryghost/tpl');
 const {extract, hasProvider} = require('oembed-parser');
@@ -11,6 +10,10 @@ const messages = {
     insufficientMetadata: 'URL contains insufficient metadata.'
 };
 
+/**
+ * @param {string} url
+ * @returns {{url: string, provider: boolean}}
+ */
 const findUrlWithProvider = (url) => {
     let provider;
 
@@ -44,6 +47,12 @@ const findUrlWithProvider = (url) => {
  * @typedef {(url: string, config: Object) => Promise} IExternalRequest
  */
 
+/**
+ * @typedef {object} ICustomProvider
+ * @prop {(url: URL) => Promise<boolean>} canSupportRequest
+ * @prop {(url: URL, externalRequest: IExternalRequest) => Promise<import('oembed-parser').OembedData>} getOEmbedData
+ */
+
 class OEmbed {
     /**
      *
@@ -53,29 +62,62 @@ class OEmbed {
      */
     constructor({config, externalRequest}) {
         this.config = config;
-        this.externalRequest = externalRequest;
+        /** @type {IExternalRequest} */
+        this.externalRequest = async (url, requestConfig) => {
+            if (this.isIpOrLocalhost(url)) {
+                return this.unknownProvider(url);
+            }
+            const response = await externalRequest(url, requestConfig);
+            if (this.isIpOrLocalhost(response.url)) {
+                return this.unknownProvider(url);
+            }
+            return response;
+        };
+        /** @type {ICustomProvider[]} */
+        this.customProviders = [];
+    }
+
+    /**
+     * @param {ICustomProvider} provider
+     */
+    registerProvider(provider) {
+        this.customProviders.push(provider);
     }
 
-    unknownProvider(url) {
-        return Promise.reject(new errors.ValidationError({
+    /**
+     * @param {string} url
+     */
+    async unknownProvider(url) {
+        throw new errors.ValidationError({
             message: tpl(messages.unknownProvider),
             context: url
-        }));
+        });
     }
 
-    knownProvider(url) {
-        return extract(url).catch((err) => {
-            return Promise.reject(new errors.InternalServerError({
+    /**
+     * @param {string} url
+     */
+    async knownProvider(url) {
+        try {
+            return await extract(url);
+        } catch (err) {
+            throw new errors.InternalServerError({
                 message: err.message
-            }));
-        });
+            });
+        }
     }
 
+    /**
+     * @param {string} url
+     */
     errorHandler(url) {
-        return (err) => {
+        /**
+         * @param {Error|errors.GhostError} err
+         */
+        return async (err) => {
             // allow specific validation errors through for better error messages
             if (errors.utils.isIgnitionError(err) && err.errorType === 'ValidationError') {
-                return Promise.reject(err);
+                throw err;
             }
 
             // default to unknown provider to avoid leaking any app specifics
@@ -97,19 +139,11 @@ class OEmbed {
 
         let scraperResponse;
 
-        try {
-            const cookieJar = new CookieJar();
-            const response = await this.externalRequest(url, {cookieJar});
+        const cookieJar = new CookieJar();
+        const response = await this.externalRequest(url, {cookieJar});
 
-            if (this.isIpOrLocalhost(response.url)) {
-                scraperResponse = {};
-            } else {
-                const html = response.body;
-                scraperResponse = await metascraper({html, url});
-            }
-        } catch (err) {
-            return Promise.reject(err);
-        }
+        const html = response.body;
+        scraperResponse = await metascraper({html, url});
 
         const metadata = Object.assign({}, scraperResponse, {
             thumbnail: scraperResponse.image,
@@ -119,20 +153,25 @@ class OEmbed {
         delete metadata.image;
         delete metadata.logo;
 
-        if (metadata.title) {
-            return Promise.resolve({
-                type: 'bookmark',
-                url,
-                metadata
+        if (!metadata.title) {
+            throw new errors.ValidationError({
+                message: tpl(messages.insufficientMetadata),
+                context: url
             });
         }
 
-        return Promise.reject(new errors.ValidationError({
-            message: tpl(messages.insufficientMetadata),
-            context: url
-        }));
+        return {
+            version: '1.0',
+            type: 'bookmark',
+            url,
+            metadata
+        };
     }
 
+    /**
+     * @param {string} url
+     * @returns {boolean}
+     */
     isIpOrLocalhost(url) {
         try {
             const IPV4_REGEX = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
@@ -163,14 +202,7 @@ class OEmbed {
      *
      * @returns {Promise<Object>}
      */
-    fetchOembedData(_url, cardType) {
-        // parse the url then validate the protocol and host to make sure it's
-        // http(s) and not an IP address or localhost to avoid potential access to
-        // internal network endpoints
-        if (this.isIpOrLocalhost(_url)) {
-            return this.unknownProvider();
-        }
-
+    async fetchOembedData(_url, cardType) {
         // check against known oembed list
         let {url, provider} = findUrlWithProvider(_url);
         if (provider) {
@@ -180,88 +212,81 @@ class OEmbed {
         // url not in oembed list so fetch it in case it's a redirect or has a
         // <link rel="alternate" type="application/json+oembed"> element
         const cookieJar = new CookieJar();
-        return this.externalRequest(url, {
+        const pageResponse = await this.externalRequest(url, {
             method: 'GET',
             timeout: 2 * 1000,
             followRedirect: true,
             cookieJar
-        }).then((pageResponse) => {
-            // url changed after fetch, see if we were redirected to a known oembed
-            if (pageResponse.url !== url) {
-                ({url, provider} = findUrlWithProvider(pageResponse.url));
-                if (provider) {
-                    return this.knownProvider(url);
-                }
+        });
+        // url changed after fetch, see if we were redirected to a known oembed
+        if (pageResponse.url !== url) {
+            ({url, provider} = findUrlWithProvider(pageResponse.url));
+            if (provider) {
+                return this.knownProvider(url);
             }
+        }
 
-            // check for <link rel="alternate" type="application/json+oembed"> element
-            let oembedUrl;
-            try {
-                oembedUrl = cheerio('link[type="application/json+oembed"]', pageResponse.body).attr('href');
-            } catch (e) {
-                return this.unknownProvider(url);
+        // check for <link rel="alternate" type="application/json+oembed"> element
+        let oembedUrl;
+        try {
+            oembedUrl = cheerio('link[type="application/json+oembed"]', pageResponse.body).attr('href');
+        } catch (e) {
+            return this.unknownProvider(url);
+        }
+
+        if (oembedUrl) {
+            // for standard WP oembed's we want to insert a bookmark card rather than their blockquote+script
+            // which breaks in the editor and most Ghost themes. Only fallback if card type was not explicitly chosen
+            if (!cardType && oembedUrl.match(/wp-json\/oembed/)) {
+                return;
             }
 
-            if (oembedUrl) {
-                // make sure the linked url is not an ip address or localhost
-                if (this.isIpOrLocalhost(oembedUrl)) {
-                    return this.unknownProvider(oembedUrl);
+            // fetch oembed response from embedded rel="alternate" url
+            const oembedResponse = await this.externalRequest(oembedUrl, {
+                method: 'GET',
+                json: true,
+                timeout: 2 * 1000,
+                followRedirect: true,
+                cookieJar
+            });
+            // validate the fetched json against the oembed spec to avoid
+            // leaking non-oembed responses
+            const body = oembedResponse.body;
+            const hasRequiredFields = body.type && body.version;
+            const hasValidType = ['photo', 'video', 'link', 'rich'].includes(body.type);
+
+            if (hasRequiredFields && hasValidType) {
+                // extract known oembed fields from the response to limit leaking of unrecognised data
+                const knownFields = [
+                    'type',
+                    'version',
+                    'html',
+                    'url',
+                    'title',
+                    'width',
+                    'height',
+                    'author_name',
+                    'author_url',
+                    'provider_name',
+                    'provider_url',
+                    'thumbnail_url',
+                    'thumbnail_width',
+                    'thumbnail_height'
+                ];
+                const oembed = _.pick(body, knownFields);
+
+                // ensure we have required data for certain types
+                if (oembed.type === 'photo' && !oembed.url) {
+                    return;
                 }
-
-                // for standard WP oembed's we want to insert a bookmark card rather than their blockquote+script
-                // which breaks in the editor and most Ghost themes. Only fallback if card type was not explicitly chosen
-                if (!cardType && oembedUrl.match(/wp-json\/oembed/)) {
+                if ((oembed.type === 'video' || oembed.type === 'rich') && (!oembed.html || !oembed.width || !oembed.height)) {
                     return;
                 }
 
-                // fetch oembed response from embedded rel="alternate" url
-                return this.externalRequest(oembedUrl, {
-                    method: 'GET',
-                    json: true,
-                    timeout: 2 * 1000,
-                    followRedirect: true,
-                    cookieJar
-                }).then((oembedResponse) => {
-                    // validate the fetched json against the oembed spec to avoid
-                    // leaking non-oembed responses
-                    const body = oembedResponse.body;
-                    const hasRequiredFields = body.type && body.version;
-                    const hasValidType = ['photo', 'video', 'link', 'rich'].includes(body.type);
-
-                    if (hasRequiredFields && hasValidType) {
-                        // extract known oembed fields from the response to limit leaking of unrecognised data
-                        const knownFields = [
-                            'type',
-                            'version',
-                            'html',
-                            'url',
-                            'title',
-                            'width',
-                            'height',
-                            'author_name',
-                            'author_url',
-                            'provider_name',
-                            'provider_url',
-                            'thumbnail_url',
-                            'thumbnail_width',
-                            'thumbnail_height'
-                        ];
-                        const oembed = _.pick(body, knownFields);
-
-                        // ensure we have required data for certain types
-                        if (oembed.type === 'photo' && !oembed.url) {
-                            return;
-                        }
-                        if ((oembed.type === 'video' || oembed.type === 'rich') && (!oembed.html || !oembed.width || !oembed.height)) {
-                            return;
-                        }
-
-                        // return the extracted object, don't pass through the response body
-                        return oembed;
-                    }
-                }).catch(() => {});
+                // return the extracted object, don't pass through the response body
+                return oembed;
             }
-        });
+        }
     }
 
     /**
@@ -274,6 +299,16 @@ class OEmbed {
         let data;
 
         try {
+            const urlObject = new URL(url);
+            for (const provider of this.customProviders) {
+                if (await provider.canSupportRequest(urlObject)) {
+                    const result = await provider.getOEmbedData(urlObject, this.externalRequest);
+                    if (result !== null) {
+                        return result;
+                    }
+                }
+            }
+
             if (type === 'bookmark') {
                 return this.fetchBookmarkData(url);
             }

package/core/server/services/offers/service.js

@@ -1,6 +1,3 @@
-const labs = require('../../../shared/labs');
-const events = require('../../lib/common/events');
-
 const DynamicRedirectManager = require('@tryghost/express-dynamic-redirects');
 const OffersModule = require('@tryghost/members-offers');
 
@@ -28,34 +25,7 @@ module.exports = {
 
         this.api = offersModule.api;
 
-        let initCalled = false;
-        if (labs.isSet('offers')) {
-            // handles setting up redirects
-            const promise = offersModule.init();
-            initCalled = true;
-            await promise;
-        }
-
-        // TODO: Delete after GA
-        let offersEnabled = labs.isSet('offers');
-        events.on('settings.labs.edited', async () => {
-            if (labs.isSet('offers') && !initCalled) {
-                const promise = offersModule.init();
-                initCalled = true;
-                await promise;
-            } else if (labs.isSet('offers') !== offersEnabled) {
-                offersEnabled = labs.isSet('offers');
-
-                if (offersEnabled) {
-                    const offers = await this.api.listOffers({});
-                    for (const offer of offers) {
-                        redirectManager.addRedirect(`/${offer.code}`, `/#/portal/offers/${offer.id}`, {permanent: false});
-                    }
-                } else {
-                    redirectManager.removeAllRedirects();
-                }
-            }
-        });
+        await offersModule.init();
     },
 
     api: null,

package/core/server/services/public-config/config.js

@@ -16,7 +16,8 @@ module.exports = function getConfigProperties() {
         stripeDirect: config.get('stripeDirect'),
         mailgunIsConfigured: config.get('bulkEmail') && config.get('bulkEmail').mailgun,
         emailAnalytics: config.get('emailAnalytics'),
-        hostSettings: config.get('hostSettings')
+        hostSettings: config.get('hostSettings'),
+        tenorApiKey: config.get('tenorApiKey')
     };
 
     const billingUrl = config.get('hostSettings:billing:enabled') ? config.get('hostSettings:billing:url') : '';

package/core/server/services/redirects/api.js

@@ -0,0 +1,270 @@
+const fs = require('fs-extra');
+const path = require('path');
+const moment = require('moment-timezone');
+const yaml = require('js-yaml');
+
+const logging = require('@tryghost/logging');
+const tpl = require('@tryghost/tpl');
+const errors = require('@tryghost/errors');
+
+const validation = require('./validation');
+
+const messages = {
+    jsonParse: 'Could not parse JSON: {context}.',
+    yamlParse: 'Could not parse YAML: {context}.',
+    yamlPlainString: 'YAML input cannot be a plain string. Check the format of your YAML file.',
+    redirectsHelp: 'https://ghost.org/docs/themes/routing/#redirects',
+    redirectsRegister: 'Could not register custom redirects.'
+};
+
+/**
+ * Redirect configuration object
+ * @typedef {Object} RedirectConfig
+ * @property {String} from - Defines the relative incoming URL or pattern (regex)
+ * @property {String} to - Defines where the incoming traffic should be redirected to, which can be a static URL, or a dynamic value using regex (example: "to": "/$1/")
+ * @property {boolean} permanent - Can be defined with true for a permanent HTTP 301 redirect, or false for a temporary HTTP 302 redirect
+ */
+
+/**
+ * @param {string} redirectsPath
+ * @returns {Promise<string>}
+ */
+const readRedirectsFile = async (redirectsPath) => {
+    try {
+        return await fs.readFile(redirectsPath, 'utf-8');
+    } catch (err) {
+        if (err.code === 'ENOENT') {
+            return '';
+        }
+
+        if (errors.utils.isIgnitionError(err)) {
+            throw err;
+        }
+
+        throw new errors.NotFoundError({
+            err: err
+        });
+    }
+};
+
+/**
+ *
+ * @param {String} content serialized JSON or YAML configuration
+ * @param {String} ext one of `.json` or `.yaml` extensions
+ *
+ * @returns {RedirectConfig[]} of parsed redirect config objects
+ */
+const parseRedirectsFile = (content, ext) => {
+    if (ext === '.json') {
+        let redirects;
+
+        try {
+            redirects = JSON.parse(content);
+        } catch (err) {
+            throw new errors.BadRequestError({
+                message: tpl(messages.jsonParse, {context: err.message})
+            });
+        }
+
+        return redirects;
+    }
+
+    if (ext === '.yaml') {
+        let redirects = [];
+        let configYaml;
+
+        try {
+            configYaml = yaml.load(content);
+        } catch (err) {
+            throw new errors.BadRequestError({
+                message: tpl(messages.yamlParse, {context: err.message})
+            });
+        }
+
+        // yaml.load passes almost every yaml code.
+        // Because of that, it's hard to detect if there's an error in the file.
+        // But one of the obvious errors is the plain string output.
+        // Here we check if the user made this mistake.
+        if (typeof configYaml === 'string') {
+            throw new errors.BadRequestError({
+                message: tpl(messages.yamlPlainString),
+                help: tpl(messages.redirectsHelp)
+            });
+        }
+
+        /**
+         * 302: Temporary redirects
+         */
+        for (const redirect in configYaml['302']) {
+            redirects.push({
+                from: redirect,
+                to: configYaml['302'][redirect],
+                permanent: false
+            });
+        }
+
+        /**
+         * 301: Permanent redirects
+         */
+        for (const redirect in configYaml['301']) {
+            redirects.push({
+                from: redirect,
+                to: configYaml['301'][redirect],
+                permanent: true
+            });
+        }
+
+        return redirects;
+    }
+
+    throw new errors.IncorrectUsageError();
+};
+
+/**
+ * @param {string} filePath
+ * @returns {string}
+ */
+const getBackupRedirectsFilePath = (filePath) => {
+    const {dir, name, ext} = path.parse(filePath);
+
+    return path.join(dir, `${name}-${moment().format('YYYY-MM-DD-HH-mm-ss')}${ext}`);
+};
+
+/**
+ * @typedef {object} IRedirectManager
+ */
+
+class CustomRedirectsAPI {
+    /**
+     * @param {object} config
+     * @param {string} config.basePath
+     *
+     * @param {IRedirectManager} redirectManager
+     */
+    constructor(config, redirectManager) {
+        /** @private */
+        this.config = config;
+        /** @private */
+        this.redirectManager = redirectManager;
+    }
+
+    async init() {
+        // NOTE: the try/catch block here is due to possible breaking change for existing misconfigured
+        //       instances in the wild. Would be a good idea to remove it during v5 migration to enforce
+        //       fail-fast initialization.
+        try {
+            const filePath = await this.getRedirectsFilePath();
+
+            if (filePath !== null) {
+                const content = await readRedirectsFile(filePath);
+                const ext = path.extname(filePath);
+                const redirects = parseRedirectsFile(content, ext);
+                validation.validate(redirects);
+
+                this.redirectManager.removeAllRedirects();
+                for (const redirect of redirects) {
+                    this.redirectManager.addRedirect(redirect.from, redirect.to, {permanent: redirect.permanent});
+                }
+            }
+        } catch (err) {
+            if (errors.utils.isIgnitionError(err)) {
+                logging.error(err);
+            } else {
+                logging.error(new errors.IncorrectUsageError({
+                    message: tpl(messages.redirectsRegister),
+                    context: err.message,
+                    help: tpl(messages.redirectsHelp),
+                    err
+                }));
+            }
+        }
+    }
+
+    /**
+     * @private
+     * @param {'.yaml'|'.json'} ext
+     *
+     * @returns {string}
+     */
+    createRedirectsFilePath(ext) {
+        return path.join(this.config.basePath, `redirects${ext}`);
+    }
+
+    /**
+     * @returns {Promise<string>}
+     */
+    async getRedirectsFilePath() {
+        const yamlPath = this.createRedirectsFilePath('.yaml');
+        const jsonPath = this.createRedirectsFilePath('.json');
+
+        const yamlExists = await fs.pathExists(yamlPath);
+
+        if (yamlExists) {
+            return yamlPath;
+        }
+
+        const jsonExist = await fs.pathExists(jsonPath);
+
+        if (jsonExist) {
+            return jsonPath;
+        }
+
+        return null;
+    }
+
+    /**
+     * @param {string} filePath
+     * @param {'.yaml'|'.json'} [ext]
+     *
+     * @returns {Promise<>}
+     */
+    async setFromFilePath(filePath, ext = '.json') {
+        const redirectsFilePath = await this.getRedirectsFilePath();
+
+        if (redirectsFilePath) {
+            const backupRedirectsPath = getBackupRedirectsFilePath(redirectsFilePath);
+
+            const backupExists = await fs.pathExists(backupRedirectsPath);
+            if (backupExists) {
+                await fs.unlink(backupRedirectsPath);
+            }
+
+            await fs.move(redirectsFilePath, backupRedirectsPath);
+        }
+
+        const content = await readRedirectsFile(filePath);
+        const parsed = parseRedirectsFile(content, ext);
+        validation.validate(parsed);
+
+        if (ext === '.json') {
+            await fs.writeFile(this.createRedirectsFilePath('.json'), JSON.stringify(content), 'utf-8');
+        } else if (ext === '.yaml') {
+            await fs.copy(filePath, this.createRedirectsFilePath('.yaml'));
+        }
+
+        this.redirectManager.removeAllRedirects();
+        for (const redirect of parsed) {
+            this.redirectManager.addRedirect(redirect.from, redirect.to, {permanent: redirect.permanent});
+        }
+    }
+
+    /**
+     * @returns {Promise<RedirectConfig[]>}
+     */
+    async get() {
+        const filePath = await this.getRedirectsFilePath();
+        if (filePath === null) {
+            return [];
+        }
+
+        const content = await readRedirectsFile(filePath);
+
+        if (path.extname(filePath) === '.json') {
+            return parseRedirectsFile(content, '.json');
+        }
+
+        return content;
+    }
+}
+
+module.exports = CustomRedirectsAPI;