ghost 4.22.3 → 4.25.0

package/core/server/services/route-settings/yaml-parser.js

@@ -35,7 +35,7 @@ module.exports = function parseYaml(file) {
 
         return parsed;
     } catch (error) {
-        if (errors.utils.isIgnitionError(error)) {
+        if (errors.utils.isGhostError(error)) {
             throw error;
         }
 

package/core/server/services/slack.js

@@ -136,7 +136,7 @@ function ping(post) {
                 'Content-type': 'application/json'
             }
         }).catch(function (err) {
-            logging.error(new errors.GhostError({
+            logging.error(new errors.InternalServerError({
                 err: err,
                 context: tpl(messages.requestFailedError, {service: 'slack'}),
                 help: tpl(messages.requestFailedHelp, {url: 'https://ghost.org/docs/'})

package/core/server/services/themes/activation-bridge.js

@@ -14,7 +14,7 @@ module.exports = {
         if (labs.isSet('customThemeSettings')) {
             await customThemeSettings.api.activateTheme(themeName, checkedTheme);
         }
-        bridge.activateTheme(theme, checkedTheme);
+        await bridge.activateTheme(theme, checkedTheme);
     },
     activateFromAPI: async (themeName, theme, checkedTheme) => {
         debug('Activating theme (method B on API "activate")', themeName);
@@ -22,7 +22,7 @@ module.exports = {
         if (labs.isSet('customThemeSettings')) {
             await customThemeSettings.api.activateTheme(themeName, checkedTheme);
         }
-        bridge.activateTheme(theme, checkedTheme);
+        await bridge.activateTheme(theme, checkedTheme);
     },
     activateFromAPIOverride: async (themeName, theme, checkedTheme) => {
         debug('Activating theme (method C on API "override")', themeName);
@@ -30,6 +30,6 @@ module.exports = {
         if (labs.isSet('customThemeSettings')) {
             await customThemeSettings.api.activateTheme(themeName, checkedTheme);
         }
-        bridge.activateTheme(theme, checkedTheme);
+        await bridge.activateTheme(theme, checkedTheme);
     }
 };

package/core/server/services/themes/storage.js

@@ -112,7 +112,7 @@ module.exports = {
             if (checkedTheme) {
                 fs.remove(checkedTheme.path)
                     .catch((err) => {
-                        logging.error(new errors.GhostError({err: err}));
+                        logging.error(new errors.InternalServerError({err: err}));
                     });
             }
 
@@ -120,7 +120,7 @@ module.exports = {
             getStorage()
                 .delete(backupName)
                 .catch((err) => {
-                    logging.error(new errors.GhostError({err: err}));
+                    logging.error(new errors.InternalServerError({err: err}));
                 });
         }
     },

package/core/server/services/twitter-embed.js

@@ -0,0 +1,80 @@
+const {extract} = require('oembed-parser');
+
+/**
+ * @typedef {import('./oembed').ICustomProvider} ICustomProvider
+ * @typedef {import('./oembed').IExternalRequest} IExternalRequest
+ */
+
+const TWITTER_PATH_REGEX = /\/status\/(\d+)/;
+
+/**
+ * @implements ICustomProvider
+ */
+class TwitterOEmbedProvider {
+    /**
+     * @param {object} dependencies
+     */
+    constructor(dependencies) {
+        this.dependencies = dependencies;
+    }
+
+    /**
+     * @param {URL} url
+     * @returns {Promise<boolean>}
+     */
+    async canSupportRequest(url) {
+        return url.host === 'twitter.com' && TWITTER_PATH_REGEX.test(url.pathname);
+    }
+
+    /**
+     * @param {URL} url
+     * @param {IExternalRequest} externalRequest
+     *
+     * @returns {Promise<object>}
+     */
+    async getOEmbedData(url, externalRequest) {
+        const [match, tweetId] = url.pathname.match(TWITTER_PATH_REGEX);
+        if (!match) {
+            return null;
+        }
+
+        /** @type {object} */
+        const oembedData = await extract(url.href);
+
+        if (this.dependencies.config.bearerToken) {
+            const query = {
+                expansions: ['attachments.poll_ids', 'attachments.media_keys', 'author_id', 'entities.mentions.username', 'geo.place_id', 'in_reply_to_user_id', 'referenced_tweets.id', 'referenced_tweets.id.author_id'],
+                'media.fields': ['duration_ms', 'height', 'media_key', 'preview_image_url', 'type', 'url', 'width', 'public_metrics', 'alt_text'],
+                'place.fields': ['contained_within', 'country', 'country_code', 'full_name', 'geo', 'id', 'name', 'place_type'],
+                'poll.fields': ['duration_minutes', 'end_datetime', 'id', 'options', 'voting_status'],
+                'tweet.fields': ['attachments', 'author_id', 'context_annotations', 'conversation_id', 'created_at', 'entities', 'geo', 'id', 'in_reply_to_user_id', 'lang', 'public_metrics', 'possibly_sensitive', 'referenced_tweets', 'reply_settings', 'source', 'text', 'withheld'],
+                'user.fields': ['created_at', 'description', 'entities', 'id', 'location', 'name', 'pinned_tweet_id', 'profile_image_url', 'protected', 'public_metrics', 'url', 'username', 'verified', 'withheld']
+            };
+
+            const queryString = Object.keys(query).map((key) => {
+                return `${key}=${query[key].join(',')}`;
+            }).join('&');
+
+            try {
+                const result = await externalRequest(`https://api.twitter.com/2/tweets/${tweetId}?${queryString}`, {
+                    responseType: 'json',
+                    headers: {
+                        Authorization: `Bearer ${this.dependencies.config.bearerToken}`
+                    }
+                });
+
+                const body = JSON.parse(result.body);
+
+                oembedData.tweet_data = body.data;
+            } catch (err) {
+                this.dependencies.logging.error(err);
+            }
+        }
+
+        oembedData.type = 'twitter';
+
+        return oembedData;
+    }
+}
+
+module.exports = TwitterOEmbedProvider;

package/core/server/services/url/LocalFileCache.js

@@ -0,0 +1,75 @@
+const fs = require('fs-extra');
+const path = require('path');
+
+class LocalFileCache {
+    /**
+     * @param {Object} options
+     * @param {String} options.storagePath - cached storage path
+     * @param {Boolean} options.writeDisabled - controls if cache can write
+     */
+    constructor({storagePath, writeDisabled}) {
+        const urlsStoragePath = path.join(storagePath, 'urls.json');
+        const resourcesCachePath = path.join(storagePath, 'resources.json');
+
+        this.storagePaths = {
+            urls: urlsStoragePath,
+            resources: resourcesCachePath
+        };
+        this.writeDisabled = writeDisabled;
+    }
+
+    /**
+     * Handles reading and parsing JSON from the filesystem.
+     * In case the file is corrupted or does not exist, returns null.
+     * @param {String} filePath path to read from
+     * @returns {Promise<Object>}
+     * @private
+     */
+    async readCacheFile(filePath) {
+        let cacheExists = false;
+        let cacheData = null;
+
+        try {
+            await fs.stat(filePath);
+            cacheExists = true;
+        } catch (e) {
+            cacheExists = false;
+        }
+
+        if (cacheExists) {
+            try {
+                const cacheFile = await fs.readFile(filePath, 'utf8');
+                cacheData = JSON.parse(cacheFile);
+            } catch (e) {
+                //noop as we'd start a long boot process if there are any errors in the file
+            }
+        }
+
+        return cacheData;
+    }
+
+    /**
+     *
+     * @param {'urls'|'resources'} type
+     * @returns {Promise<Object>}
+     */
+    async read(type) {
+        return await this.readCacheFile(this.storagePaths[type]);
+    }
+
+    /**
+     *
+     * @param {'urls'|'resources'} type of data to persist
+     * @param {Object} data - data to be persisted
+     * @returns {Promise<Object>}
+     */
+    async write(type, data) {
+        if (this.writeDisabled) {
+            return null;
+        }
+
+        return fs.writeFile(this.storagePaths[type], JSON.stringify(data, null, 4));
+    }
+}
+
+module.exports = LocalFileCache;

package/core/server/services/url/Resources.js

@@ -17,10 +17,16 @@ const events = require('../../lib/common/events');
  * Each entry in the database will be represented by a "Resource" (see /Resource.js).
  */
 class Resources {
-    constructor(queue) {
+    /**
+     *
+     * @param {Object} options
+     * @param {Object} [options.resources] - resources to initialize with instead of fetching them from the database
+     * @param {Object} [options.queue] - instance of the Queue class
+     */
+    constructor({resources = {}, queue} = {}) {
         this.queue = queue;
         this.resourcesConfig = [];
-        this.data = {};
+        this.data = resources;
 
         this.listeners = [];
     }

package/core/server/services/url/UrlGenerator.js

@@ -33,18 +33,29 @@ const EXPANSIONS = [{
  * Each router is represented by a url generator.
  */
 class UrlGenerator {
-    constructor(router, queue, resources, urls, position) {
-        this.router = router;
+    /**
+     * @param {Object} options
+     * @param {String} options.identifier frontend router ID reference
+     * @param {String} options.filter NQL filter string
+     * @param {String} options.resourceType resource type (e.g. 'posts', 'tags')
+     * @param {String} options.permalink permalink string
+     * @param {Object} options.queue instance of the backend Queue
+     * @param {Object} options.resources instance of the backend Resources
+     * @param {Object} options.urls instance of the backend URLs (used to store the urls)
+     * @param {Number} options.position an ID of the generator
+     */
+    constructor({identifier, filter, resourceType, permalink, queue, resources, urls, position}) {
+        this.identifier = identifier;
+        this.resourceType = resourceType;
+        this.permalink = permalink;
         this.queue = queue;
         this.urls = urls;
         this.resources = resources;
         this.uid = position;
 
-        debug('constructor', this.toString());
-
         // CASE: routers can define custom filters, but not required.
-        if (this.router.getFilter()) {
-            this.filter = this.router.getFilter();
+        if (filter) {
+            this.filter = filter;
             this.nql = nql(this.filter, {
                 expansions: EXPANSIONS,
                 transformer: nql.utils.mapKeyValues({
@@ -110,10 +121,10 @@ class UrlGenerator {
      * @private
      */
     _onInit() {
-        debug('_onInit', this.router.getResourceType());
+        debug('_onInit', this.resourceType);
 
         // @NOTE: get the resources of my type e.g. posts.
-        const resources = this.resources.getAllByType(this.router.getResourceType());
+        const resources = this.resources.getAllByType(this.resourceType);
 
         debug(resources.length);
 
@@ -131,7 +142,7 @@ class UrlGenerator {
         debug('onAdded', this.toString());
 
         // CASE: you are type "pages", but the incoming type is "users"
-        if (event.type !== this.router.getResourceType()) {
+        if (event.type !== this.resourceType) {
             return;
         }
 
@@ -182,8 +193,7 @@ class UrlGenerator {
      * @NOTE We currently generate relative urls (https://github.com/TryGhost/Ghost/commit/7b0d5d465ba41073db0c3c72006da625fa11df32).
      */
     _generateUrl(resource) {
-        const permalink = this.router.getPermalinks().getValue();
-        return localUtils.replacePermalink(permalink, resource.data);
+        return localUtils.replacePermalink(this.permalink, resource.data);
     }
 
     /**
@@ -214,7 +224,7 @@ class UrlGenerator {
                 action: 'added:' + resource.data.id,
                 eventData: {
                     id: resource.data.id,
-                    type: this.router.getResourceType()
+                    type: this.resourceType
                 }
             });
         };
@@ -246,19 +256,12 @@ class UrlGenerator {
 
     /**
      * @description Get all urls of this url generator.
+     * NOTE: the method is only used for testing purposes at the moment.
      * @returns {Array}
      */
     getUrls() {
         return this.urls.getByGeneratorId(this.uid);
     }
-
-    /**
-     * @description Override of `toString`
-     * @returns {string}
-     */
-    toString() {
-        return this.router.toString();
-    }
 }
 
 module.exports = UrlGenerator;

package/core/server/services/url/UrlService.js

@@ -1,4 +1,3 @@
-const fs = require('fs-extra');
 const _debug = require('@tryghost/debug')._base;
 const debug = _debug('ghost:services:url:service');
 const _ = require('lodash');
@@ -22,18 +21,25 @@ class UrlService {
     /**
      *
      * @param {Object} options
-     * @param {String} [options.urlCachePath] - path to store cached URLs at
+     * @param {Object} [options.cache] - cache handler instance
+     * @param {Function} [options.cache.read] - read cache by type
+     * @param {Function} [options.cache.write] - write into cache by type
      */
-    constructor({urlCachePath} = {}) {
+    constructor({cache} = {}) {
         this.utils = urlUtils;
-        this.urlCachePath = urlCachePath;
+        this.cache = cache;
+        this.onFinished = null;
         this.finished = false;
         this.urlGenerators = [];
 
         // Get urls
-        this.urls = new Urls();
         this.queue = new Queue();
-        this.resources = new Resources(this.queue);
+        // NOTE: Urls and Resources should not be initialized here but only in the init method.
+        //      Way too many tests fail if the initialization is removed so leaving it as is for time being
+        this.urls = new Urls();
+        this.resources = new Resources({
+            queue: this.queue
+        });
 
         this._listeners();
     }
@@ -76,26 +82,41 @@ class UrlService {
     _onQueueEnded(event) {
         if (event === 'init') {
             this.finished = true;
+            if (this.onFinished) {
+                this.onFinished();
+            }
         }
     }
 
     /**
      * @description Router was created, connect it with a url generator.
-     * @param {ExpressRouter} router
+     * @param {String} identifier frontend router ID reference
+     * @param {String} filter NQL filter
+     * @param {String} resourceType
+     * @param {String} permalink
      */
-    onRouterAddedType(router) {
-        debug('Registering route: ', router.name);
-
-        let urlGenerator = new UrlGenerator(router, this.queue, this.resources, this.urls, this.urlGenerators.length);
+    onRouterAddedType(identifier, filter, resourceType, permalink) {
+        debug('Registering route: ', filter, resourceType, permalink);
+
+        let urlGenerator = new UrlGenerator({
+            identifier,
+            filter,
+            resourceType,
+            permalink,
+            queue: this.queue,
+            resources: this.resources,
+            urls: this.urls,
+            position: this.urlGenerators.length
+        });
         this.urlGenerators.push(urlGenerator);
     }
 
     /**
      * @description Router update handler - regenerates it's resources
-     * @param {ExpressRouter} router
+     * @param {String} identifier router ID linked to the UrlGenerator
      */
-    onRouterUpdated(router) {
-        const generator = this.urlGenerators.find(g => g.router.id === router.id);
+    onRouterUpdated(identifier) {
+        const generator = this.urlGenerators.find(g => g.identifier === identifier);
         generator.regenerateResources();
     }
 
@@ -254,7 +275,7 @@ class UrlService {
         let urlGenerator;
 
         this.urlGenerators.every((_urlGenerator) => {
-            if (_urlGenerator.router.identifier === routerId) {
+            if (_urlGenerator.identifier === routerId) {
                 urlGenerator = _urlGenerator;
                 return false;
             }
@@ -284,68 +305,59 @@ class UrlService {
             return null;
         }
 
-        return _.find(this.urlGenerators, {uid: object.generatorId}).router.getPermalinks()
-            .getValue(options);
-    }
+        const permalink = _.find(this.urlGenerators, {uid: object.generatorId}).permalink;
 
-    /**
-     * @description Initializes components needed for the URL Service to function
-     */
-    async init() {
-        this.resources.initResourceConfig();
-        this.resources.initEvenListeners();
-
-        const persistedUrls = await this.fetchUrls();
-        if (persistedUrls) {
-            this.urls = new Urls({
-                urls: persistedUrls
-            });
-            this.finished = true;
-        } else {
-            await this.resources.fetchResources();
+        if (options.withUrlOptions) {
+            return urlUtils.urlJoin(permalink, '/:options(edit)?/');
         }
 
-        // CASE: all resources are fetched, start the queue
-        this.queue.start({
-            event: 'init',
-            tolerance: 100,
-            requiredSubscriberCount: 1
-        });
+        return permalink;
     }
 
-    async persistUrls() {
-        if (!labs.isSet('urlCache') || !this.urlCachePath) {
-            return null;
-        }
+    /**
+     * @description Initializes components needed for the URL Service to function
+     * @param {Object} options
+     * @param {Function} [options.onFinished] - callback when url generation is finished
+     * @param {Boolean} [options.urlCache] - whether to init using url cache or not
+     */
+    async init({onFinished, urlCache} = {}) {
+        this.onFinished = onFinished;
 
-        return fs.writeFile(this.urlCachePath, JSON.stringify(this.urls.urls, null, 4));
-    }
+        let persistedUrls;
+        let persistedResources;
 
-    async fetchUrls() {
-        if (!labs.isSet('urlCache') || !this.urlCachePath) {
-            return null;
+        if (this.cache && (labs.isSet('urlCache') || urlCache)) {
+            persistedUrls = await this.cache.read('urls');
+            persistedResources = await this.cache.read('resources');
         }
 
-        let urlsCacheExists = false;
-        let urls;
+        if (persistedUrls && persistedResources) {
+            this.urls.urls = persistedUrls;
+            this.resources.data = persistedResources;
+            this.resources.initResourceConfig();
+            this.resources.initEvenListeners();
 
-        try {
-            await fs.stat(this.urlCachePath);
-            urlsCacheExists = true;
-        } catch (e) {
-            urlsCacheExists = false;
+            this._onQueueEnded('init');
+        } else {
+            this.resources.initResourceConfig();
+            this.resources.initEvenListeners();
+            await this.resources.fetchResources();
+            // CASE: all resources are fetched, start the queue
+            this.queue.start({
+                event: 'init',
+                tolerance: 100,
+                requiredSubscriberCount: 1
+            });
         }
+    }
 
-        if (urlsCacheExists) {
-            try {
-                const urlsFile = await fs.readFile(this.urlCachePath, 'utf8');
-                urls = JSON.parse(urlsFile);
-            } catch (e) {
-                //noop as we'd start a long boot process if there are any errors in the file
-            }
+    async shutdown() {
+        if (!labs.isSet('urlCache')) {
+            return null;
         }
 
-        return urls;
+        await this.cache.write('urls', this.urls.urls);
+        await this.cache.write('resources', this.resources.getAll());
     }
 
     /**

package/core/server/services/url/index.js

@@ -1,12 +1,26 @@
-const path = require('path');
 const config = require('../../../shared/config');
+const LocalFileCache = require('./LocalFileCache');
 const UrlService = require('./UrlService');
 
 // NOTE: instead of a path we could give UrlService a "data-resolver" of some sort
 //       so it doesn't have to contain the logic to read data at all. This would be
 //       a possible improvement in the future
-const urlCachePath = path.join(config.getContentPath('data'), 'urls.json');
-const urlService = new UrlService({urlCachePath});
+let writeDisabled = false;
+let storagePath = config.getContentPath('data');
+
+// TODO: remove this hack in favor of loading from the content path when it's possible to do so
+//       by mocking content folders in pre-boot phase
+if (process.env.NODE_ENV.startsWith('test')){
+    storagePath = config.get('paths').urlCache;
+
+    // NOTE: prevents test suites from overwriting cache fixtures.
+    //       A better solution would be injecting a different implementation of the
+    //       cache based on the environment, this approach should do the trick for now
+    writeDisabled = true;
+}
+
+const cache = new LocalFileCache({storagePath, writeDisabled});
+const urlService = new UrlService({cache});
 
 // Singleton
 module.exports = urlService;

package/core/server/services/xmlrpc.js

@@ -87,7 +87,7 @@ function ping(post) {
                 if (!goodResponse.test(res.body)) {
                     const matches = res.body.match(errorMessage);
                     const message = matches ? matches[1] : res.body;
-                    throw new errors.GhostError({message});
+                    throw new errors.InternalServerError({message});
                 }
             })
             .catch(function (err) {
@@ -100,7 +100,7 @@ function ping(post) {
                         help: tpl(messages.requestFailedHelp, {url: 'https://ghost.org/docs/'})
                     });
                 } else {
-                    error = new errors.GhostError({
+                    error = new errors.InternalServerError({
                         err: err,
                         message: err.message,
                         context: tpl(messages.requestFailedError, {service: 'xmlrpc'}),

package/core/server/web/admin/app.js

@@ -5,7 +5,7 @@ const config = require('../../../shared/config');
 const constants = require('@tryghost/constants');
 const urlUtils = require('../../../shared/url-utils');
 const shared = require('../shared');
-const adminMiddleware = require('./middleware');
+const redirectAdminUrls = require('./middleware/redirect-admin-urls');
 
 module.exports = function setupAdminApp() {
     debug('Admin setup start');
@@ -26,29 +26,26 @@ module.exports = function setupAdminApp() {
         });
     }
 
-    // Render error page in case of maintenance
-    adminApp.use(shared.middlewares.maintenance);
-
     // Force SSL if required
     // must happen AFTER asset loading and BEFORE routing
-    adminApp.use(shared.middlewares.urlRedirects.adminSSLAndHostRedirect);
+    adminApp.use(shared.middleware.urlRedirects.adminSSLAndHostRedirect);
 
     // Add in all trailing slashes & remove uppercase
     // must happen AFTER asset loading and BEFORE routing
-    adminApp.use(shared.middlewares.prettyUrls);
+    adminApp.use(shared.middleware.prettyUrls);
 
     // Cache headers go last before serving the request
     // Admin is currently set to not be cached at all
-    adminApp.use(shared.middlewares.cacheControl('private'));
+    adminApp.use(shared.middleware.cacheControl('private'));
 
     // Special redirects for the admin (these should have their own cache-control headers)
-    adminApp.use(adminMiddleware);
+    adminApp.use(redirectAdminUrls);
 
     // Finally, routing
     adminApp.get('*', require('./controller'));
 
-    adminApp.use(shared.middlewares.errorHandler.pageNotFound);
-    adminApp.use(shared.middlewares.errorHandler.handleHTMLResponse);
+    adminApp.use(shared.middleware.errorHandler.pageNotFound);
+    adminApp.use(shared.middleware.errorHandler.handleHTMLResponse);
 
     debug('Admin setup end');