ghost 5.43.0 → 5.45.0

package/core/server/models/post.js

@@ -19,6 +19,8 @@ const urlUtils = require('../../shared/url-utils');
 const {Tag} = require('./tag');
 const {Newsletter} = require('./newsletter');
 const {BadRequestError} = require('@tryghost/errors');
+const PostRevisions = require('@tryghost/post-revisions');
+const labs = require('../../shared/labs');
 
 const messages = {
     isAlreadyPublished: 'Your post is already published, please reload your page.',
@@ -341,6 +343,11 @@ Post = ghostBookshelf.Model.extend({
                 tableName: 'posts_meta',
                 type: 'oneToOne',
                 joinFrom: 'post_id'
+            },
+            post_revisions: {
+                tableName: 'post_revisions',
+                type: 'oneToMany',
+                joinFrom: 'post_id'
             }
         };
     },
@@ -862,37 +869,76 @@ Post = ghostBookshelf.Model.extend({
             });
         }
 
-        // CASE: Handle post backups/revisions. This is a pure database feature.
-        if (model.hasChanged('lexical') && !model.get('mobiledoc') && !options.importing && !options.migrating) {
-            ops.push(function updateRevisions() {
-                return ghostBookshelf.model('PostRevision')
-                    .findAll(Object.assign({
-                        filter: `post_id:${model.id}`,
-                        columns: ['id']
-                    }, _.pick(options, 'transacting')))
-                    .then((revisions) => {
-                        // Store previous + latest lexical content
-                        if (!revisions.length && options.method !== 'insert') {
-                            model.set('post_revisions', [{
-                                post_id: model.id,
-                                lexical: model.previous('lexical'),
-                                created_at_ts: Date.now() - 1
-                            }, {
-                                post_id: model.id,
-                                lexical: model.get('lexical'),
-                                created_at_ts: Date.now()
-                            }]);
-                        } else {
-                            const revisionsJSON = revisions.toJSON().slice(0, POST_REVISIONS_COUNT - 1);
+        if (!labs.isSet('postHistory')) {
+            if (model.hasChanged('lexical') && !model.get('mobiledoc') && !options.importing && !options.migrating) {
+                ops.push(function updateRevisions() {
+                    return ghostBookshelf.model('PostRevision')
+                        .findAll(Object.assign({
+                            filter: `post_id:${model.id}`,
+                            columns: ['id']
+                        }, _.pick(options, 'transacting')))
+                        .then((revisions) => {
+                            // Store previous + latest lexical content
+                            if (!revisions.length && options.method !== 'insert') {
+                                model.set('post_revisions', [{
+                                    post_id: model.id,
+                                    lexical: model.previous('lexical'),
+                                    created_at_ts: Date.now() - 1
+                                }, {
+                                    post_id: model.id,
+                                    lexical: model.get('lexical'),
+                                    created_at_ts: Date.now()
+                                }]);
+                            } else {
+                                const revisionsJSON = revisions.toJSON().slice(0, POST_REVISIONS_COUNT - 1);
+
+                                model.set('post_revisions', revisionsJSON.concat([{
+                                    post_id: model.id,
+                                    lexical: model.get('lexical'),
+                                    created_at_ts: Date.now()
+                                }]));
+                            }
+                        });
+                });
+            }
+        } else {
+            if (!model.get('mobiledoc') && !options.importing && !options.migrating) {
+                const postRevisions = new PostRevisions({
+                    config: {
+                        max_revisions: POST_REVISIONS_COUNT
+                    }
+                });
+                const authorId = this.contextUser(options);
+                ops.push(async function updateRevisions() {
+                    const revisionModels = await ghostBookshelf.model('PostRevision')
+                        .findAll(Object.assign({
+                            filter: `post_id:${model.id}`,
+                            columns: ['id', 'lexical', 'created_at', 'author_id', 'title', 'reason', 'post_status']
+                        }, _.pick(options, 'transacting')));
+
+                    const revisions = revisionModels.toJSON();
+
+                    const current = {
+                        id: model.id,
+                        lexical: model.get('lexical'),
+                        html: model.get('html'),
+                        author_id: authorId,
+                        feature_image: model.get('feature_image'),
+                        feature_image_alt: model.get('posts_meta')?.feature_image_alt,
+                        feature_image_caption: model.get('posts_meta')?.feature_image_caption,
+                        title: model.get('title'),
+                        post_status: model.get('status')
+                    };
 
-                            model.set('post_revisions', revisionsJSON.concat([{
-                                post_id: model.id,
-                                lexical: model.get('lexical'),
-                                created_at_ts: Date.now()
-                            }]));
-                        }
-                    });
-            });
+                    // This can be refactored once we have the status stored in each revision
+                    const revisionOptions = {
+                        forceRevision: options.save_revision,
+                        isPublished: newStatus === 'published'
+                    };
+                    const newRevisions = await postRevisions.getRevisions(current, revisions, revisionOptions);
+                    model.set('post_revisions', newRevisions);
+                });
+            }
         }
 
         if (this.get('tiers')) {
@@ -1004,9 +1050,8 @@ Post = ghostBookshelf.Model.extend({
 
         attrs = this.formatsToJSON(attrs, options);
 
-        // CASE: never expose the revisions
+        // CASE: never expose the mobiledoc revisions
         delete attrs.mobiledoc_revisions;
-        delete attrs.post_revisions;
 
         // If the current column settings allow it...
         if (!options.columns || (options.columns && options.columns.indexOf('primary_tag') > -1)) {
@@ -1128,7 +1173,7 @@ Post = ghostBookshelf.Model.extend({
             findPage: ['status'],
             findAll: ['columns', 'filter'],
             destroy: ['destroyAll', 'destroyBy'],
-            edit: ['filter', 'email_segment', 'force_rerender', 'newsletter']
+            edit: ['filter', 'email_segment', 'force_rerender', 'newsletter', 'save_revision']
         };
 
         // The post model additionally supports having a formats option
@@ -1151,7 +1196,7 @@ Post = ghostBookshelf.Model.extend({
      */
     defaultRelations: function defaultRelations(methodName, options) {
         if (['edit', 'add', 'destroy'].indexOf(methodName) !== -1) {
-            options.withRelated = _.union(['authors', 'tags'], options.withRelated || []);
+            options.withRelated = _.union(['authors', 'tags', 'post_revisions', 'post_revisions.author'], options.withRelated || []);
         }
 
         const META_ATTRIBUTES = _.without(ghostBookshelf.model('PostsMeta').prototype.permittedAttributes(), 'id', 'post_id');

package/core/server/models/settings.js

@@ -161,7 +161,7 @@ Settings = ghostBookshelf.Model.extend({
     },
 
     formatOnWrite(attrs) {
-        if (attrs.value && ['cover_image', 'logo', 'icon', 'portal_button_icon', 'og_image', 'twitter_image'].includes(attrs.key)) {
+        if (attrs.value && ['cover_image', 'logo', 'icon', 'portal_button_icon', 'og_image', 'twitter_image', 'pintura_js_url', 'pintura_css_url'].includes(attrs.key)) {
             attrs.value = urlUtils.toTransformReady(attrs.value);
         }
 
@@ -183,7 +183,7 @@ Settings = ghostBookshelf.Model.extend({
         }
 
         // transform URLs from __GHOST_URL__ to absolute
-        if (['cover_image', 'logo', 'icon', 'portal_button_icon', 'og_image', 'twitter_image'].includes(attrs.key)) {
+        if (['cover_image', 'logo', 'icon', 'portal_button_icon', 'og_image', 'twitter_image', 'pintura_js_url', 'pintura_css_url'].includes(attrs.key)) {
             attrs.value = urlUtils.transformReadyToAbsolute(attrs.value);
         }
 

package/core/server/models/snippet.js

@@ -1,6 +1,8 @@
 const ghostBookshelf = require('./base');
 const urlUtils = require('../../shared/url-utils');
 const mobiledocLib = require('../lib/mobiledoc');
+const lexicalLib = require('../lib/lexical');
+const _ = require('lodash');
 
 const Snippet = ghostBookshelf.Model.extend({
     tableName: 'snippets',
@@ -10,6 +12,13 @@ const Snippet = ghostBookshelf.Model.extend({
             attrs.mobiledoc = urlUtils.mobiledocToTransformReady(attrs.mobiledoc, {cardTransformers: mobiledocLib.cards});
         }
 
+        if (attrs.lexical) {
+            attrs.lexical = urlUtils.lexicalToTransformReady(attrs.lexical, {
+                nodes: lexicalLib.nodes,
+                transformMap: lexicalLib.urlTransformMap
+            });
+        }
+
         return attrs;
     },
 
@@ -20,8 +29,35 @@ const Snippet = ghostBookshelf.Model.extend({
             attrs.mobiledoc = urlUtils.transformReadyToAbsolute(attrs.mobiledoc);
         }
 
+        if (attrs.lexical) {
+            attrs.lexical = urlUtils.transformReadyToAbsolute(attrs.lexical);
+        }
+
+        return attrs;
+    },
+
+    formatsToJSON: function formatsToJSON(attrs, options) {
+        const defaultFormats = ['mobiledoc'];
+        const formatsToKeep = options.formats || defaultFormats;
+
+        // Iterate over all known formats, and if they are not in the keep list, remove them
+        _.each(Snippet.allowedFormats, function (format) {
+            if (formatsToKeep.indexOf(format) === -1) {
+                delete attrs[format];
+            }
+        });
+
+        return attrs;
+    },
+    toJSON: function toJSON(options) {
+        let attrs = ghostBookshelf.Model.prototype.toJSON.call(this, options);
+
+        attrs = this.formatsToJSON(attrs, options);
+
         return attrs;
     }
+}, {
+    allowedFormats: ['mobiledoc', 'lexical']
 });
 
 const Snippets = ghostBookshelf.Collection.extend({

package/core/server/services/announcement-bar-service/index.js

@@ -0,0 +1,12 @@
+const settingsCache = require('../../../shared/settings-cache');
+const AnnouncementBarSettings = require('@tryghost/announcement-bar-settings');
+
+const announcementBarService = new AnnouncementBarSettings({
+    getAnnouncementSettings: () => ({
+        announcement: settingsCache.get('announcement_content'),
+        announcement_background: settingsCache.get('announcement_background'),
+        announcement_visibility: settingsCache.get('announcement_visibility')
+    })
+});
+
+module.exports = announcementBarService;

package/core/server/services/lexical-multiplayer/index.js

@@ -0,0 +1 @@
+module.exports = require('./service');

package/core/server/services/lexical-multiplayer/service.js

@@ -0,0 +1,141 @@
+const debug = require('@tryghost/debug')('lexical-multiplayer'); // eslint-disable-line no-unused-vars
+const logging = require('@tryghost/logging');
+const {getSession} = require('../auth/session/express-session');
+const models = require('../../models');
+const labs = require('../../../shared/labs');
+
+let wss;
+
+const onSocketError = (error) => {
+    logging.error(error);
+};
+
+const onUnauthorized = (socket) => {
+    socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
+    socket.destroy();
+};
+
+const handleUpgrade = async (request, socket, head) => {
+    socket.on('error', onSocketError);
+
+    // make sure the request is on the supported path
+    // TODO: check handling of subdirectories
+    if (!request.url.startsWith('/ghost/api/admin/posts/multiplayer/')) {
+        socket.write('HTTP/1.1 404 Not Found\r\n\r\n');
+        socket.destroy();
+        return;
+    }
+
+    // grab the session from the request
+    const session = await getSession(request, {});
+    if (!session || !session.user_id) {
+        return onUnauthorized(socket);
+    }
+
+    // fetch the session's user from the db
+    const user = await models.User.findOne({id: session.user_id});
+    if (!user) {
+        return onUnauthorized(socket);
+    }
+    request.user = user;
+
+    // TODO: check if user has access to the post
+
+    // TODO: (elsewhere) close websocket connections on logout
+    // - probably need to create a map of sockets to users?
+
+    socket.removeListener('error', onSocketError);
+
+    wss.handleUpgrade(request, socket, head, (ws) => {
+        wss.emit('connection', ws, request);
+    });
+};
+
+let _enable;
+let _disable;
+let _isClosing = false;
+let _closePromise;
+
+module.exports = {
+    async init(ghostServer) {
+        _enable = async () => {
+            if (_isClosing) {
+                logging.info('Waiting for previous Lexical multiplayer websockets service to close');
+                await _closePromise;
+            }
+
+            if (wss) {
+                logging.info('Lexical multiplayer websockets service already started');
+                return;
+            }
+
+            if (labs.isSet('lexicalMultiplayer')) {
+                logging.info('Starting lexical multiplayer websockets service');
+
+                // TODO: can we use or adapt patterns from https://github.com/HenningM/express-ws?
+                const WS = require('ws');
+                wss = new WS.Server({noServer: true});
+                const {setupWSConnection} = require('./y-websocket');
+
+                wss.on('connection', (socket, request) => {
+                    socket.on('error', onSocketError);
+
+                    // TODO: better method for extracting doc name from URL
+                    const docName = request.url.replace('/ghost/api/admin/posts/multiplayer/', '');
+                    setupWSConnection(socket, request, {docName});
+                });
+
+                // TODO: this should probably be at a higher level, especially if we
+                // want to support multiple websocket services
+                ghostServer.httpServer.on('upgrade', handleUpgrade);
+            }
+        };
+
+        _disable = async () => {
+            logging.info('Stopping lexical multiplayer websockets service');
+            ghostServer.httpServer.off('upgrade', handleUpgrade);
+
+            if (wss) {
+                _isClosing = true;
+                _closePromise = new Promise((resolve) => {
+                    // first sweep, soft close
+                    wss.clients.forEach((socket) => {
+                        socket.close();
+                    });
+
+                    setTimeout(() => {
+                        // second sweep, hard close
+                        wss.clients.forEach((socket) => {
+                            if ([socket.OPEN, socket.CLOSING].includes(socket.readyState)) {
+                                socket.terminate();
+                            }
+                        });
+
+                        resolve();
+                    }, 5000);
+                }).finally(() => {
+                    wss = null;
+                    _isClosing = false;
+                });
+
+                return _closePromise;
+            }
+        };
+    },
+
+    async enable() {
+        if (!_enable) {
+            logging.error('Lexical multiplayer service must be initialized before it can be enabled/disabled');
+            return;
+        }
+        return _enable();
+    },
+
+    async disable() {
+        if (!_enable) {
+            logging.error('Lexical multiplayer service must be initialized before it can be enabled/disabled');
+            return;
+        }
+        return _disable();
+    }
+};

package/core/server/services/lexical-multiplayer/y-websocket.js

@@ -0,0 +1,244 @@
+// based on https://github.com/yjs/y-websocket/blob/master/bin/utils.js
+
+const Y = require('yjs');
+const syncProtocol = require('y-protocols/dist/sync.cjs');
+const awarenessProtocol = require('y-protocols/dist/awareness.cjs');
+
+const encoding = require('lib0/dist/encoding.cjs');
+const decoding = require('lib0/dist/decoding.cjs');
+const map = require('lib0/dist/map.cjs');
+
+const wsReadyStateConnecting = 0;
+const wsReadyStateOpen = 1;
+const wsReadyStateClosing = 2 // eslint-disable-line
+const wsReadyStateClosed = 3 // eslint-disable-line
+
+/**
+ * @type {Map<string,WSSharedDoc>}
+ */
+const docs = new Map();
+module.exports.docs = docs;
+
+const messageSync = 0;
+const messageAwareness = 1;
+
+/**
+ * @param {Uint8Array} update
+ * @param {any} origin
+ * @param {WSSharedDoc} doc
+ */
+const updateHandler = (update, origin, doc) => {
+    const encoder = encoding.createEncoder();
+    encoding.writeVarUint(encoder, messageSync);
+    syncProtocol.writeUpdate(encoder, update);
+    const message = encoding.toUint8Array(encoder);
+    doc.conns.forEach((_, conn) => send(doc, conn, message));
+};
+
+class WSSharedDoc extends Y.Doc {
+    /**
+     * @param {string} name
+     */
+    constructor(name) {
+        super({gc: true});
+        this.name = name;
+        /**
+         * Maps from conn to set of controlled user ids. Delete all user ids from awareness when this conn is closed
+         * @type {Map<Object, Set<number>>}
+         */
+        this.conns = new Map();
+        /**
+         * @type {awarenessProtocol.Awareness}
+         */
+        this.awareness = new awarenessProtocol.Awareness(this);
+        this.awareness.setLocalState(null);
+        /**
+         * @param {{ added: Array<number>, updated: Array<number>, removed: Array<number> }} changes
+         * @param {Object | null} conn Origin is the connection that made the change
+         */
+        const awarenessChangeHandler = ({added, updated, removed}, conn) => {
+            const changedClients = added.concat(updated, removed);
+            if (conn !== null) {
+                const connControlledIDs = /** @type {Set<number>} */ (this.conns.get(conn));
+                if (connControlledIDs !== undefined) {
+                    added.forEach((clientID) => {
+                        connControlledIDs.add(clientID);
+                    });
+                    removed.forEach((clientID) => {
+                        connControlledIDs.delete(clientID);
+                    });
+                }
+            }
+            // broadcast awareness update
+            const encoder = encoding.createEncoder();
+            encoding.writeVarUint(encoder, messageAwareness);
+            encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.awareness, changedClients));
+            const buff = encoding.toUint8Array(encoder);
+            this.conns.forEach((_, c) => {
+                send(this, c, buff);
+            });
+        };
+        this.awareness.on('update', awarenessChangeHandler);
+        this.on('update', updateHandler);
+        // if (isCallbackSet) {
+        //     this.on('update', debounce(
+        //         callbackHandler,
+        //         CALLBACK_DEBOUNCE_WAIT,
+        //         { maxWait: CALLBACK_DEBOUNCE_MAXWAIT }
+        //     ))
+        // }
+    }
+}
+
+/**
+ * Gets a Y.Doc by name, whether in memory or on disk
+ *
+ * @param {string} docname - the name of the Y.Doc to find or create
+ * @param {boolean} gc - whether to allow gc on the doc (applies only when created)
+ * @return {WSSharedDoc}
+ */
+const getYDoc = (docname, gc = true) => map.setIfUndefined(docs, docname, () => {
+    const doc = new WSSharedDoc(docname);
+    doc.gc = gc;
+    // if (persistence !== null) {
+    //     persistence.bindState(docname, doc);
+    // }
+    docs.set(docname, doc);
+    return doc;
+});
+
+module.exports.getYDoc = getYDoc;
+
+/**
+ * @param {any} conn
+ * @param {WSSharedDoc} doc
+ * @param {Uint8Array} message
+ */
+const messageListener = (conn, doc, message) => {
+    try {
+        const encoder = encoding.createEncoder();
+        const decoder = decoding.createDecoder(message);
+        const messageType = decoding.readVarUint(decoder);
+        switch (messageType) {
+        case messageSync:
+            encoding.writeVarUint(encoder, messageSync);
+            syncProtocol.readSyncMessage(decoder, encoder, doc, conn);
+
+            // If the `encoder` only contains the type of reply message and no
+            // message, there is no need to send the message. When `encoder` only
+            // contains the type of reply, its length is 1.
+            if (encoding.length(encoder) > 1) {
+                send(doc, conn, encoding.toUint8Array(encoder));
+            }
+            break;
+        case messageAwareness: {
+            awarenessProtocol.applyAwarenessUpdate(doc.awareness, decoding.readVarUint8Array(decoder), conn);
+            break;
+        }
+        }
+    } catch (err) {
+        doc.emit('error', [err]);
+    }
+};
+
+/**
+ * @param {WSSharedDoc} doc
+ * @param {any} conn
+ */
+const closeConn = (doc, conn) => {
+    if (doc.conns.has(conn)) {
+        /**
+         * @type {Set<number>}
+         */
+        const controlledIds = doc.conns.get(conn);
+        doc.conns.delete(conn);
+        awarenessProtocol.removeAwarenessStates(doc.awareness, Array.from(controlledIds), null);
+        if (doc.conns.size === 0/* && persistence !== null*/) {
+            // if persisted, we store state and destroy ydocument
+            // persistence.writeState(doc.name, doc).then(() => {
+            //     doc.destroy();
+            // });
+            docs.delete(doc.name);
+        }
+    }
+    conn.close();
+};
+
+/**
+ * @param {WSSharedDoc} doc
+ * @param {any} conn
+ * @param {Uint8Array} m
+ */
+const send = (doc, conn, m) => {
+    if (conn.readyState !== wsReadyStateConnecting && conn.readyState !== wsReadyStateOpen) {
+        closeConn(doc, conn);
+    }
+    try {
+        conn.send(m, (err) => {
+            if (err !== null && err !== undefined) {
+                closeConn(doc, conn);
+            }
+        });
+    } catch (e) {
+        closeConn(doc, conn);
+    }
+};
+
+const pingTimeout = 30000;
+
+/**
+ * @param {any} conn
+ * @param {any} req
+ * @param {any} opts
+ */
+module.exports.setupWSConnection = (conn, req, {docName = req.url.slice(1).split('?')[0], gc = true} = {}) => {
+    conn.binaryType = 'arraybuffer';
+    // get doc, initialize if it does not exist yet
+    const doc = getYDoc(docName, gc);
+    doc.conns.set(conn, new Set());
+    // listen and reply to events
+    conn.on('message', /** @param {ArrayBuffer} message */ message => messageListener(conn, doc, new Uint8Array(message)));
+
+    // Check if connection is still alive
+    let pongReceived = true;
+    const pingInterval = setInterval(() => {
+        if (!pongReceived) {
+            if (doc.conns.has(conn)) {
+                closeConn(doc, conn);
+            }
+            clearInterval(pingInterval);
+        } else if (doc.conns.has(conn)) {
+            pongReceived = false;
+            try {
+                conn.ping();
+            } catch (e) {
+                closeConn(doc, conn);
+                clearInterval(pingInterval);
+            }
+        }
+    }, pingTimeout);
+    conn.on('close', () => {
+        closeConn(doc, conn);
+        clearInterval(pingInterval);
+    });
+    conn.on('pong', () => {
+        pongReceived = true;
+    });
+    // put the following in a variables in a block so the interval handlers don't keep in in
+    // scope
+    {
+        // send sync step 1
+        const syncEncoder = encoding.createEncoder();
+        encoding.writeVarUint(syncEncoder, messageSync);
+        syncProtocol.writeSyncStep1(syncEncoder, doc);
+        send(doc, conn, encoding.toUint8Array(syncEncoder));
+
+        const awarenessStates = doc.awareness.getStates();
+        if (awarenessStates.size > 0) {
+            const awarenessEncoder = encoding.createEncoder();
+            encoding.writeVarUint(awarenessEncoder, messageAwareness);
+            encoding.writeVarUint8Array(awarenessEncoder, awarenessProtocol.encodeAwarenessUpdate(doc.awareness, Array.from(awarenessStates.keys())));
+            send(doc, conn, encoding.toUint8Array(awarenessEncoder));
+        }
+    }
+};

package/core/server/services/public-config/config.js

@@ -19,7 +19,8 @@ module.exports = function getConfigProperties() {
         emailAnalytics: config.get('emailAnalytics'),
         hostSettings: config.get('hostSettings'),
         tenor: config.get('tenor'),
-        editor: config.get('editor')
+        editor: config.get('editor'),
+        pintura: config.get('pintura')
     };
 
     const billingUrl = config.get('hostSettings:billing:enabled') ? config.get('hostSettings:billing:url') : '';

package/core/server/services/settings/settings-bread-service.js

@@ -214,6 +214,21 @@ class SettingsBREADService {
         const {filteredSettings: refilteredSettings, emailsToVerify} = await this.prepSettingsForEmailVerification(filteredSettings, getSetting);
 
         const modelArray = await this.SettingsModel.edit(refilteredSettings, options).then((result) => {
+            // TODO: temporary fix for starting/stopping lexicalMultiplayer service when labs flag is changed
+            //       this should be removed along with the flag, or set up in a more generic way
+            const labsSetting = result.find(setting => setting.get('key') === 'labs');
+            if (labsSetting) {
+                const lexicalMultiplayer = require('../lexical-multiplayer');
+                const previous = JSON.parse(labsSetting.previousAttributes().value);
+                const current = JSON.parse(labsSetting.get('value'));
+
+                if (!previous.lexicalMultiplayer && current.lexicalMultiplayer) {
+                    lexicalMultiplayer.enable();
+                } else if (previous.lexicalMultiplayer && !current.lexicalMultiplayer) {
+                    lexicalMultiplayer.disable();
+                }
+            }
+
             return this._formatBrowse(_.keyBy(_.invokeMap(result, 'toJSON'), 'key'), options.context);
         });