ghost 5.42.3 → 5.44.0

package/core/server/services/lexical-multiplayer/service.js

@@ -0,0 +1,141 @@
+const debug = require('@tryghost/debug')('lexical-multiplayer'); // eslint-disable-line no-unused-vars
+const logging = require('@tryghost/logging');
+const {getSession} = require('../auth/session/express-session');
+const models = require('../../models');
+const labs = require('../../../shared/labs');
+
+let wss;
+
+const onSocketError = (error) => {
+    logging.error(error);
+};
+
+const onUnauthorized = (socket) => {
+    socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
+    socket.destroy();
+};
+
+const handleUpgrade = async (request, socket, head) => {
+    socket.on('error', onSocketError);
+
+    // make sure the request is on the supported path
+    // TODO: check handling of subdirectories
+    if (!request.url.startsWith('/ghost/api/admin/posts/multiplayer/')) {
+        socket.write('HTTP/1.1 404 Not Found\r\n\r\n');
+        socket.destroy();
+        return;
+    }
+
+    // grab the session from the request
+    const session = await getSession(request, {});
+    if (!session || !session.user_id) {
+        return onUnauthorized(socket);
+    }
+
+    // fetch the session's user from the db
+    const user = await models.User.findOne({id: session.user_id});
+    if (!user) {
+        return onUnauthorized(socket);
+    }
+    request.user = user;
+
+    // TODO: check if user has access to the post
+
+    // TODO: (elsewhere) close websocket connections on logout
+    // - probably need to create a map of sockets to users?
+
+    socket.removeListener('error', onSocketError);
+
+    wss.handleUpgrade(request, socket, head, (ws) => {
+        wss.emit('connection', ws, request);
+    });
+};
+
+let _enable;
+let _disable;
+let _isClosing = false;
+let _closePromise;
+
+module.exports = {
+    async init(ghostServer) {
+        _enable = async () => {
+            if (_isClosing) {
+                logging.info('Waiting for previous Lexical multiplayer websockets service to close');
+                await _closePromise;
+            }
+
+            if (wss) {
+                logging.info('Lexical multiplayer websockets service already started');
+                return;
+            }
+
+            if (labs.isSet('lexicalMultiplayer')) {
+                logging.info('Starting lexical multiplayer websockets service');
+
+                // TODO: can we use or adapt patterns from https://github.com/HenningM/express-ws?
+                const WS = require('ws');
+                wss = new WS.Server({noServer: true});
+                const {setupWSConnection} = require('./y-websocket');
+
+                wss.on('connection', (socket, request) => {
+                    socket.on('error', onSocketError);
+
+                    // TODO: better method for extracting doc name from URL
+                    const docName = request.url.replace('/ghost/api/admin/posts/multiplayer/', '');
+                    setupWSConnection(socket, request, {docName});
+                });
+
+                // TODO: this should probably be at a higher level, especially if we
+                // want to support multiple websocket services
+                ghostServer.httpServer.on('upgrade', handleUpgrade);
+            }
+        };
+
+        _disable = async () => {
+            logging.info('Stopping lexical multiplayer websockets service');
+            ghostServer.httpServer.off('upgrade', handleUpgrade);
+
+            if (wss) {
+                _isClosing = true;
+                _closePromise = new Promise((resolve) => {
+                    // first sweep, soft close
+                    wss.clients.forEach((socket) => {
+                        socket.close();
+                    });
+
+                    setTimeout(() => {
+                        // second sweep, hard close
+                        wss.clients.forEach((socket) => {
+                            if ([socket.OPEN, socket.CLOSING].includes(socket.readyState)) {
+                                socket.terminate();
+                            }
+                        });
+
+                        resolve();
+                    }, 5000);
+                }).finally(() => {
+                    wss = null;
+                    _isClosing = false;
+                });
+
+                return _closePromise;
+            }
+        };
+    },
+
+    async enable() {
+        if (!_enable) {
+            logging.error('Lexical multiplayer service must be initialized before it can be enabled/disabled');
+            return;
+        }
+        return _enable();
+    },
+
+    async disable() {
+        if (!_enable) {
+            logging.error('Lexical multiplayer service must be initialized before it can be enabled/disabled');
+            return;
+        }
+        return _disable();
+    }
+};

package/core/server/services/lexical-multiplayer/y-websocket.js

@@ -0,0 +1,244 @@
+// based on https://github.com/yjs/y-websocket/blob/master/bin/utils.js
+
+const Y = require('yjs');
+const syncProtocol = require('y-protocols/dist/sync.cjs');
+const awarenessProtocol = require('y-protocols/dist/awareness.cjs');
+
+const encoding = require('lib0/dist/encoding.cjs');
+const decoding = require('lib0/dist/decoding.cjs');
+const map = require('lib0/dist/map.cjs');
+
+const wsReadyStateConnecting = 0;
+const wsReadyStateOpen = 1;
+const wsReadyStateClosing = 2 // eslint-disable-line
+const wsReadyStateClosed = 3 // eslint-disable-line
+
+/**
+ * @type {Map<string,WSSharedDoc>}
+ */
+const docs = new Map();
+module.exports.docs = docs;
+
+const messageSync = 0;
+const messageAwareness = 1;
+
+/**
+ * @param {Uint8Array} update
+ * @param {any} origin
+ * @param {WSSharedDoc} doc
+ */
+const updateHandler = (update, origin, doc) => {
+    const encoder = encoding.createEncoder();
+    encoding.writeVarUint(encoder, messageSync);
+    syncProtocol.writeUpdate(encoder, update);
+    const message = encoding.toUint8Array(encoder);
+    doc.conns.forEach((_, conn) => send(doc, conn, message));
+};
+
+class WSSharedDoc extends Y.Doc {
+    /**
+     * @param {string} name
+     */
+    constructor(name) {
+        super({gc: true});
+        this.name = name;
+        /**
+         * Maps from conn to set of controlled user ids. Delete all user ids from awareness when this conn is closed
+         * @type {Map<Object, Set<number>>}
+         */
+        this.conns = new Map();
+        /**
+         * @type {awarenessProtocol.Awareness}
+         */
+        this.awareness = new awarenessProtocol.Awareness(this);
+        this.awareness.setLocalState(null);
+        /**
+         * @param {{ added: Array<number>, updated: Array<number>, removed: Array<number> }} changes
+         * @param {Object | null} conn Origin is the connection that made the change
+         */
+        const awarenessChangeHandler = ({added, updated, removed}, conn) => {
+            const changedClients = added.concat(updated, removed);
+            if (conn !== null) {
+                const connControlledIDs = /** @type {Set<number>} */ (this.conns.get(conn));
+                if (connControlledIDs !== undefined) {
+                    added.forEach((clientID) => {
+                        connControlledIDs.add(clientID);
+                    });
+                    removed.forEach((clientID) => {
+                        connControlledIDs.delete(clientID);
+                    });
+                }
+            }
+            // broadcast awareness update
+            const encoder = encoding.createEncoder();
+            encoding.writeVarUint(encoder, messageAwareness);
+            encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.awareness, changedClients));
+            const buff = encoding.toUint8Array(encoder);
+            this.conns.forEach((_, c) => {
+                send(this, c, buff);
+            });
+        };
+        this.awareness.on('update', awarenessChangeHandler);
+        this.on('update', updateHandler);
+        // if (isCallbackSet) {
+        //     this.on('update', debounce(
+        //         callbackHandler,
+        //         CALLBACK_DEBOUNCE_WAIT,
+        //         { maxWait: CALLBACK_DEBOUNCE_MAXWAIT }
+        //     ))
+        // }
+    }
+}
+
+/**
+ * Gets a Y.Doc by name, whether in memory or on disk
+ *
+ * @param {string} docname - the name of the Y.Doc to find or create
+ * @param {boolean} gc - whether to allow gc on the doc (applies only when created)
+ * @return {WSSharedDoc}
+ */
+const getYDoc = (docname, gc = true) => map.setIfUndefined(docs, docname, () => {
+    const doc = new WSSharedDoc(docname);
+    doc.gc = gc;
+    // if (persistence !== null) {
+    //     persistence.bindState(docname, doc);
+    // }
+    docs.set(docname, doc);
+    return doc;
+});
+
+module.exports.getYDoc = getYDoc;
+
+/**
+ * @param {any} conn
+ * @param {WSSharedDoc} doc
+ * @param {Uint8Array} message
+ */
+const messageListener = (conn, doc, message) => {
+    try {
+        const encoder = encoding.createEncoder();
+        const decoder = decoding.createDecoder(message);
+        const messageType = decoding.readVarUint(decoder);
+        switch (messageType) {
+        case messageSync:
+            encoding.writeVarUint(encoder, messageSync);
+            syncProtocol.readSyncMessage(decoder, encoder, doc, conn);
+
+            // If the `encoder` only contains the type of reply message and no
+            // message, there is no need to send the message. When `encoder` only
+            // contains the type of reply, its length is 1.
+            if (encoding.length(encoder) > 1) {
+                send(doc, conn, encoding.toUint8Array(encoder));
+            }
+            break;
+        case messageAwareness: {
+            awarenessProtocol.applyAwarenessUpdate(doc.awareness, decoding.readVarUint8Array(decoder), conn);
+            break;
+        }
+        }
+    } catch (err) {
+        doc.emit('error', [err]);
+    }
+};
+
+/**
+ * @param {WSSharedDoc} doc
+ * @param {any} conn
+ */
+const closeConn = (doc, conn) => {
+    if (doc.conns.has(conn)) {
+        /**
+         * @type {Set<number>}
+         */
+        const controlledIds = doc.conns.get(conn);
+        doc.conns.delete(conn);
+        awarenessProtocol.removeAwarenessStates(doc.awareness, Array.from(controlledIds), null);
+        if (doc.conns.size === 0/* && persistence !== null*/) {
+            // if persisted, we store state and destroy ydocument
+            // persistence.writeState(doc.name, doc).then(() => {
+            //     doc.destroy();
+            // });
+            docs.delete(doc.name);
+        }
+    }
+    conn.close();
+};
+
+/**
+ * @param {WSSharedDoc} doc
+ * @param {any} conn
+ * @param {Uint8Array} m
+ */
+const send = (doc, conn, m) => {
+    if (conn.readyState !== wsReadyStateConnecting && conn.readyState !== wsReadyStateOpen) {
+        closeConn(doc, conn);
+    }
+    try {
+        conn.send(m, (err) => {
+            if (err !== null && err !== undefined) {
+                closeConn(doc, conn);
+            }
+        });
+    } catch (e) {
+        closeConn(doc, conn);
+    }
+};
+
+const pingTimeout = 30000;
+
+/**
+ * @param {any} conn
+ * @param {any} req
+ * @param {any} opts
+ */
+module.exports.setupWSConnection = (conn, req, {docName = req.url.slice(1).split('?')[0], gc = true} = {}) => {
+    conn.binaryType = 'arraybuffer';
+    // get doc, initialize if it does not exist yet
+    const doc = getYDoc(docName, gc);
+    doc.conns.set(conn, new Set());
+    // listen and reply to events
+    conn.on('message', /** @param {ArrayBuffer} message */ message => messageListener(conn, doc, new Uint8Array(message)));
+
+    // Check if connection is still alive
+    let pongReceived = true;
+    const pingInterval = setInterval(() => {
+        if (!pongReceived) {
+            if (doc.conns.has(conn)) {
+                closeConn(doc, conn);
+            }
+            clearInterval(pingInterval);
+        } else if (doc.conns.has(conn)) {
+            pongReceived = false;
+            try {
+                conn.ping();
+            } catch (e) {
+                closeConn(doc, conn);
+                clearInterval(pingInterval);
+            }
+        }
+    }, pingTimeout);
+    conn.on('close', () => {
+        closeConn(doc, conn);
+        clearInterval(pingInterval);
+    });
+    conn.on('pong', () => {
+        pongReceived = true;
+    });
+    // put the following in a variables in a block so the interval handlers don't keep in in
+    // scope
+    {
+        // send sync step 1
+        const syncEncoder = encoding.createEncoder();
+        encoding.writeVarUint(syncEncoder, messageSync);
+        syncProtocol.writeSyncStep1(syncEncoder, doc);
+        send(doc, conn, encoding.toUint8Array(syncEncoder));
+
+        const awarenessStates = doc.awareness.getStates();
+        if (awarenessStates.size > 0) {
+            const awarenessEncoder = encoding.createEncoder();
+            encoding.writeVarUint(awarenessEncoder, messageAwareness);
+            encoding.writeVarUint8Array(awarenessEncoder, awarenessProtocol.encodeAwarenessUpdate(doc.awareness, Array.from(awarenessStates.keys())));
+            send(doc, conn, encoding.toUint8Array(awarenessEncoder));
+        }
+    }
+};

package/core/server/services/settings/settings-bread-service.js

@@ -214,6 +214,21 @@ class SettingsBREADService {
         const {filteredSettings: refilteredSettings, emailsToVerify} = await this.prepSettingsForEmailVerification(filteredSettings, getSetting);
 
         const modelArray = await this.SettingsModel.edit(refilteredSettings, options).then((result) => {
+            // TODO: temporary fix for starting/stopping lexicalMultiplayer service when labs flag is changed
+            //       this should be removed along with the flag, or set up in a more generic way
+            const labsSetting = result.find(setting => setting.get('key') === 'labs');
+            if (labsSetting) {
+                const lexicalMultiplayer = require('../lexical-multiplayer');
+                const previous = JSON.parse(labsSetting.previousAttributes().value);
+                const current = JSON.parse(labsSetting.get('value'));
+
+                if (!previous.lexicalMultiplayer && current.lexicalMultiplayer) {
+                    lexicalMultiplayer.enable();
+                } else if (previous.lexicalMultiplayer && !current.lexicalMultiplayer) {
+                    lexicalMultiplayer.disable();
+                }
+            }
+
             return this._formatBrowse(_.keyBy(_.invokeMap(result, 'toJSON'), 'key'), options.context);
         });
 

package/core/server/web/api/endpoints/admin/routes.js

@@ -42,6 +42,8 @@ module.exports = function apiRoutes() {
 
     // ## Pages
     router.get('/pages', mw.authAdminApi, http(api.pages.browse));
+    router.del('/pages', mw.authAdminApi, http(api.pages.bulkDestroy));
+    router.put('/pages/bulk', mw.authAdminApi, http(api.pages.bulkEdit));
     router.post('/pages', mw.authAdminApi, http(api.pages.add));
     router.get('/pages/:id', mw.authAdminApi, http(api.pages.read));
     router.get('/pages/slug/:slug', mw.authAdminApi, http(api.pages.read));

package/core/shared/config/defaults.json

@@ -176,7 +176,7 @@
     },
     "portal": {
         "url": "https://cdn.jsdelivr.net/ghost/portal@~{version}/umd/portal.min.js",
-        "version": "2.28"
+        "version": "2.29"
     },
     "sodoSearch": {
         "url": "https://cdn.jsdelivr.net/ghost/sodo-search@~{version}/umd/sodo-search.min.js",

package/core/shared/labs.js

@@ -36,6 +36,7 @@ const ALPHA_FEATURES = [
     'urlCache',
     'migrateApp',
     'lexicalEditor',
+    'lexicalMultiplayer',
     'websockets',
     'stripeAutomaticTax',
     'makingItRain'