ghost 5.62.0 → 5.64.0

package/core/server/data/migrations/versions/5.63/2023-09-13-13-03-10-add-ghost-core-content-integration.js

@@ -0,0 +1,57 @@
+// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253
+
+const logging = require('@tryghost/logging');
+const {default: ObjectID} = require('bson-objectid');
+const {createTransactionalMigration, meta} = require('../../utils');
+
+const coreContentIntegration = {
+    slug: 'ghost-core-content',
+    name: 'Ghost Core Content API',
+    description: 'Internal Content API integration for Admin access',
+    type: 'core'
+};
+
+const addIntegration = async (knex, integration) => {
+    const message = `Adding "${integration.name}" integration`;
+
+    const existing = await knex('integrations').select('id').where('slug', integration.slug).first();
+
+    if (existing?.id) {
+        logging.warn(`Skipping ${message} - already exists`);
+        return;
+    }
+
+    logging.info(message);
+
+    const now = knex.raw('CURRENT_TIMESTAMP');
+    integration.id = (new ObjectID()).toHexString();
+    integration.created_at = now;
+    integration.created_by = meta.MIGRATION_USER;
+
+    await knex('integrations').insert(integration);
+};
+
+const removeIntegration = async (knex, integration) => {
+    const message = `Removing ${integration.name} integration`;
+
+    const existing = await knex('integrations').select('id').where('slug', integration.slug).first();
+
+    if (!existing?.id) {
+        logging.warn(`Skipping ${message} - doesn't exist`);
+        return;
+    }
+
+    logging.info(message);
+
+    await knex('api_keys').where('integration_id', existing.id).del();
+    await knex('integrations').where('id', existing.id).del();
+};
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        await addIntegration(knex, coreContentIntegration);
+    },
+    async function down(knex) {
+        await removeIntegration(knex, coreContentIntegration);
+    }
+);

package/core/server/data/migrations/versions/5.63/2023-09-13-13-34-11-add-ghost-core-content-integration-key.js

@@ -0,0 +1,87 @@
+// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253
+
+const {InternalServerError} = require('@tryghost/errors');
+const logging = require('@tryghost/logging');
+const security = require('@tryghost/security');
+const {default: ObjectID} = require('bson-objectid');
+const {createTransactionalMigration, meta} = require('../../utils');
+
+const coreContentIntegration = {
+    slug: 'ghost-core-content',
+    name: 'Ghost Core Content API',
+    description: 'Internal Content API integration for Admin access',
+    type: 'core'
+};
+
+const addIntegrationContentKey = async (knex, integration) => {
+    const message = `Adding "${integration.name}" integration content key`;
+
+    const existingIntegration = await knex('integrations').select('id').where({
+        slug: integration.slug
+    }).first();
+
+    if (!existingIntegration) {
+        throw new InternalServerError({
+            message: `Could not find "${integration.name}" integration`
+        });
+    }
+
+    const existing = await knex('api_keys').select('id')
+        .where('integration_id', existingIntegration.id)
+        .where('type', 'content')
+        .first();
+
+    if (existing?.id) {
+        logging.warn(`Skipping ${message} - already exists`);
+        return;
+    }
+
+    logging.info(message);
+
+    await knex('api_keys').insert({
+        id: (new ObjectID()).toHexString(),
+        type: 'content',
+        secret: security.secret.create('content'),
+        role_id: null,
+        integration_id: existingIntegration.id,
+        created_at: knex.raw('current_timestamp'),
+        created_by: meta.MIGRATION_USER
+    });
+};
+
+const removeIntegrationContentKey = async (knex, integration) => {
+    const message = `Removing "${integration.name}" integration content key`;
+
+    const existingIntegration = await knex('integrations').select('id').where({
+        slug: integration.slug
+    }).first();
+
+    if (!existingIntegration?.id) {
+        logging.warn(`Skipping ${message} - integration does not exist`);
+        return;
+    }
+
+    const existing = await knex('api_keys').select('id').where({
+        integration_id: existingIntegration.id,
+        type: 'content'
+    }).first();
+
+    if (!existing?.id) {
+        logging.warn(`Skipping ${message} - content key does not exist`);
+        return;
+    }
+
+    logging.info(message);
+
+    await knex('api_keys').where('id', existing.id).del();
+};
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        await addIntegrationContentKey(knex, coreContentIntegration);
+    },
+
+    async function down(knex) {
+        await removeIntegrationContentKey(knex, coreContentIntegration);
+    }
+);

package/core/server/data/migrations/versions/5.64/2023-09-19-04-25-40-truncate-stale-built-in-collections-posts.js

@@ -0,0 +1,12 @@
+const logging = require('@tryghost/logging');
+const {createNonTransactionalMigration} = require('../../utils');
+
+module.exports = createNonTransactionalMigration(
+    async function up(knex) {
+        logging.info('Clearing collections_posts table');
+        await knex('collections_posts').truncate();
+    },
+    async function down() {
+        logging.info('Not doing anything - collections_posts table has been truncated');
+    }
+);

package/core/server/data/migrations/versions/5.64/2023-09-19-04-34-10-repopulate-built-in-collection-posts.js

@@ -0,0 +1,69 @@
+const logging = require('@tryghost/logging');
+const {default: ObjectID} = require('bson-objectid');
+const {createTransactionalMigration} = require('../../utils');
+
+const insertPostCollections = async (knex, collectionId, postIds) => {
+    logging.warn(`Batch inserting ${postIds.length} collection posts for collection ${collectionId}`);
+
+    const collectionPosts = postIds.map((postId) => {
+        return {
+            id: (new ObjectID()).toHexString(),
+            collection_id: collectionId,
+            post_id: postId,
+            sort_order: 0
+        };
+    });
+
+    await knex.batchInsert('collections_posts', collectionPosts, 1000);
+};
+
+module.exports = createTransactionalMigration(
+    async function up(knex) {
+        logging.info('Populating built-in collections');
+
+        const existingLatestCollection = await knex('collections')
+            .where({
+                slug: 'latest'
+            })
+            .first();
+
+        if (!existingLatestCollection) {
+            logging.warn('Latest collection does not exists, skipping');
+        } else {
+            const latestPostsRows = await knex('posts')
+                .select('id')
+                .where({
+                    type: 'post'
+                });
+
+            const latestPostsIds = latestPostsRows.map(row => row.id);
+
+            await insertPostCollections(knex, existingLatestCollection.id, latestPostsIds);
+        }
+
+        const existingFeaturedCollection = await knex('collections')
+            .where({
+                slug: 'featured'
+            })
+            .first();
+
+        if (!existingFeaturedCollection) {
+            logging.warn('Featured collection does not exist, skipping');
+        } else {
+            const featuredPostsRows = await knex('posts')
+                .select('id')
+                .where({
+                    featured: true,
+                    type: 'post'
+                });
+
+            const featuredPostsIds = featuredPostsRows.map(row => row.id);
+
+            await insertPostCollections(knex, existingFeaturedCollection.id, featuredPostsIds);
+        }
+    },
+    async function down(knex) {
+        logging.info('Clearing collections_posts table');
+        await knex('collections_posts').truncate();
+    }
+);

package/core/server/data/schema/fixtures/fixtures.json

@@ -807,6 +807,13 @@
                     "description": "Internal frontend integration",
                     "type": "internal",
                     "api_keys": [{"type": "content"}]
+                },
+                {
+                    "slug": "ghost-core-content",
+                    "name": "Ghost Core Content API",
+                    "description": "Internal Content API integration for Admin access",
+                    "type": "core",
+                    "api_keys": [{"type": "content"}]
                 }
             ]
         }

package/core/server/data/schema/schema.js

@@ -1079,5 +1079,17 @@ module.exports = {
         one_click_subscribe: {type: 'boolean', nullable: false, defaultTo: false},
         created_at: {type: 'dateTime', nullable: false},
         updated_at: {type: 'dateTime', nullable: true}
+    },
+    recommendation_click_events: {
+        id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+        recommendation_id: {type: 'string', maxlength: 24, nullable: false, references: 'recommendations.id', unique: false, cascadeDelete: true},
+        member_id: {type: 'string', maxlength: 24, nullable: true, references: 'members.id', unique: false, setNullDelete: true},
+        created_at: {type: 'dateTime', nullable: false}
+    },
+    recommendation_subscribe_events: {
+        id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+        recommendation_id: {type: 'string', maxlength: 24, nullable: false, references: 'recommendations.id', unique: false, cascadeDelete: true},
+        member_id: {type: 'string', maxlength: 24, nullable: true, references: 'members.id', unique: false, setNullDelete: true},
+        created_at: {type: 'dateTime', nullable: false}
     }
 };

package/core/server/lib/lexical.js

@@ -7,6 +7,7 @@ const storage = require('../adapters/storage');
 let nodes;
 let lexicalHtmlRenderer;
 let urlTransformMap;
+let postsService;
 
 function populateNodes() {
     const {DEFAULT_NODES} = require('@tryghost/kg-default-nodes');
@@ -28,6 +29,23 @@ module.exports = {
     },
 
     async render(lexical, userOptions = {}) {
+        if (!postsService) {
+            const getPostServiceInstance = require('../services/posts/posts-service');
+            postsService = getPostServiceInstance();
+        }
+
+        const getCollectionPosts = async (collectionSlug, postCount) => {
+            const transacting = userOptions.transacting;
+            const {data} = await postsService.browsePosts({
+                context: {public: true}, // mimic Content API request
+                collection: collectionSlug,
+                limit: postCount,
+                transacting
+            });
+            let posts = data.map(p => p.toJSON());
+            return posts;
+        };
+
         const options = Object.assign({
             siteUrl: config.get('url'),
             imageOptimization: config.get('imageOptimization'),
@@ -43,7 +61,8 @@ module.exports = {
             createDocument() {
                 const {JSDOM} = require('jsdom');
                 return (new JSDOM()).window.document;
-            }
+            },
+            getCollectionPosts
         }, userOptions);
 
         return await this.lexicalHtmlRenderer.render(lexical, options);

package/core/server/models/base/plugins/crud.js

@@ -44,12 +44,6 @@ module.exports = function (Bookshelf) {
                 });
             }
 
-            if (options.page && options.limit) {
-                itemCollection
-                    .query('limit', options.limit)
-                    .query('offset', options.limit * (options.page - 1));
-            }
-
             const result = await itemCollection.fetchAll(options);
             if (options.withRelated) {
                 _.each(result.models, function each(item) {

package/core/server/models/base/plugins/sanitize.js

@@ -45,7 +45,7 @@ module.exports = function (Bookshelf) {
             case 'findOne':
                 return baseOptions.concat(extraOptions, ['columns', 'require', 'mongoTransformer']);
             case 'findAll':
-                return baseOptions.concat(extraOptions, ['filter', 'columns', 'mongoTransformer', 'page', 'limit']);
+                return baseOptions.concat(extraOptions, ['filter', 'columns', 'mongoTransformer']);
             case 'findPage':
                 return baseOptions.concat(extraOptions, ['filter', 'order', 'autoOrder', 'page', 'limit', 'columns', 'mongoTransformer']);
             default:

package/core/server/models/post.js

@@ -700,7 +700,7 @@ Post = ghostBookshelf.Model.extend({
             )
         ) {
             try {
-                this.set('html', await lexicalLib.render(this.get('lexical')));
+                this.set('html', await lexicalLib.render(this.get('lexical'), {transacting: options.transacting}));
             } catch (err) {
                 throw new errors.ValidationError({
                     message: tpl(messages.invalidLexicalStructure),

package/core/server/models/recommendation-click-event.js

@@ -0,0 +1,22 @@
+const errors = require('@tryghost/errors');
+const ghostBookshelf = require('./base');
+
+const RecommendationClickEvent = ghostBookshelf.Model.extend({
+    tableName: 'recommendation_click_events'
+}, {
+    async edit() {
+        throw new errors.IncorrectUsageError({
+            message: 'Cannot edit RecommendationClickEvent'
+        });
+    },
+
+    async destroy() {
+        throw new errors.IncorrectUsageError({
+            message: 'Cannot destroy RecommendationClickEvent'
+        });
+    }
+});
+
+module.exports = {
+    RecommendationClickEvent: ghostBookshelf.model('RecommendationClickEvent', RecommendationClickEvent)
+};

package/core/server/models/recommendation-subscribe-event.js

@@ -0,0 +1,22 @@
+const errors = require('@tryghost/errors');
+const ghostBookshelf = require('./base');
+
+const RecommendationSubscribeEvent = ghostBookshelf.Model.extend({
+    tableName: 'recommendation_subscribe_events'
+}, {
+    async edit() {
+        throw new errors.IncorrectUsageError({
+            message: 'Cannot edit RecommendationSubscribeEvent'
+        });
+    },
+
+    async destroy() {
+        throw new errors.IncorrectUsageError({
+            message: 'Cannot destroy RecommendationSubscribeEvent'
+        });
+    }
+});
+
+module.exports = {
+    RecommendationSubscribeEvent: ghostBookshelf.model('RecommendationSubscribeEvent', RecommendationSubscribeEvent)
+};

package/core/server/services/collections/BookshelfCollectionsRepository.js

@@ -1,6 +1,7 @@
 const logger = require('@tryghost/logging');
 const Collection = require('@tryghost/collections').Collection;
 const sentry = require('../../../shared/sentry');
+const {default: ObjectID} = require('bson-objectid');
 /**
  * @typedef {import('@tryghost/collections/src/CollectionRepository')} CollectionRepository
  */
@@ -10,8 +11,10 @@ const sentry = require('../../../shared/sentry');
  */
 module.exports = class BookshelfCollectionsRepository {
     #model;
-    constructor(model) {
+    #relationModel;
+    constructor(model, relationModel) {
         this.#model = model;
+        this.#relationModel = relationModel;
     }
 
     async createTransaction(cb) {
@@ -25,12 +28,14 @@ module.exports = class BookshelfCollectionsRepository {
     async getById(id, options = {}) {
         const model = await this.#model.findOne({id}, {
             require: false,
-            withRelated: ['collectionPosts'],
             transacting: options.transaction
         });
         if (!model) {
             return null;
         }
+
+        model.collectionPostIds = await this.#fetchCollectionPostIds(model.id, options);
+
         return this.#modelToCollection(model);
     }
 
@@ -41,28 +46,63 @@ module.exports = class BookshelfCollectionsRepository {
     async getBySlug(slug, options = {}) {
         const model = await this.#model.findOne({slug}, {
             require: false,
-            withRelated: ['collectionPosts'],
             transacting: options.transaction
         });
+
         if (!model) {
             return null;
         }
+
+        model.collectionPostIds = await this.#fetchCollectionPostIds(model.id, options);
+
         return this.#modelToCollection(model);
     }
 
+    /**
+     * NOTE: we are only fetching post_id column here to save memory on
+     *       instances with a large amount of posts
+     *
+     *       The method could be further optimized to fetch posts for
+     *       multiple collections at a time.
+     *
+     * @param {string} collectionId collection to fetch post ids for
+     * @param {Object} options bookshelf options
+     *
+     * @returns {Promise<Array<{post_id: string}>>}
+     */
+    async #fetchCollectionPostIds(collectionId, options = {}) {
+        const toSelect = options.columns || ['post_id'];
+
+        const query = this.#relationModel.query();
+        if (options.transaction) {
+            query.transacting(options.transaction);
+        }
+
+        return await query
+            .select(toSelect)
+            .where('collection_id', collectionId);
+    }
+
     /**
      * @param {object} [options]
      * @param {string} [options.filter]
      * @param {string} [options.order]
+     * @param {string[]} [options.withRelated]
      * @param {import('knex').Transaction} [options.transaction]
      */
     async getAll(options = {}) {
         const models = await this.#model.findAll({
             ...options,
-            transacting: options.transaction,
-            withRelated: ['collectionPosts']
+            transacting: options.transaction
         });
 
+        for (const model of models) {
+            // NOTE: Ideally collection posts would be fetching as a part of findAll query.
+            //       Because bookshelf introduced a massive processing and memory overhead
+            //       we are fetching collection post ids separately using raw knex query
+            model.collectionPostIds = await this.#fetchCollectionPostIds(model.id, options);
+        }
+
         return (await Promise.all(models.map(model => this.#modelToCollection(model)))).filter(entity => !!entity);
     }
 
@@ -75,6 +115,10 @@ module.exports = class BookshelfCollectionsRepository {
         }
 
         try {
+            // NOTE: collectionPosts are not a part of serialized model
+            //       and are fetched separately to save memory
+            const posts = model.collectionPostIds;
+
             return await Collection.create({
                 id: json.id,
                 slug: json.slug,
@@ -83,7 +127,7 @@ module.exports = class BookshelfCollectionsRepository {
                 filter: filter,
                 type: json.type,
                 featureImage: json.feature_image,
-                posts: json.collectionPosts.map(collectionPost => collectionPost.post_id),
+                posts: posts.map(collectionPost => collectionPost.post_id),
                 createdAt: json.created_at,
                 updatedAt: json.updated_at
             });
@@ -101,10 +145,21 @@ module.exports = class BookshelfCollectionsRepository {
      * @returns {Promise<void>}
      */
     async save(collection, options = {}) {
+        if (!options.transaction) {
+            return this.createTransaction((transaction) => {
+                return this.save(collection, {
+                    ...options,
+                    transaction
+                });
+            });
+        }
+
         if (collection.deleted) {
-            await this.#model.destroy({id: collection.id});
+            await this.#relationModel.query().delete().where('collection_id', collection.id).transacting(options.transaction);
+            await this.#model.query().delete().where('id', collection.id).transacting(options.transaction);
             return;
         }
+
         const data = {
             id: collection.id,
             slug: collection.slug,
@@ -113,7 +168,6 @@ module.exports = class BookshelfCollectionsRepository {
             filter: collection.filter,
             type: collection.type,
             feature_image: collection.featureImage || null,
-            posts: collection.posts.map(postId => ({id: postId})),
             created_at: collection.createdAt,
             updated_at: collection.updatedAt
         };
@@ -130,11 +184,63 @@ module.exports = class BookshelfCollectionsRepository {
             await this.#model.add(data, {
                 transacting: options.transaction
             });
+            const collectionPostsRelations = collection.posts.map((postId, index) => {
+                return {
+                    id: (new ObjectID).toHexString(),
+                    sort_order: collection.type === 'manual' ? index : 0,
+                    collection_id: collection.id,
+                    post_id: postId
+                };
+            });
+            if (collectionPostsRelations.length > 0) {
+                await this.#relationModel.query().insert(collectionPostsRelations).transacting(options.transaction);
+            }
         } else {
-            return this.#model.edit(data, {
+            await this.#model.edit(data, {
                 id: data.id,
                 transacting: options.transaction
             });
+
+            const collectionPostsRelations = collection.posts.map((postId, index) => {
+                return {
+                    id: (new ObjectID).toHexString(),
+                    sort_order: collection.type === 'manual' ? index : 0,
+                    collection_id: collection.id,
+                    post_id: postId
+                };
+            });
+
+            const collectionPostRelationsToDeleteIds = [];
+
+            if (collection.type === 'manual') {
+                await this.#relationModel.query().delete().where('collection_id', collection.id).transacting(options.transaction);
+            } else {
+                const collectionPostsOptions = {
+                    transaction: options.transaction,
+                    columns: ['id', 'post_id']
+                };
+                const existingRelations = await this.#fetchCollectionPostIds(collection.id, collectionPostsOptions);
+
+                for (const existingRelation of existingRelations) {
+                    const found = collectionPostsRelations.find((thing) => {
+                        return thing.post_id === existingRelation.post_id;
+                    });
+                    if (found) {
+                        found.id = null;
+                    } else {
+                        collectionPostRelationsToDeleteIds.push(existingRelation.id);
+                    }
+                }
+            }
+
+            const missingCollectionPostsRelations = collectionPostsRelations.filter(thing => thing.id !== null);
+
+            if (missingCollectionPostsRelations.length > 0) {
+                await this.#relationModel.query().insert(missingCollectionPostsRelations).transacting(options.transaction);
+            }
+            if (collectionPostRelationsToDeleteIds.length > 0) {
+                await this.#relationModel.query().delete().whereIn('id', collectionPostRelationsToDeleteIds).transacting(options.transaction);
+            }
         }
     }
 };

package/core/server/services/collections/service.js

@@ -12,7 +12,7 @@ class CollectionsServiceWrapper {
         const DomainEvents = require('@tryghost/domain-events');
         const postsRepository = require('./PostsRepository').getInstance();
         const models = require('../../models');
-        const collectionsRepositoryInMemory = new BookshelfCollectionsRepository(models.Collection);
+        const collectionsRepositoryInMemory = new BookshelfCollectionsRepository(models.Collection, models.CollectionPost);
 
         const collectionsService = new CollectionsService({
             collectionsRepository: collectionsRepositoryInMemory,
@@ -33,16 +33,16 @@ class CollectionsServiceWrapper {
     async init() {
         const config = require('../../../shared/config');
         const labs = require('../../../shared/labs');
+
         // host setting OR labs "collections" flag has to be enabled to run collections service
-        if (!config.get('hostSettings:collections:enabled') && !(labs.isSet('collections'))) {
-            return;
-        }
+        if (config.get('hostSettings:collections:enabled') || labs.isSet('collections')) {
+            if (inited) {
+                return;
+            }
 
-        if (inited) {
-            return;
+            inited = true;
+            this.api.subscribeToEvents();
         }
-        inited = true;
-        this.api.subscribeToEvents();
     }
 }
 

package/core/server/services/members/middleware.js

@@ -5,7 +5,10 @@ const emailSuppressionList = require('../email-suppression-list');
 const models = require('../../models');
 const urlUtils = require('../../../shared/url-utils');
 const spamPrevention = require('../../web/shared/middleware/api/spam-prevention');
-const {formattedMemberResponse} = require('./utils');
+const {
+    formattedMemberResponse,
+    formatNewsletterResponse
+} = require('./utils');
 const errors = require('@tryghost/errors');
 const tpl = require('@tryghost/tpl');
 
@@ -144,6 +147,11 @@ const getMemberNewsletters = async function getMemberNewsletters(req, res) {
         }
 
         const data = _.pick(memberData.toJSON(), 'uuid', 'email', 'name', 'newsletters', 'enable_comment_notifications', 'status');
+
+        if (data.newsletters) {
+            data.newsletters = formatNewsletterResponse(data.newsletters);
+        }
+
         return res.json(data);
     } catch (err) {
         res.writeHead(400);
@@ -175,6 +183,11 @@ const updateMemberNewsletters = async function updateMemberNewsletters(req, res)
 
         const updatedMember = await membersService.api.members.update(data, options);
         const updatedMemberData = _.pick(updatedMember.toJSON(), ['uuid', 'email', 'name', 'newsletters', 'enable_comment_notifications', 'status']);
+
+        if (updatedMemberData.newsletters) {
+            updatedMemberData.newsletters = formatNewsletterResponse(updatedMemberData.newsletters);
+        }
+
         res.json(updatedMemberData);
     } catch (err) {
         res.writeHead(400);