ghost 5.110.4 → 5.112.0

package/core/server/services/activitypub/ActivityPubService.ts

@@ -0,0 +1,139 @@
+import ObjectID from 'bson-objectid';
+import {Knex} from 'knex';
+import {IdentityTokenService} from '@tryghost/identity-token-service';
+import fetch from 'node-fetch';
+
+type ExpectedWebhook = {
+    event: string;
+    target_url: URL;
+    api_version: string;
+    secret: string;
+};
+
+interface Logger {
+    info(message: string): void
+    warn(message: string): void
+    error(message: string): void
+}
+
+export class ActivityPubService {
+    constructor(
+        private knex: Knex,
+        private siteUrl: URL,
+        private logging: Logger,
+        private identityTokenService: IdentityTokenService
+    ) {}
+
+    getExpectedWebhooks(secret: string): ExpectedWebhook[] {
+        return [{
+            event: 'post.published',
+            target_url: new URL('.ghost/activitypub/webhooks/post/published', this.siteUrl),
+            api_version: 'v5.100.0',
+            secret
+        }, {
+            event: 'site.changed',
+            target_url: new URL('.ghost/activitypub/webhooks/site/changed', this.siteUrl),
+            api_version: 'v5.100.0',
+            secret
+        }];
+    }
+
+    async checkWebhookState(expectedWebhooks: ExpectedWebhook[], integration: {id: string}) {
+        this.logging.info(`Checking ActivityPub Webhook state`);
+
+        const webhooks = await this.knex
+            .select('*')
+            .from('webhooks')
+            .where('integration_id', '=', integration.id);
+
+        if (webhooks.length !== expectedWebhooks.length) {
+            this.logging.warn(`Expected ${expectedWebhooks.length} webhooks for ActivityPub`);
+            return false;
+        }
+
+        for (const expectedWebhook of expectedWebhooks) {
+            const foundWebhook = webhooks.find((webhook) => {
+                return webhook.event === expectedWebhook.event && webhook.target_url === expectedWebhook.target_url.href && webhook.secret === expectedWebhook.secret;
+            });
+            if (!foundWebhook) {
+                this.logging.error(`Could not find webhook for ${expectedWebhook.event} ${expectedWebhook.target_url}`);
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    async getWebhookSecret(): Promise<string | null> {
+        try {
+            const ownerUser = await this.knex.select('*').from('users').where('id', '=', '1').first();
+            const token = await this.identityTokenService.getTokenForUser(ownerUser.email, 'Owner');
+
+            const res = await fetch(new URL('.ghost/activitypub/site', this.siteUrl), {
+                headers: {
+                    Authorization: `Bearer ${token}`
+                }
+            });
+
+            const body = await res.json();
+
+            return body.webhook_secret;
+        } catch (err: unknown) {
+            this.logging.error(`Could not get webhook secret for ActivityPub ${err}`);
+            return null;
+        }
+    }
+
+    async initialiseWebhooks() {
+        const integration = await this.knex
+            .select('*')
+            .from('integrations')
+            .where('slug', '=', 'ghost-activitypub')
+            .andWhere('type', '=', 'internal')
+            .first();
+
+        if (!integration) {
+            this.logging.error('No ActivityPub integration found - cannot initialise');
+            return;
+        }
+
+        const secret = await this.getWebhookSecret();
+
+        if (!secret) {
+            this.logging.error('No webhook secret found - cannot initialise');
+            return;
+        }
+
+        const expectedWebhooks = this.getExpectedWebhooks(secret);
+        const isInCorrectState = await this.checkWebhookState(expectedWebhooks, integration);
+
+        if (isInCorrectState) {
+            this.logging.info(`ActivityPub webhooks in correct state`);
+            return;
+        }
+
+        this.logging.info(`ActivityPub webhooks in incorrect state, deleting all of them and starting fresh`);
+        await this.knex
+            .del()
+            .from('webhooks')
+            .where('integration_id', '=', integration.id);
+
+        const webhooksToInsert = expectedWebhooks.map((expectedWebhook) => {
+            return {
+                id: (new ObjectID).toHexString(),
+                event: expectedWebhook.event,
+                target_url: expectedWebhook.target_url.href,
+                api_version: expectedWebhook.api_version,
+                name: `ActivityPub ${expectedWebhook.event} Webhook`,
+                secret: secret,
+                integration_id: integration.id,
+                created_at: this.knex.raw('current_timestamp'),
+                created_by: '1'
+            };
+        });
+
+        await this.knex
+            .insert(webhooksToInsert)
+            .into('webhooks');
+    }
+}

package/core/server/services/activitypub/ActivityPubServiceWrapper.js

@@ -1,4 +1,4 @@
-const {ActivityPubService} = require('@tryghost/activitypub');
+const {ActivityPubService} = require('./ActivityPubService');
 
 module.exports = class ActivityPubServiceWrapper {
     /** @type ActivityPubService */

package/core/server/services/link-tracking/ClickEvent.js

@@ -0,0 +1,25 @@
+const ObjectID = require('bson-objectid').default;
+
+module.exports = class ClickEvent {
+    /** @type {ObjectID} */
+    event_id;
+    /** @type {string} */
+    member_uuid;
+    /** @type {ObjectID} */
+    link_id;
+
+    constructor(data) {
+        if (!data.id) {
+            this.event_id = new ObjectID();
+        }
+
+        if (typeof data.id === 'string') {
+            this.event_id = ObjectID.createFromHexString(data.id);
+        } else {
+            this.event_id = data.id;
+        }
+
+        this.member_uuid = data.member_uuid;
+        this.link_id = data.link_id;
+    }
+};

package/core/server/services/link-tracking/FullPostLink.js

@@ -0,0 +1,36 @@
+const ObjectID = require('bson-objectid').default;
+
+/**
+ * @typedef {Object} FullPostLinkCount
+ * @property {number} clicks
+ */
+
+/**
+ * Stores the connection between a LinkRedirect and a Post
+ */
+module.exports = class FullPostLink {
+    /** @type {ObjectID} */
+    post_id;
+    
+    /** @type {import('@tryghost/link-redirects/lib/LinkRedirect')} */
+    link;
+
+    /** @type {FullPostLinkCount} */
+    count;
+
+    /**
+     * @param {object} data
+     * @param {string|ObjectID} data.post_id
+     * @param {import('@tryghost/link-redirects/lib/LinkRedirect')} data.link
+     * @param {FullPostLinkCount} data.count
+     */
+    constructor(data) {
+        if (typeof data.post_id === 'string') {
+            this.post_id = ObjectID.createFromHexString(data.post_id);
+        } else {
+            this.post_id = data.post_id;
+        }
+        this.link = data.link;
+        this.count = data.count;
+    }
+};

package/core/server/services/link-tracking/LinkClickRepository.js

@@ -1,4 +1,4 @@
-const {LinkClick} = require('@tryghost/link-tracking');
+const LinkClick = require('./ClickEvent');
 const ObjectID = require('bson-objectid').default;
 const sentry = require('../../../shared/sentry');
 const config = require('../../../shared/config');

package/core/server/services/link-tracking/LinkClickTrackingService.js

@@ -0,0 +1,237 @@
+const {RedirectEvent} = require('@tryghost/link-redirects');
+const LinkClick = require('./ClickEvent');
+const PostLink = require('./PostLink');
+const ObjectID = require('bson-objectid').default;
+const errors = require('@tryghost/errors');
+const nql = require('@tryghost/nql');
+const _ = require('lodash');
+const tpl = require('@tryghost/tpl');
+const moment = require('moment');
+
+/**
+ * @typedef {object} ILinkClickRepository
+ * @prop {(event: LinkClick) => Promise<void>} save
+ * @prop {({filter: string}) => Promise<LinkClick[]>} getAll
+ */
+
+/**
+ * @typedef {object} ILinkRedirect
+ * @prop {ObjectID} link_id
+ * @prop {URL} to
+ * @prop {URL} from
+ */
+
+/**
+ * @typedef {import('./FullPostLink')} FullPostLink
+ */
+
+/**
+ * @typedef {object} ILinkRedirectService
+ * @prop {(to: URL, slug: string) => Promise<ILinkRedirect>} addRedirect
+ * @prop {() => Promise<string>} getSlug
+ * @prop {({filter: string}) => Promise<ILinkRedirect[]>} getAll
+ * @prop {({filter: string}) => Promise<string[]>} getFilteredIds
+ */
+
+/**
+ * @typedef {object} IPostLinkRepository
+ * @prop {(postLink: PostLink) => Promise<void>} save
+ * @prop {({filter: string}) => Promise<FullPostLink[]>} getAll
+ * @prop {(linkIds: array, data, options) => Promise<FullPostLink[]>} updateLinks
+ */
+
+const messages = {
+    invalidFilter: 'Invalid filter value received',
+    unsupportedBulkAction: 'Unsupported bulk action',
+    invalidRedirectUrl: 'Invalid redirect URL value'
+};
+
+class LinkClickTrackingService {
+    #initialised = false;
+
+    /** @type ILinkClickRepository */
+    #linkClickRepository;
+    /** @type ILinkRedirectService */
+    #linkRedirectService;
+    /** @type IPostLinkRepository */
+    #postLinkRepository;
+    /** @type DomainEvents */
+    #DomainEvents;
+    /** @type {Object} */
+    #LinkRedirect;
+    /** @type {Object} */
+    #urlUtils;
+
+    /**
+     * @param {object} deps
+     * @param {ILinkClickRepository} deps.linkClickRepository
+     * @param {ILinkRedirectService} deps.linkRedirectService
+     * @param {IPostLinkRepository} deps.postLinkRepository
+     * @param {DomainEvents} deps.DomainEvents
+     * @param {urlUtils} deps.urlUtils
+     */
+    constructor(deps) {
+        this.#linkClickRepository = deps.linkClickRepository;
+        this.#linkRedirectService = deps.linkRedirectService;
+        this.#postLinkRepository = deps.postLinkRepository;
+        this.#DomainEvents = deps.DomainEvents;
+        this.#urlUtils = deps.urlUtils;
+    }
+
+    async init() {
+        if (this.#initialised) {
+            return;
+        }
+        this.subscribe();
+        this.#initialised = true;
+    }
+
+    /**
+     * @param {object} options
+     * @param {string} options.filter
+     * @return {Promise<FullPostLink[]>}
+     */
+    async getLinks(options) {
+        return await this.#postLinkRepository.getAll({
+            filter: options.filter
+        });
+    }
+
+    /**
+   * validate and manage the new redirect url in filter
+   * `to` url needs decoding and transformation to relative url for comparision
+   * @param {string} filter
+   * @returns {Object} parsed filter
+   * @throws {errors.BadRequestError}
+   */
+    #parseLinkFilter(filter) {
+        try {
+            const filterJson = nql(filter).parse();
+            const postId = filterJson?.$and?.[0]?.post_id;
+            const redirectUrl = new URL(filterJson?.$and?.[1]?.to);
+            if (!postId || !redirectUrl) {
+                throw new errors.BadRequestError({
+                    message: tpl(messages.invalidFilter)
+                });
+            }
+            return {
+                postId,
+                redirectUrl
+            };
+        } catch (e) {
+            throw new errors.BadRequestError({
+                message: tpl(messages.invalidFilter),
+                context: e.message
+            });
+        }
+    }
+
+    #getRedirectLinkWithAttribution({newLink, oldLink, postId}) {
+        const newUrl = new URL(newLink);
+        const oldUrl = new URL(oldLink);
+        // append newsletter ref query param from oldUrl to newUrl
+        if (oldUrl.searchParams.has('ref')) {
+            newUrl.searchParams.set('ref', oldUrl.searchParams.get('ref'));
+        }
+
+        // append post attribution to site urls
+        const isSite = this.#urlUtils.isSiteUrl(newUrl);
+        if (isSite) {
+            newUrl.searchParams.set('attribution_type', 'post');
+            newUrl.searchParams.set('attribution_id', postId);
+        }
+        return newUrl;
+    }
+
+    async #updateLinks(data, options) {
+        const filterOptions = _.pick(options, ['transacting', 'context', 'filter']);
+
+        // decode and parse filter to manage new redirect url
+        const {postId, redirectUrl} = this.#parseLinkFilter(filterOptions.filter);
+
+        // manages transformation of current url to relative for comparision
+        const transformedOldUrl = this.#urlUtils.absoluteToTransformReady(redirectUrl.href);
+        const filterQuery = `post_id:'${postId}'+to:'${transformedOldUrl}'`;
+
+        const updatedFilterOptions = {
+            ...filterOptions,
+            filter: filterQuery
+        };
+
+        // get new redirect link with proper attribution
+        const newRedirectUrl = this.#getRedirectLinkWithAttribution({
+            newLink: data.meta?.link?.to,
+            oldLink: redirectUrl.href,
+            postId
+        });
+        const linkIds = await this.#linkRedirectService.getFilteredIds(updatedFilterOptions);
+
+        const bulkUpdateOptions = _.pick(options, ['transacting']);
+        const updateData = {
+            to: this.#urlUtils.absoluteToTransformReady(newRedirectUrl.href),
+            updated_at: moment().format('YYYY-MM-DD HH:mm:ss')
+        };
+
+        return await this.#postLinkRepository.updateLinks(linkIds, updateData, bulkUpdateOptions);
+    }
+
+    async bulkEdit(data, options) {
+        if (data.action === 'updateLink') {
+            return await this.#updateLinks(data, options);
+        }
+        throw new errors.IncorrectUsageError({
+            message: tpl(messages.unsupportedBulkAction)
+        });
+    }
+
+    /**
+     * @private (not using # to allow tests)
+     * Replace URL with a redirect that redirects to the original URL, and link that redirect with the given post
+     */
+    async addRedirectToUrl(url, post) {
+        // Generate a unique redirect slug
+        const slugUrl = await this.#linkRedirectService.getSlugUrl();
+
+        // Add redirect for link click tracking
+        const redirect = await this.#linkRedirectService.addRedirect(slugUrl, url);
+
+        // Store a reference of the link against the post
+        const postLink = new PostLink({
+            link_id: redirect.link_id,
+            post_id: ObjectID.createFromHexString(post.id)
+        });
+        await this.#postLinkRepository.save(postLink);
+
+        return redirect.from;
+    }
+
+    /**
+     * Add tracking to a URL and returns a new URL (if link click tracking is enabled)
+     * @param {URL} url
+     * @param {Post} post
+     * @param {string} memberUuid
+     * @return {Promise<URL>}
+     */
+    async addTrackingToUrl(url, post, memberUuid) {
+        url = await this.addRedirectToUrl(url, post);
+        url.searchParams.set('m', memberUuid);
+        return url;
+    }
+
+    subscribe() {
+        this.#DomainEvents.subscribe(RedirectEvent, async (event) => {
+            const uuid = event.data.url.searchParams.get('m');
+            if (!uuid) {
+                return;
+            }
+
+            const click = new LinkClick({
+                member_uuid: uuid,
+                link_id: event.data.link.link_id
+            });
+            await this.#linkClickRepository.save(click);
+        });
+    }
+}
+
+module.exports = LinkClickTrackingService;

package/core/server/services/link-tracking/PostLink.js

@@ -0,0 +1,29 @@
+const ObjectID = require('bson-objectid').default;
+
+/**
+ * Stores the connection between a LinkRedirect and a Post
+ */
+module.exports = class PostLink {
+    /** @type {ObjectID} */
+    post_id;
+    /** @type {ObjectID} */
+    link_id;
+
+    /**
+     * @param {object} data
+     * @param {string|ObjectID} data.post_id
+     * @param {string|ObjectID} data.link_id
+     */
+    constructor(data) {
+        if (typeof data.post_id === 'string') {
+            this.post_id = ObjectID.createFromHexString(data.post_id);
+        } else {
+            this.post_id = data.post_id;
+        }
+        if (typeof data.link_id === 'string') {
+            this.link_id = ObjectID.createFromHexString(data.link_id);
+        } else {
+            this.link_id = data.link_id;
+        }
+    }
+};

package/core/server/services/link-tracking/PostLinkRepository.js

@@ -1,9 +1,9 @@
-const {FullPostLink} = require('@tryghost/link-tracking');
+const FullPostLink = require('./FullPostLink');
 const _ = require('lodash');
 
 /**
  * @typedef {import('bson-objectid').default} ObjectID
- * @typedef {import('@tryghost/link-tracking/lib/PostLink')} PostLink
+ * @typedef {import('./PostLink')} PostLink
  */
 
 module.exports = class PostLinkRepository {

package/core/server/services/link-tracking/index.js

@@ -20,7 +20,7 @@ class LinkTrackingServiceWrapper {
         const {MemberLinkClickEvent} = require('@tryghost/member-events');
         const DomainEvents = require('@tryghost/domain-events');
 
-        const {LinkClickTrackingService} = require('@tryghost/link-tracking');
+        const LinkClickTrackingService = require('./LinkClickTrackingService');
 
         const postLinkRepository = new PostLinkRepository({
             LinkRedirect: models.Redirect,

package/core/server/services/members-events/EventStorage.js

@@ -0,0 +1,61 @@
+const {MemberCreatedEvent, SubscriptionCreatedEvent} = require('@tryghost/member-events');
+
+/**
+ * Store events in the database
+ */
+class EventStorage {
+    /**
+     *
+     * @param {Object} deps
+     * @param {Object} deps.labsService
+     * @param {Object} deps.models
+     * @param {Object} deps.models.MemberCreatedEvent
+     * @param {Object} deps.models.SubscriptionCreatedEvent
+     */
+    constructor({labsService, models}) {
+        this.models = models;
+        this.labsService = labsService;
+    }
+
+    /**
+     * Subscribe to events of this domainEvents service
+     * @param {Object} domainEvents The DomainEvents service
+     */
+    subscribe(domainEvents) {
+        domainEvents.subscribe(MemberCreatedEvent, async (event) => {
+            let attribution = event.data.attribution;
+
+            await this.models.MemberCreatedEvent.add({
+                member_id: event.data.memberId,
+                created_at: event.timestamp,
+                attribution_id: attribution?.id ?? null,
+                attribution_url: attribution?.url ?? null,
+                attribution_type: attribution?.type ?? null,
+                source: event.data.source,
+                referrer_source: attribution?.referrerSource ?? null,
+                referrer_medium: attribution?.referrerMedium ?? null,
+                referrer_url: attribution?.referrerUrl ?? null,
+                batch_id: event.data.batchId ?? null
+            });
+        });
+
+        domainEvents.subscribe(SubscriptionCreatedEvent, async (event) => {
+            let attribution = event.data.attribution;
+
+            await this.models.SubscriptionCreatedEvent.add({
+                member_id: event.data.memberId,
+                subscription_id: event.data.subscriptionId,
+                created_at: event.timestamp,
+                attribution_id: attribution?.id ?? null,
+                attribution_url: attribution?.url ?? null,
+                attribution_type: attribution?.type ?? null,
+                referrer_source: attribution?.referrerSource ?? null,
+                referrer_medium: attribution?.referrerMedium ?? null,
+                referrer_url: attribution?.referrerUrl ?? null,
+                batch_id: event.data.batchId ?? null
+            });
+        });
+    }
+}
+
+module.exports = EventStorage;

package/core/server/services/members-events/LastSeenAtCache.js

@@ -0,0 +1,96 @@
+const moment = require('moment-timezone');
+
+/**
+ * A cache that stores the member ids that have been seen today. This cache is used to avoid having to query the database for the last_seen_at timestamp of a member multiple times in the same day.
+ * 
+ * @constructor
+ * @param {Object} settingsCache - An instance of the settings cache
+ * @property {Set} _cache - A set that stores all the member ids that have been seen today
+ * @property {Object} _settingsCache - An instance of the settings cache
+ * @property {string} _startOfDay - The start of the current day in the site timezone, formatted in ISO 8601
+ */
+class LastSeenAtCache {
+    /**
+     * 
+     * @param {Object} deps - Dependencies
+     * @param {Object} deps.services - The list of service dependencies
+     * @param {Object} deps.services.settingsCache - The settings service
+     */
+    constructor({services: {settingsCache}}) {
+        this._cache = new Set();
+        this._settingsCache = settingsCache;
+        this._startOfDay = this._getStartOfCurrentDay();
+    }
+
+    /**
+     * @method add - Adds a member id to the cache
+     * @param {string} memberId 
+     */
+    add(memberId) {
+        this._cache.add(memberId);
+    }
+
+    /**
+     * @method remove - Removes a member id from the cache
+     * @param {string} memberId 
+     */
+    remove(memberId) {
+        this._cache.delete(memberId);
+    }
+
+    /**
+     * @method shouldUpdateMember - Checks if a member should be updated
+     * @param {string} memberId 
+     * @returns {boolean} - Returns true if the member should be updated
+     */
+    shouldUpdateMember(memberId) {
+        return !this._has(memberId);
+    }
+
+    /**
+     * @method clear - Clears the cache
+     */
+    clear() {
+        this._cache.clear();
+    }
+
+    /**
+     * @method _has - Refreshes the cache and checks if a member id is in the cache
+     * @param {string} memberId 
+     * @returns {boolean}
+     */
+    _has(memberId) {
+        this._refresh();
+        return this._cache.has(memberId);
+    }
+
+    /**
+     * @method _shouldClear - Checks if the cache should be cleared, based on the current day
+     * @returns {boolean} - Returns true if the cache should be cleared
+     */
+    _shouldClear() {
+        return this._startOfDay !== this._getStartOfCurrentDay();
+    }
+
+    /**
+     * @method _refresh - Clears the cache if the day has changed
+     */
+    _refresh() {
+        if (this._shouldClear()) {
+            this.clear();
+            this._startOfDay = this._getStartOfCurrentDay();
+        }
+    }
+
+    /**
+     * Returns the start of the current day in the site timezone
+     * @returns {string} The start of the current day in the site timezone, formatted as a ISO 8601 string
+     */
+    _getStartOfCurrentDay() {
+        const timezone = this._settingsCache.get('timezone') || 'Etc/UTC';
+        const startOfDay = moment().tz(timezone).startOf('day').utc().toISOString();
+        return startOfDay;
+    }
+}
+
+module.exports = LastSeenAtCache;