ghost 5.115.0 → 5.116.0

package/core/server/services/recommendations/service/BookshelfSubscribeEventRepository.ts

@@ -0,0 +1,49 @@
+import {BookshelfRepository, ModelClass, ModelInstance} from './BookshelfRepository';
+import logger from '@tryghost/logging';
+import {SubscribeEvent} from './SubscribeEvent';
+
+type Sentry = {
+    captureException(err: unknown): void;
+}
+
+export class BookshelfSubscribeEventRepository extends BookshelfRepository<string, SubscribeEvent> {
+    sentry?: Sentry;
+
+    constructor(Model: ModelClass<string>, deps: {sentry?: Sentry} = {}) {
+        super(Model);
+        this.sentry = deps.sentry;
+    }
+
+    toPrimitive(entity: SubscribeEvent): object {
+        return {
+            id: entity.id,
+            recommendation_id: entity.recommendationId,
+            member_id: entity.memberId,
+            created_at: entity.createdAt
+        };
+    }
+
+    modelToEntity(model: ModelInstance<string>): SubscribeEvent | null {
+        try {
+            return SubscribeEvent.create({
+                id: model.id,
+                recommendationId: model.get('recommendation_id') as string,
+                memberId: model.get('member_id') as string,
+                createdAt: model.get('created_at') as Date
+            });
+        } catch (err) {
+            logger.error(err);
+            this.sentry?.captureException(err);
+            return null;
+        }
+    }
+
+    getFieldToColumnMap() {
+        return {
+            id: 'id',
+            recommendationId: 'recommendation_id',
+            memberId: 'member_id',
+            createdAt: 'created_at'
+        } as Record<keyof SubscribeEvent, string>;
+    }
+}

package/core/server/services/recommendations/service/ClickEvent.js

@@ -0,0 +1,33 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClickEvent = void 0;
+const bson_objectid_1 = __importDefault(require("bson-objectid"));
+class ClickEvent {
+    id;
+    recommendationId;
+    memberId;
+    createdAt;
+    get deleted() {
+        return false;
+    }
+    constructor(data) {
+        this.id = data.id;
+        this.recommendationId = data.recommendationId;
+        this.memberId = data.memberId;
+        this.createdAt = data.createdAt;
+    }
+    static create(data) {
+        const id = data.id ?? (0, bson_objectid_1.default)().toString();
+        const d = {
+            id,
+            recommendationId: data.recommendationId,
+            memberId: data.memberId ?? null,
+            createdAt: data.createdAt ?? new Date()
+        };
+        return new ClickEvent(d);
+    }
+}
+exports.ClickEvent = ClickEvent;

package/core/server/services/recommendations/service/ClickEvent.ts

@@ -0,0 +1,32 @@
+import ObjectId from 'bson-objectid';
+
+export class ClickEvent {
+    id: string;
+    recommendationId: string;
+    memberId: string|null;
+    createdAt: Date;
+
+    get deleted() {
+        return false;
+    }
+
+    private constructor(data: {id: string, recommendationId: string, memberId: string|null, createdAt: Date}) {
+        this.id = data.id;
+        this.recommendationId = data.recommendationId;
+        this.memberId = data.memberId;
+        this.createdAt = data.createdAt;
+    }
+
+    static create(data: {id?: string, recommendationId: string, memberId?: string|null, createdAt?: Date}) {
+        const id = data.id ?? ObjectId().toString();
+
+        const d = {
+            id,
+            recommendationId: data.recommendationId,
+            memberId: data.memberId ?? null,
+            createdAt: data.createdAt ?? new Date()
+        };
+
+        return new ClickEvent(d);
+    }
+}

package/core/server/services/recommendations/service/InMemoryRecommendationRepository.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InMemoryRecommendationRepository = void 0;
+const InMemoryRepository_1 = require("../../lib/InMemoryRepository");
+class InMemoryRecommendationRepository extends InMemoryRepository_1.InMemoryRepository {
+    toPrimitive(entity) {
+        return entity;
+    }
+    async getByUrl(url) {
+        //  Find URL based on the hostname and pathname.
+        //  Query params, hash fragements, protocol and www are ignored.
+        const existing = this.store.find((r) => {
+            return r.url.hostname.replace('www.', '') === url.hostname.replace('www.', '') &&
+                r.url.pathname.replace(/\/$/, '') === url.pathname.replace(/\/$/, '');
+        }) || null;
+        return existing;
+    }
+}
+exports.InMemoryRecommendationRepository = InMemoryRecommendationRepository;

package/core/server/services/recommendations/service/InMemoryRecommendationRepository.ts

@@ -0,0 +1,20 @@
+import {Recommendation} from './Recommendation';
+import {RecommendationRepository} from './RecommendationRepository';
+import {InMemoryRepository} from '../../lib/InMemoryRepository';
+
+export class InMemoryRecommendationRepository extends InMemoryRepository<string, Recommendation> implements RecommendationRepository {
+    toPrimitive(entity: Recommendation): object {
+        return entity;
+    }
+
+    async getByUrl(url: URL): Promise<Recommendation | null > {
+        //  Find URL based on the hostname and pathname.
+        //  Query params, hash fragements, protocol and www are ignored.
+        const existing = this.store.find((r) => {
+            return r.url.hostname.replace('www.', '') === url.hostname.replace('www.', '') &&
+                   r.url.pathname.replace(/\/$/, '') === url.pathname.replace(/\/$/, '');
+        }) || null;
+
+        return existing;
+    }
+}

package/core/server/services/recommendations/service/IncomingRecommendationController.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IncomingRecommendationController = void 0;
+const UnsafeData_1 = require("./UnsafeData");
+class IncomingRecommendationController {
+    service;
+    constructor(deps) {
+        this.service = deps.service;
+    }
+    async browse(frame) {
+        const options = new UnsafeData_1.UnsafeData(frame.options);
+        const page = options.optionalKey('page')?.integer ?? 1;
+        const limit = options.optionalKey('limit')?.integer ?? 5;
+        const { incomingRecommendations, meta } = await this.service.listIncomingRecommendations({ page, limit });
+        return this.#serialize(incomingRecommendations, meta);
+    }
+    #serialize(recommendations, meta) {
+        return {
+            data: recommendations.map((entity) => {
+                return {
+                    id: entity.id,
+                    title: entity.title,
+                    excerpt: entity.excerpt,
+                    featured_image: entity.featuredImage?.toString() ?? null,
+                    favicon: entity.favicon?.toString() ?? null,
+                    url: entity.url.toString(),
+                    recommending_back: !!entity.recommendingBack
+                };
+            }),
+            meta
+        };
+    }
+}
+exports.IncomingRecommendationController = IncomingRecommendationController;

package/core/server/services/recommendations/service/IncomingRecommendationController.ts

@@ -0,0 +1,51 @@
+import {IncomingRecommendationService} from './IncomingRecommendationService';
+import {IncomingRecommendation} from './IncomingRecommendationService';
+import {UnsafeData} from './UnsafeData';
+
+type Frame = {
+    data: unknown,
+    options: unknown,
+};
+
+type Meta = {
+    pagination: object,
+}
+
+export class IncomingRecommendationController {
+    service: IncomingRecommendationService;
+
+    constructor(deps: {service: IncomingRecommendationService}) {
+        this.service = deps.service;
+    }
+
+    async browse(frame: Frame) {
+        const options = new UnsafeData(frame.options);
+
+        const page = options.optionalKey('page')?.integer ?? 1;
+        const limit = options.optionalKey('limit')?.integer ?? 5;
+
+        const {incomingRecommendations, meta} = await this.service.listIncomingRecommendations({page, limit});
+
+        return this.#serialize(
+            incomingRecommendations,
+            meta
+        );
+    }
+
+    #serialize(recommendations: IncomingRecommendation[], meta?: Meta) {
+        return {
+            data: recommendations.map((entity) => {
+                return {
+                    id: entity.id,
+                    title: entity.title,
+                    excerpt: entity.excerpt,
+                    featured_image: entity.featuredImage?.toString() ?? null,
+                    favicon: entity.favicon?.toString() ?? null,
+                    url: entity.url.toString(),
+                    recommending_back: !!entity.recommendingBack
+                };
+            }),
+            meta
+        };
+    }
+}

package/core/server/services/recommendations/service/IncomingRecommendationEmailRenderer.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IncomingRecommendationEmailRenderer = void 0;
+class IncomingRecommendationEmailRenderer {
+    #staffService;
+    constructor({ staffService }) {
+        this.#staffService = staffService;
+    }
+    async renderSubject(recommendation) {
+        return `👍 New recommendation: ${recommendation.title}`;
+    }
+    async renderHTML(recommendation, recipient) {
+        return this.#staffService.api.emails.renderHTML('recommendation-received', {
+            recommendation,
+            recipient
+        });
+    }
+    async renderText(recommendation, recipient) {
+        return this.#staffService.api.emails.renderText('recommendation-received', {
+            recommendation,
+            recipient
+        });
+    }
+}
+exports.IncomingRecommendationEmailRenderer = IncomingRecommendationEmailRenderer;

package/core/server/services/recommendations/service/IncomingRecommendationEmailRenderer.ts

@@ -0,0 +1,37 @@
+import {EmailRecipient} from './IncomingRecommendationService';
+import {IncomingRecommendation} from './IncomingRecommendationService';
+
+type StaffService = {
+    api: {
+        emails: {
+            renderHTML(template: string, data: unknown): Promise<string>,
+            renderText(template: string, data: unknown): Promise<string>
+        }
+    }
+}
+
+export class IncomingRecommendationEmailRenderer {
+    #staffService: StaffService;
+
+    constructor({staffService}: {staffService: StaffService}) {
+        this.#staffService = staffService;
+    }
+
+    async renderSubject(recommendation: IncomingRecommendation) {
+        return `👍 New recommendation: ${recommendation.title}`;
+    }
+
+    async renderHTML(recommendation: IncomingRecommendation, recipient: EmailRecipient) {
+        return this.#staffService.api.emails.renderHTML('recommendation-received', {
+            recommendation,
+            recipient
+        });
+    }
+
+    async renderText(recommendation: IncomingRecommendation, recipient: EmailRecipient) {
+        return this.#staffService.api.emails.renderText('recommendation-received', {
+            recommendation,
+            recipient
+        });
+    }
+}

package/core/server/services/recommendations/service/IncomingRecommendationService.js

@@ -0,0 +1,93 @@
+"use strict";
+/* eslint-disable ghost/filenames/match-exported-class */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IncomingRecommendationService = void 0;
+const logging_1 = __importDefault(require("@tryghost/logging"));
+class IncomingRecommendationService {
+    #mentionsApi;
+    #recommendationService;
+    #emailService;
+    #emailRenderer;
+    #getEmailRecipients;
+    constructor(deps) {
+        this.#recommendationService = deps.recommendationService;
+        this.#mentionsApi = deps.mentionsApi;
+        this.#emailService = deps.emailService;
+        this.#emailRenderer = deps.emailRenderer;
+        this.#getEmailRecipients = deps.getEmailRecipients;
+    }
+    async init() {
+        // When we boot, it is possible that we missed some webmentions from other sites recommending you
+        // More importantly, we might have missed some deletes which we can detect.
+        // So we do a slow revalidation of all incoming recommendations
+        // This also prevents doing multiple external fetches when doing quick reboots of Ghost after each other (requires Ghost to be up for at least 15 seconds)
+        if (!process.env.NODE_ENV?.startsWith('test') && process.env.NODE_ENV !== 'development') {
+            setTimeout(() => {
+                logging_1.default.info('Updating incoming recommendations on boot');
+                this.#updateIncomingRecommendations().catch((err) => {
+                    logging_1.default.error('Failed to update incoming recommendations on boot', err);
+                });
+            }, 15 * 1000 + Math.random() * 5 * 60 * 1000);
+        }
+    }
+    #getMentionFilter() {
+        return `source:~$'/.well-known/recommendations.json'`;
+    }
+    async #updateIncomingRecommendations() {
+        // We refresh all incoming recommendations, including:
+        // - recommendations that were not verified, as the verification could have failed
+        // - recommendations that were deleted previously. Implementation note: given that we have `deleted:false` as default filter in the Mention model, we need to override it here
+        const filter = this.#getMentionFilter() + '+deleted:[true,false]';
+        await this.#mentionsApi.refreshMentions({ filter, limit: 100 });
+    }
+    async #mentionToIncomingRecommendation(mention) {
+        try {
+            const url = new URL(mention.source.toString().replace(/\/.well-known\/recommendations\.json$/, ''));
+            // Check if we are also recommending this URL
+            const existing = await this.#recommendationService.readRecommendationByUrl(url);
+            const recommendingBack = !!existing;
+            return {
+                id: mention.id,
+                title: mention.sourceSiteTitle || mention.sourceTitle,
+                url,
+                excerpt: mention.sourceExcerpt,
+                favicon: mention.sourceFavicon,
+                featuredImage: mention.sourceFeaturedImage,
+                recommendingBack
+            };
+        }
+        catch (e) {
+            logging_1.default.error('Failed to parse mention to incoming recommendation data type', e);
+        }
+        return null;
+    }
+    async sendRecommendationEmail(mention) {
+        const recommendation = await this.#mentionToIncomingRecommendation(mention);
+        if (!recommendation) {
+            return;
+        }
+        const recipients = await this.#getEmailRecipients();
+        for (const recipient of recipients) {
+            const subject = await this.#emailRenderer.renderSubject(recommendation);
+            const html = await this.#emailRenderer.renderHTML(recommendation, recipient);
+            const text = await this.#emailRenderer.renderText(recommendation, recipient);
+            await this.#emailService.send(recipient.email, subject, html, text);
+        }
+    }
+    async listIncomingRecommendations(options) {
+        const page = options.page ?? 1;
+        const limit = options.limit ?? 5;
+        const filter = this.#getMentionFilter();
+        const mentions = await this.#mentionsApi.listMentions({ filter, page, limit });
+        const mentionsToIncomingRecommendations = await Promise.all(mentions.data.map(mention => this.#mentionToIncomingRecommendation(mention)));
+        const incomingRecommendations = mentionsToIncomingRecommendations.filter((recommendation) => !!recommendation);
+        return {
+            incomingRecommendations,
+            meta: mentions.meta
+        };
+    }
+}
+exports.IncomingRecommendationService = IncomingRecommendationService;

package/core/server/services/recommendations/service/IncomingRecommendationService.ts

@@ -0,0 +1,160 @@
+/* eslint-disable ghost/filenames/match-exported-class */
+
+import {IncomingRecommendationEmailRenderer} from './IncomingRecommendationEmailRenderer';
+import {RecommendationService} from './RecommendationService';
+import logging from '@tryghost/logging';
+
+export type IncomingRecommendation = {
+    id: string;
+    title: string;
+    url: URL;
+    excerpt: string|null;
+    favicon: URL|null;
+    featuredImage: URL|null;
+    recommendingBack: boolean;
+}
+
+export type Report = {
+    startDate: Date,
+    endDate: Date,
+    recommendations: IncomingRecommendation[]
+}
+
+type Mention = {
+    id: string,
+    source: URL,
+    sourceTitle: string,
+    sourceSiteTitle: string|null,
+    sourceAuthor: string|null,
+    sourceExcerpt: string|null,
+    sourceFavicon: URL|null,
+    sourceFeaturedImage: URL|null
+}
+
+type MentionMeta = {
+    pagination: {
+        page: number;
+        limit: number;
+        pages: number;
+        total: number;
+        next: null | number;
+        prev: null | number;
+    }
+}
+
+type MentionsAPI = {
+    refreshMentions(options: {filter: string, limit: number|'all'}): Promise<void>
+    listMentions(options: {filter: string, page: number, limit: number|'all'}): Promise<{data: Mention[], meta?: MentionMeta}>
+}
+
+export type EmailRecipient = {
+    email: string
+}
+
+type EmailService = {
+    send(to: string, subject: string, html: string, text: string): Promise<void>
+}
+
+export class IncomingRecommendationService {
+    #mentionsApi: MentionsAPI;
+    #recommendationService: RecommendationService;
+
+    #emailService: EmailService;
+    #emailRenderer: IncomingRecommendationEmailRenderer;
+    #getEmailRecipients: () => Promise<EmailRecipient[]>;
+
+    constructor(deps: {
+        recommendationService: RecommendationService,
+        mentionsApi: MentionsAPI,
+        emailService: EmailService,
+        emailRenderer: IncomingRecommendationEmailRenderer,
+        getEmailRecipients: () => Promise<EmailRecipient[]>,
+    }) {
+        this.#recommendationService = deps.recommendationService;
+        this.#mentionsApi = deps.mentionsApi;
+        this.#emailService = deps.emailService;
+        this.#emailRenderer = deps.emailRenderer;
+        this.#getEmailRecipients = deps.getEmailRecipients;
+    }
+
+    async init() {
+        // When we boot, it is possible that we missed some webmentions from other sites recommending you
+        // More importantly, we might have missed some deletes which we can detect.
+        // So we do a slow revalidation of all incoming recommendations
+        // This also prevents doing multiple external fetches when doing quick reboots of Ghost after each other (requires Ghost to be up for at least 15 seconds)
+        if (!process.env.NODE_ENV?.startsWith('test') && process.env.NODE_ENV !== 'development') {
+            setTimeout(() => {
+                logging.info('Updating incoming recommendations on boot');
+                this.#updateIncomingRecommendations().catch((err) => {
+                    logging.error('Failed to update incoming recommendations on boot', err);
+                });
+            }, 15 * 1000 + Math.random() * 5 * 60 * 1000);
+        }
+    }
+
+    #getMentionFilter() {
+        return `source:~$'/.well-known/recommendations.json'`;
+    }
+
+    async #updateIncomingRecommendations() {
+        // We refresh all incoming recommendations, including:
+        // - recommendations that were not verified, as the verification could have failed
+        // - recommendations that were deleted previously. Implementation note: given that we have `deleted:false` as default filter in the Mention model, we need to override it here
+        const filter = this.#getMentionFilter() + '+deleted:[true,false]';
+        await this.#mentionsApi.refreshMentions({filter, limit: 100});
+    }
+
+    async #mentionToIncomingRecommendation(mention: Mention): Promise<IncomingRecommendation|null> {
+        try {
+            const url = new URL(mention.source.toString().replace(/\/.well-known\/recommendations\.json$/, ''));
+
+            // Check if we are also recommending this URL
+            const existing = await this.#recommendationService.readRecommendationByUrl(url);
+            const recommendingBack = !!existing;
+
+            return {
+                id: mention.id,
+                title: mention.sourceSiteTitle || mention.sourceTitle,
+                url,
+                excerpt: mention.sourceExcerpt,
+                favicon: mention.sourceFavicon,
+                featuredImage: mention.sourceFeaturedImage,
+                recommendingBack
+            };
+        } catch (e) {
+            logging.error('Failed to parse mention to incoming recommendation data type', e);
+        }
+        return null;
+    }
+
+    async sendRecommendationEmail(mention: Mention) {
+        const recommendation = await this.#mentionToIncomingRecommendation(mention);
+        if (!recommendation) {
+            return;
+        }
+        const recipients = await this.#getEmailRecipients();
+
+        for (const recipient of recipients) {
+            const subject = await this.#emailRenderer.renderSubject(recommendation);
+            const html = await this.#emailRenderer.renderHTML(recommendation, recipient);
+            const text = await this.#emailRenderer.renderText(recommendation, recipient);
+
+            await this.#emailService.send(recipient.email, subject, html, text);
+        }
+    }
+
+    async listIncomingRecommendations(options: { page?: number; limit?: number|'all'}): Promise<{ incomingRecommendations: IncomingRecommendation[]; meta?: MentionMeta }> {
+        const page = options.page ?? 1;
+        const limit = options.limit ?? 5;
+        const filter = this.#getMentionFilter();
+
+        const mentions = await this.#mentionsApi.listMentions({filter, page, limit});
+        const mentionsToIncomingRecommendations = await Promise.all(mentions.data.map(mention => this.#mentionToIncomingRecommendation(mention)));
+        const incomingRecommendations = mentionsToIncomingRecommendations.filter((recommendation): recommendation is IncomingRecommendation => !!recommendation);
+
+        return {
+            incomingRecommendations,
+            meta: mentions.meta
+        };
+    }
+}