ghost 6.54.0 → 6.55.0

package/core/server/services/link-tracking/link-click-repository.js

@@ -31,8 +31,8 @@ module.exports = class LinkClickRepository {
         this.#DomainEvents = deps.DomainEvents;
 
         // Memoize the findOne function
-        this.memoizedFindOne = _.memoize(async (uuid) => {
-            return await this.#Member.findOne({uuid});
+        this.memoizedFindOne = _.memoize(async (uuid, options) => {
+            return await this.#Member.findOne({uuid}, options);
         });
     }
 
@@ -54,15 +54,18 @@ module.exports = class LinkClickRepository {
 
     /**
      * @param {LinkClick} linkClick
-     * @returns {Promise<void>}
+     * @param {object} [options]
+     * @param {object} [options.transacting]
+     * @returns {Promise<string | undefined>}
      */
-    async save(linkClick) {
+    async save(linkClick, options = {}) {
         let member;
+        const memberLookupOptions = options.transacting ? {transacting: options.transacting} : undefined;
 
         if (config && config.get('linkClickTrackingCacheMemberUuid')) {
-            member = await this.memoizedFindOne(linkClick.member_uuid);
+            member = await this.memoizedFindOne(linkClick.member_uuid, memberLookupOptions);
         } else {
-            member = await this.#Member.findOne({uuid: linkClick.member_uuid});
+            member = await this.#Member.findOne({uuid: linkClick.member_uuid}, memberLookupOptions);
         }
 
         if (!member) {
@@ -76,11 +79,29 @@ module.exports = class LinkClickRepository {
             // Only store the pathname (no support for variable query strings)
             redirect_id: linkClick.link_id.toHexString(),
             member_id: member.id
-        }, {});
+        }, options);
 
         linkClick.event_id = ObjectID.createFromHexString(model.id);
+        const timestamp = linkClick.timestamp ?? new Date();
+
+        const event = this.#MemberLinkClickEvent.create({
+            memberId: member.id,
+            memberLastSeenAt: member.get('last_seen_at'),
+            linkId: linkClick.link_id.toHexString()
+        }, timestamp);
+
+        if (options.transacting) {
+            options.transacting.executionPromise.then(() => {
+                try {
+                    this.#DomainEvents.dispatch(event);
+                } catch (error) {
+                    sentry.captureException(error);
+                }
+            }, () => undefined);
+        } else {
+            this.#DomainEvents.dispatch(event);
+        }
 
-        // Dispatch event
-        this.#DomainEvents.dispatch(this.#MemberLinkClickEvent.create({memberId: member.id, memberLastSeenAt: member.get('last_seen_at'), linkId: linkClick.link_id.toHexString()}, new Date()));
+        return member.id;
     }
 };

package/core/server/services/link-tracking/link-click-tracking-service.js

@@ -10,7 +10,7 @@ const moment = require('moment');
 
 /**
  * @typedef {object} ILinkClickRepository
- * @prop {(event: LinkClick) => Promise<void>} save
+ * @prop {(event: LinkClick, options?: {transacting?: object}) => Promise<string | undefined>} save
  * @prop {({filter: string}) => Promise<LinkClick[]>} getAll
  */
 
@@ -61,6 +61,10 @@ class LinkClickTrackingService {
     #LinkRedirect;
     /** @type {Object} */
     #urlUtils;
+    /** @type {object} */
+    #automationsApi;
+    /** @type {(callback: (transacting: object) => Promise<void>) => Promise<void>} */
+    #runInTransaction;
 
     /**
      * @param {object} deps
@@ -69,6 +73,8 @@ class LinkClickTrackingService {
      * @param {IPostLinkRepository} deps.postLinkRepository
      * @param {DomainEvents} deps.DomainEvents
      * @param {urlUtils} deps.urlUtils
+     * @param {object} deps.automationsApi
+     * @param {(callback: (transacting: object) => Promise<void>) => Promise<void>} deps.runInTransaction
      */
     constructor(deps) {
         this.#linkClickRepository = deps.linkClickRepository;
@@ -76,6 +82,8 @@ class LinkClickTrackingService {
         this.#postLinkRepository = deps.postLinkRepository;
         this.#DomainEvents = deps.DomainEvents;
         this.#urlUtils = deps.urlUtils;
+        this.#automationsApi = deps.automationsApi;
+        this.#runInTransaction = deps.runInTransaction;
     }
 
     async init() {
@@ -233,6 +241,20 @@ class LinkClickTrackingService {
         return url;
     }
 
+    /**
+     * Add first-party tracking to a URL in an automation email
+     * @param {Readonly<URL>} url
+     * @param {string} automationActionRevisionId
+     * @param {string} memberUuid
+     * @return {Promise<URL>}
+     */
+    async addAutomationTrackingToUrl(url, automationActionRevisionId, memberUuid) {
+        const redirect = await this.#linkRedirectService.getOrAddAutomationRedirect(automationActionRevisionId, url);
+        const trackedUrl = new URL(redirect.from.href);
+        trackedUrl.searchParams.set('m', memberUuid);
+        return trackedUrl;
+    }
+
     subscribe() {
         this.#DomainEvents.subscribe(RedirectEvent, async (event) => {
             const uuid = event.data.url.searchParams.get('m');
@@ -242,9 +264,28 @@ class LinkClickTrackingService {
 
             const click = new LinkClick({
                 member_uuid: uuid,
-                link_id: event.data.link.link_id
+                link_id: event.data.link.link_id,
+                timestamp: event.timestamp
+            });
+
+            const automationActionRevisionId = event.data.link.automationActionRevisionId;
+            if (!automationActionRevisionId) {
+                await this.#linkClickRepository.save(click);
+                return;
+            }
+
+            await this.#runInTransaction(async (transacting) => {
+                const memberId = await this.#linkClickRepository.save(click, {transacting});
+                if (!memberId) {
+                    return;
+                }
+
+                await this.#automationsApi.trackEmailClicked({
+                    automationActionRevisionId,
+                    memberId,
+                    clickedAt: event.timestamp
+                }, {transacting});
             });
-            await this.#linkClickRepository.save(click);
         });
     }
 }

package/core/server/services/member-attribution/outbound-link-tagger.js

@@ -11,7 +11,9 @@ const blockedReferrerDomains = [
     'www.chicagomag.com',
     'bailii.org',
     'www.bailii.org',
-    '*.doubleclick.net'
+    '*.doubleclick.net',
+    // Substack's client app throws on any non-empty ?ref= value and swaps the post for an error screen
+    '*.substack.com'
 ];
 
 const isBlockedReferrerDomain = (domain) => {

package/core/server/services/member-welcome-emails/member-welcome-email-renderer.js

@@ -2,11 +2,13 @@ const fs = require('fs');
 const path = require('path');
 const lexicalLib = require('../../lib/lexical');
 const labs = require('../../../shared/labs');
+const urlUtils = require('../../../shared/url-utils').default;
 const {finalize} = require('../email-rendering/finalize');
 const errors = require('@tryghost/errors');
 const {MESSAGES} = require('./constants');
 const {wrapReplacementStrings} = require('@tryghost/kg-default-nodes').utils.replacementStrings;
 const linkReplacer = require('../lib/link-replacer');
+const linkTracking = require('../link-tracking');
 const emailDesign = require('../email-rendering/email-design');
 const {registerHelpers} = require('../email-service/helpers/register-helpers');
 
@@ -115,9 +117,11 @@ class MemberWelcomeEmailRenderer {
      * @param {Object} options.member - Member data (name, email)
      * @param {Object} options.siteSettings - Site settings (title, url, accentColor)
      * @param {string} [options.unsubscribeUrl] - When set, the footer shows an "Unsubscribe from these emails" link instead of "Manage your preferences"
+     * @param {boolean} [options.trackClicks]
+     * @param {string | null} [options.automationActionRevisionId]
      * @returns {Promise<{html: string, text: string, subject: string}>}
      */
-    async render({lexical, subject, designSettings, member, siteSettings, unsubscribeUrl}) {
+    async render({lexical, subject, designSettings, member, siteSettings, unsubscribeUrl, trackClicks = false, automationActionRevisionId = null}) {
         designSettings = designSettings || {};
 
         const design = emailDesign.getEmailDesign({
@@ -138,7 +142,8 @@ class MemberWelcomeEmailRenderer {
 
         let content;
         try {
-            content = await lexicalLib.render(lexical, {target: 'email', design});
+            const absoluteLexical = urlUtils.transformReadyToAbsolute(lexical);
+            content = await lexicalLib.render(absoluteLexical, {target: 'email', design});
         } catch (err) {
             throw new errors.IncorrectUsageError({
                 message: MESSAGES.INVALID_LEXICAL_STRUCTURE,
@@ -158,8 +163,18 @@ class MemberWelcomeEmailRenderer {
         const contentWithReplacements = this.#applyReplacements({definitions, text: content, escapeHtml: true});
         const subjectWithReplacements = this.#applyReplacements({definitions, text: subject, escapeHtml: false});
 
-        // Resolve relative links (e.g. #/portal/signup) to absolute URLs using the site URL
-        const contentWithAbsoluteLinks = await linkReplacer.replace(contentWithReplacements, (url) => {
+        // Resolve relative links, including Ghost Portal routes such as #/portal/signup,
+        // while preserving document-local anchors such as #section.
+        const contentWithAbsoluteLinks = await linkReplacer.replace(contentWithReplacements, async (url, originalPath) => {
+            const isDocumentAnchor = originalPath.startsWith('#') && !originalPath.startsWith('#/');
+            if (isDocumentAnchor) {
+                return originalPath;
+            }
+            const isTrackable = ['http:', 'https:'].includes(url.protocol);
+            if (trackClicks && automationActionRevisionId && member.uuid && isTrackable) {
+                await linkTracking.init();
+                return await linkTracking.service.addAutomationTrackingToUrl(url, automationActionRevisionId, member.uuid);
+            }
             return url;
         }, {base: siteSettings.url});
 

package/core/server/services/member-welcome-emails/service.js

@@ -368,10 +368,12 @@ class MemberWelcomeEmailService {
      * @param {null | object} options.email.designSettings
      * @param {'welcome' | 'automation'} options.emailType
      * @param {boolean} [options.trackOpens]
+     * @param {boolean} [options.trackClicks]
+     * @param {string | null} [options.automationActionRevisionId]
      * @param {null | {url: string, oneClickUrl: string}} [options.unsubscribe] - When set, the footer links to an unsubscribe URL and the email carries one-click List-Unsubscribe headers
      * @returns {Promise<unknown>}
      */
-    async #sendEmail({member, memberStatus, email, emailType, trackOpens, unsubscribe = null}) {
+    async #sendEmail({member, memberStatus, email, emailType, trackOpens, trackClicks = false, automationActionRevisionId = null, unsubscribe = null}) {
         if (!member.email) {
             throw new errors.IncorrectUsageError({
                 message: MESSAGES.MISSING_RECIPIENT_EMAIL
@@ -396,7 +398,9 @@ class MemberWelcomeEmailService {
                 uuid: member.uuid
             },
             siteSettings: this.#getSiteSettings(),
-            unsubscribeUrl: unsubscribe?.url
+            unsubscribeUrl: unsubscribe?.url,
+            trackClicks,
+            automationActionRevisionId
         });
 
         const senderOptions = await this.#getEffectiveSenderOptions(
@@ -474,9 +478,11 @@ class MemberWelcomeEmailService {
      * @param {string} options.member.uuid
      * @param {'free' | 'paid'} options.memberStatus
      * @param {boolean} options.trackOpens
+     * @param {boolean} options.trackClicks
+     * @param {string} options.automationActionRevisionId
      * @returns {Promise<unknown>}
      */
-    async sendAutomationEmail({email, member, memberStatus, trackOpens}) {
+    async sendAutomationEmail({email, member, memberStatus, trackOpens, trackClicks, automationActionRevisionId}) {
         const designSettings = email.designSettingId ?
             await EmailDesignSetting.findOne({id: email.designSettingId}) :
             null;
@@ -502,6 +508,8 @@ class MemberWelcomeEmailService {
                 designSettings: designSettingsJson
             },
             trackOpens,
+            trackClicks,
+            automationActionRevisionId,
             unsubscribe
         });
     }

package/core/server/services/members/{importer → import-export}/config.js

@@ -2,15 +2,10 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.resolveInlineThreshold = resolveInlineThreshold;
 const zod_1 = require("zod");
-/**
- * Resolving Ghost's config into the values the importer runs on.
- *
- * Config arrives untyped: defaults.json supplies the shipped value, and any
- * higher layer — an operator's config file, an environment variable, argv — can
- * replace it with anything at all. Turning that into something usable is a
- * config concern, so it happens here, at the boundary. The importer is handed an
- * answer and never learns where it came from.
- */
+// Config arrives untyped: defaults.json ships a value, and any higher layer -- an
+// operator's config file, an environment variable, argv -- can replace it with anything.
+// Validating it into a usable number is a config concern, so it happens here at the
+// boundary; the importer is handed a resolved value.
 // The row count at or below which an import is performed while the request is
 // still open. Zero is allowed and means every import is deferred. A numeric
 // string is accepted because a value supplied through an environment variable

package/core/server/services/members/{importer → import-export}/config.ts

@@ -1,14 +1,9 @@
 import {z} from 'zod';
 
-/**
- * Resolving Ghost's config into the values the importer runs on.
- *
- * Config arrives untyped: defaults.json supplies the shipped value, and any
- * higher layer — an operator's config file, an environment variable, argv — can
- * replace it with anything at all. Turning that into something usable is a
- * config concern, so it happens here, at the boundary. The importer is handed an
- * answer and never learns where it came from.
- */
+// Config arrives untyped: defaults.json ships a value, and any higher layer -- an
+// operator's config file, an environment variable, argv -- can replace it with anything.
+// Validating it into a usable number is a config concern, so it happens here at the
+// boundary; the importer is handed a resolved value.
 
 // The row count at or below which an import is performed while the request is
 // still open. Zero is allowed and means every import is deferred. A numeric

package/core/server/services/members/import-export/csv/formula.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.stripFormulaGuard = stripFormulaGuard;
+// The export writes with papaparse's escapeFormulae on (see serialize), which prefixes a
+// cell starting with one of these with an apostrophe so a spreadsheet won't read it as a
+// formula: `=SUM(A1)` is written as `'=SUM(A1)`.
+const FORMULA_TRIGGERS = ['=', '+', '-', '@', '\t', '\r'];
+// Strip that guard back off, so a re-imported value doesn't gain an apostrophe on every
+// round trip. Only a lone apostrophe before a trigger is the guard; one before anything
+// else is a character the value actually starts with, and is left alone.
+function stripFormulaGuard(cell) {
+    if (cell.charAt(0) === '\'' && FORMULA_TRIGGERS.includes(cell.charAt(1))) {
+        return cell.slice(1);
+    }
+    return cell;
+}

package/core/server/services/members/import-export/csv/formula.ts

@@ -0,0 +1,14 @@
+// The export writes with papaparse's escapeFormulae on (see serialize), which prefixes a
+// cell starting with one of these with an apostrophe so a spreadsheet won't read it as a
+// formula: `=SUM(A1)` is written as `'=SUM(A1)`.
+const FORMULA_TRIGGERS = ['=', '+', '-', '@', '\t', '\r'];
+
+// Strip that guard back off, so a re-imported value doesn't gain an apostrophe on every
+// round trip. Only a lone apostrophe before a trigger is the guard; one before anything
+// else is a character the value actually starts with, and is left alone.
+export function stripFormulaGuard(cell: string): string {
+    if (cell.charAt(0) === '\'' && FORMULA_TRIGGERS.includes(cell.charAt(1))) {
+        return cell.slice(1);
+    }
+    return cell;
+}

package/core/server/services/members/import-export/csv/index.js

@@ -0,0 +1,12 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.stripFormulaGuard = exports.serialize = exports.parse = void 0;
+var parse_1 = require("./parse");
+Object.defineProperty(exports, "parse", { enumerable: true, get: function () { return __importDefault(parse_1).default; } });
+var serialize_1 = require("./serialize");
+Object.defineProperty(exports, "serialize", { enumerable: true, get: function () { return __importDefault(serialize_1).default; } });
+var formula_1 = require("./formula");
+Object.defineProperty(exports, "stripFormulaGuard", { enumerable: true, get: function () { return formula_1.stripFormulaGuard; } });

package/core/server/services/members/import-export/csv/index.ts

@@ -0,0 +1,3 @@
+export {default as parse, type Row} from './parse';
+export {default as serialize} from './serialize';
+export {stripFormulaGuard} from './formula';

package/core/server/services/members/import-export/csv/parse.js

@@ -0,0 +1,64 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = parse;
+const node_stream_1 = require("node:stream");
+const papaparse_1 = __importDefault(require("papaparse"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+// A column named after an Object.prototype member (toString, __proto__, ...) is unsafe
+// as a key: writing it would shadow a method or reach the prototype.
+function isSafeColumnName(name) {
+    return !(name in Object.prototype);
+}
+// headerMapping renames the CSV's headers to the columns they emit under; unmapped
+// columns carry through under their own name.
+function parse(path, headerMapping) {
+    return new Promise(function (resolve, reject) {
+        const csvFileStream = fs_extra_1.default.createReadStream(path);
+        const csvParserStream = papaparse_1.default.parse(papaparse_1.default.NODE_STREAM_INPUT, {
+            header: true
+        });
+        const rows = [];
+        const parsedCSVStream = (0, node_stream_1.pipeline)(csvFileStream, csvParserStream, (err) => {
+            if (err) {
+                return reject(err);
+            }
+            resolve(rows);
+        });
+        parsedCSVStream.on('data', (parsedRow) => {
+            // a throw here escapes as an uncaught exception and leaves this
+            // promise forever unsettled, so it has to become a rejection
+            try {
+                const row = {};
+                for (const [header, value] of Object.entries(parsedRow)) {
+                    // papaparse gathers the overflow from a row carrying more
+                    // fields than there are headers under __parsed_extra, as an
+                    // array rather than a cell any mapping can name
+                    if (typeof value !== 'string') {
+                        continue;
+                    }
+                    // hasOwn, not `in`: a prototype-named header would otherwise match an
+                    // inherited method on the mapping and take a function as its mapped name.
+                    if (headerMapping && Object.hasOwn(headerMapping, header)) {
+                        row[headerMapping[header]] = value;
+                    }
+                    else if (isSafeColumnName(header)) {
+                        // Carry any unmapped column through untouched, so the import is not
+                        // constrained to a known vocabulary: a custom_fields.* column
+                        // survives parsing even though nothing consumes it yet.
+                        row[header] = value;
+                    }
+                }
+                if (!Object.keys(row).length) {
+                    return;
+                }
+                rows.push(row);
+            }
+            catch (err) {
+                reject(err);
+            }
+        });
+    });
+}

package/core/server/services/members/import-export/csv/parse.ts

@@ -0,0 +1,69 @@
+import {pipeline} from 'node:stream';
+import papaparse from 'papaparse';
+import fs from 'fs-extra';
+
+// A parsed CSV row: raw string cells, keyed by (renamed) column. Parsing is mechanical;
+// giving the columns meaning and coercing their values is the domain's job (the import
+// schema).
+export type Row = Record<string, string>;
+
+// A column named after an Object.prototype member (toString, __proto__, ...) is unsafe
+// as a key: writing it would shadow a method or reach the prototype.
+function isSafeColumnName(name: string): boolean {
+    return !(name in Object.prototype);
+}
+
+// headerMapping renames the CSV's headers to the columns they emit under; unmapped
+// columns carry through under their own name.
+export default function parse(path: string, headerMapping?: Record<string, string>): Promise<Row[]> {
+    return new Promise(function (resolve, reject) {
+        const csvFileStream = fs.createReadStream(path);
+        const csvParserStream = papaparse.parse(papaparse.NODE_STREAM_INPUT, {
+            header: true
+        });
+
+        const rows: Row[] = [];
+        const parsedCSVStream = pipeline(csvFileStream, csvParserStream, (err) => {
+            if (err) {
+                return reject(err);
+            }
+            resolve(rows);
+        });
+
+        parsedCSVStream.on('data', (parsedRow: Record<string, unknown>) => {
+            // a throw here escapes as an uncaught exception and leaves this
+            // promise forever unsettled, so it has to become a rejection
+            try {
+                const row: Row = {};
+
+                for (const [header, value] of Object.entries(parsedRow)) {
+                    // papaparse gathers the overflow from a row carrying more
+                    // fields than there are headers under __parsed_extra, as an
+                    // array rather than a cell any mapping can name
+                    if (typeof value !== 'string') {
+                        continue;
+                    }
+
+                    // hasOwn, not `in`: a prototype-named header would otherwise match an
+                    // inherited method on the mapping and take a function as its mapped name.
+                    if (headerMapping && Object.hasOwn(headerMapping, header)) {
+                        row[headerMapping[header]] = value;
+                    } else if (isSafeColumnName(header)) {
+                        // Carry any unmapped column through untouched, so the import is not
+                        // constrained to a known vocabulary: a custom_fields.* column
+                        // survives parsing even though nothing consumes it yet.
+                        row[header] = value;
+                    }
+                }
+
+                if (!Object.keys(row).length) {
+                    return;
+                }
+
+                rows.push(row);
+            } catch (err) {
+                reject(err);
+            }
+        });
+    });
+}

package/core/server/services/members/import-export/csv/serialize.js

@@ -0,0 +1,13 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = serialize;
+const papaparse_1 = __importDefault(require("papaparse"));
+// The papaparse boundary: given rows already shaped into cells, it writes CSV and
+// nothing else. escapeFormulae is forced on -- papaparse defaults it off, but this CSV
+// is opened by a person, so a leading =, +, -, @ or tab must be escaped.
+function serialize(rows, { columns, header = true } = {}) {
+    return papaparse_1.default.unparse(rows, { columns, header, escapeFormulae: true });
+}

package/core/server/services/members/import-export/csv/serialize.ts

@@ -0,0 +1,18 @@
+import papaparse from 'papaparse';
+
+interface SerializeOptions {
+    // Omitted, papaparse derives the columns (and their order) from the first row's
+    // keys -- so a typed shaper's row can define them, with no separate list to drift.
+    columns?: string[];
+    header?: boolean;
+}
+
+// The papaparse boundary: given rows already shaped into cells, it writes CSV and
+// nothing else. escapeFormulae is forced on -- papaparse defaults it off, but this CSV
+// is opened by a person, so a leading =, +, -, @ or tab must be escaped.
+export default function serialize(
+    rows: Array<Record<string, unknown>>,
+    {columns, header = true}: SerializeOptions = {}
+): string {
+    return papaparse.unparse(rows, {columns, header, escapeFormulae: true});
+}