ghost 6.54.0 → 6.55.0

package/core/server/services/members/import-export/import/spool.ts

@@ -0,0 +1,38 @@
+import os from 'node:os';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import fs from 'fs-extra';
+import type {MemberImportRow} from './row';
+
+// A handle to rows written to a spool: the deferred job reads them back once it
+// runs, then removes the file. Removal swallows errors -- a failed cleanup must
+// never fail the import it was cleaning up after.
+export interface SpooledRows {
+    read(): Promise<MemberImportRow[]>;
+    remove(): Promise<void>;
+}
+
+export interface RowSpool {
+    write(rows: MemberImportRow[]): Promise<SpooledRows>;
+}
+
+// Spools import rows to a private JSON file under the OS temp dir, so a deferred
+// import can hand them to a background job and read them back after the request has
+// already returned. The rows go in and come out as MemberImportRow, so nothing but
+// the import's own row shape crosses this boundary.
+export function createRowSpool(): RowSpool {
+    return {
+        async write(rows) {
+            const spoolPath = path.join(os.tmpdir(), `members-import-${crypto.randomUUID()}.json`);
+            await fs.writeFile(spoolPath, JSON.stringify(rows), {mode: 0o600});
+            return {
+                async read() {
+                    return JSON.parse(await fs.readFile(spoolPath, 'utf8'));
+                },
+                async remove() {
+                    await fs.remove(spoolPath).catch(() => {});
+                }
+            };
+        }
+    };
+}

package/core/server/services/members/import-export/index.js

@@ -0,0 +1,99 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.makeImporter = makeImporter;
+exports.makeExporter = makeExporter;
+const importer_1 = __importDefault(require("./import/importer"));
+const reader_1 = __importDefault(require("./import/reader"));
+const spool_1 = require("./import/spool");
+const exporter_1 = __importDefault(require("./export/exporter"));
+const MembersCSVImporterStripeUtils = require('./import/stripe-utils');
+const db = require('../../../data/db');
+const models = require('../../../models');
+const labs = require('../../../../shared/labs');
+// Build the members CSV importer. This is the composition root: today's models and
+// services are wired behind the collaborators the importer declares, one per
+// concern, so nothing Bookshelf-shaped leaks into the import service itself.
+function makeImporter(deps) {
+    // The members repository resolves asynchronously and is stable once ready, so
+    // cache the promise and reuse it across every call the import makes.
+    let membersRepositoryPromise;
+    const getMembersRepository = () => (membersRepositoryPromise ??= deps.getMembersRepository());
+    // The members aggregate, plus the import label lookup folded in so the label
+    // (member-tagging data) does not need a source of its own.
+    const members = {
+        get: async (query, options) => (await getMembersRepository()).get(query, options),
+        create: async (values, options) => (await getMembersRepository()).create(values, options),
+        update: async (values, options) => (await getMembersRepository()).update(values, options),
+        getCustomerIdByEmail: async (email) => (await getMembersRepository()).getCustomerIdByEmail(email),
+        linkStripeCustomer: async (link, options) => (await getMembersRepository()).linkStripeCustomer(link, options),
+        getImportLabel: name => models.Label.findOne({ name })
+    };
+    // The completion email: its recipient, links and delivery in one collaborator.
+    const email = {
+        send: deps.sendEmail,
+        getDefaultRecipient: async () => (await models.User.getOwnerUser()).get('email'),
+        urlFor: deps.urlFor
+    };
+    // Gifts is initialised at boot and always present at request time; the getter
+    // resolves it lazily so the ready service is picked up whenever a row uses it.
+    const gifts = {
+        reassignRedeemer: (giftId, memberId, options) => deps.getGiftService().reassignRedeemer(giftId, memberId, options)
+    };
+    // Gated by the same labs flag as the export, so the two halves round-trip or stay
+    // silent together: off, activeFields resolves empty and every custom_fields.* column
+    // is dropped.
+    const customFields = {
+        activeFields: async () => (labs.isSet('membersCustomFields') ? deps.customFields.definitions.browse() : []),
+        planWrite: values => deps.customFields.values.planWrite(values),
+        applyWrite: (memberId, plan, executor) => deps.customFields.values.applyWrite(memberId, plan, executor)
+    };
+    return new importer_1.default({
+        knex: deps.knex,
+        readRows: reader_1.default,
+        spool: (0, spool_1.createRowSpool)(),
+        members,
+        tiers: {
+            getDefault: deps.getDefaultTier,
+            getByName: deps.getTierByName
+        },
+        stripe: new MembersCSVImporterStripeUtils({
+            stripeAPIService: deps.stripeAPIService,
+            productRepository: deps.productRepository
+        }),
+        gifts,
+        customFields,
+        email,
+        addJob: deps.addJob,
+        getTimezone: deps.getTimezone,
+        getInlineThreshold: deps.getInlineThreshold
+    });
+}
+// Build the members CSV exporter. The same composition root from the other direction:
+// knex and the members id lookup are wired here, and the custom fields definitions and
+// values services are injected (boot builds them before this one). The labs flag alone
+// decides whether custom field columns appear, so nothing flag-shaped leaks into the
+// exporter itself.
+function makeExporter({ definitions, values }) {
+    const exporter = new exporter_1.default({
+        knex: db.knex,
+        members: {
+            // Minimal query, only to fetch the ids of the filtered members; the stream
+            // reads their related data itself.
+            findFilteredIds: async (options) => {
+                const page = await models.Member.findPage({ ...options, withRelated: [], columns: ['id'], limit: 'all' });
+                return page.data.map((member) => member.id);
+            }
+        },
+        customFields: {
+            // Boot builds the definitions and values services before this one, so they
+            // are always present -- no not-initialised state to guard. The flag decides
+            // whether their columns are included at all.
+            activeDefinitions: async () => (labs.isSet('membersCustomFields') ? definitions.browse() : []),
+            valuesForMembers: memberIds => values.getValuesForMembers(memberIds)
+        }
+    });
+    return (options = {}) => exporter.export(options);
+}

package/core/server/services/members/import-export/index.ts

@@ -0,0 +1,137 @@
+import type {Knex} from 'knex';
+import type {CsvField} from '@tryghost/custom-field-types/csv';
+import MembersCSVImporter, {type MembersRepository, type GiftService, type EmailNotifications, type Tier, type CustomFieldsImport} from './import/importer';
+import readMemberRows from './import/reader';
+import {createRowSpool} from './import/spool';
+import MembersCSVExporter, {type ExportOptions, type CustomFieldDefinition} from './export/exporter';
+
+const MembersCSVImporterStripeUtils = require('./import/stripe-utils');
+const db = require('../../../data/db');
+const models = require('../../../models');
+const labs = require('../../../../shared/labs');
+
+// The raw collaborators the members service hands the import composition root, before
+// they are adapted into the ports the importer declares. The members repository is the
+// importer's aggregate minus the import-label lookup, which the root folds in itself.
+interface ImporterServices {
+    knex: Knex;
+    getMembersRepository(): Promise<Omit<MembersRepository, 'getImportLabel'>>;
+    getDefaultTier(): Promise<Tier>;
+    getTierByName(name: string): Promise<Tier | null>;
+    getGiftService(): GiftService;
+    sendEmail: EmailNotifications['send'];
+    urlFor: EmailNotifications['urlFor'];
+    addJob(job: {job: () => Promise<void>; offloaded: boolean; name: string}): void;
+    getTimezone(): string;
+    getInlineThreshold(): number;
+    stripeAPIService: unknown;
+    productRepository: unknown;
+    // The custom fields services the members service hands the import composition root.
+    customFields: {
+        definitions: {browse(): Promise<CsvField[]>};
+        values: {
+            planWrite(values: Record<string, unknown>): Promise<unknown[]>;
+            applyWrite(memberId: string, plan: unknown[], executor: Knex): Promise<void>;
+        };
+    };
+}
+
+// The custom fields services the members service hands the export composition root.
+interface CustomFieldsServices {
+    definitions: {browse(): Promise<CustomFieldDefinition[]>};
+    values: {getValuesForMembers(memberIds: string[]): Promise<Map<string, Record<string, unknown>>>};
+}
+
+// Build the members CSV importer. This is the composition root: today's models and
+// services are wired behind the collaborators the importer declares, one per
+// concern, so nothing Bookshelf-shaped leaks into the import service itself.
+export function makeImporter(deps: ImporterServices) {
+    // The members repository resolves asynchronously and is stable once ready, so
+    // cache the promise and reuse it across every call the import makes.
+    let membersRepositoryPromise: Promise<Omit<MembersRepository, 'getImportLabel'>> | undefined;
+    const getMembersRepository = () => (membersRepositoryPromise ??= deps.getMembersRepository());
+
+    // The members aggregate, plus the import label lookup folded in so the label
+    // (member-tagging data) does not need a source of its own.
+    const members: MembersRepository = {
+        get: async (query, options) => (await getMembersRepository()).get(query, options),
+        create: async (values, options) => (await getMembersRepository()).create(values, options),
+        update: async (values, options) => (await getMembersRepository()).update(values, options),
+        getCustomerIdByEmail: async email => (await getMembersRepository()).getCustomerIdByEmail(email),
+        linkStripeCustomer: async (link, options) => (await getMembersRepository()).linkStripeCustomer(link, options),
+        getImportLabel: name => models.Label.findOne({name})
+    };
+
+    // The completion email: its recipient, links and delivery in one collaborator.
+    const email: EmailNotifications = {
+        send: deps.sendEmail,
+        getDefaultRecipient: async () => (await models.User.getOwnerUser()).get('email'),
+        urlFor: deps.urlFor
+    };
+
+    // Gifts is initialised at boot and always present at request time; the getter
+    // resolves it lazily so the ready service is picked up whenever a row uses it.
+    const gifts: GiftService = {
+        reassignRedeemer: (giftId, memberId, options) => deps.getGiftService().reassignRedeemer(giftId, memberId, options)
+    };
+
+    // Gated by the same labs flag as the export, so the two halves round-trip or stay
+    // silent together: off, activeFields resolves empty and every custom_fields.* column
+    // is dropped.
+    const customFields: CustomFieldsImport = {
+        activeFields: async () => (labs.isSet('membersCustomFields') ? deps.customFields.definitions.browse() : []),
+        planWrite: values => deps.customFields.values.planWrite(values),
+        applyWrite: (memberId, plan, executor) => deps.customFields.values.applyWrite(memberId, plan, executor)
+    };
+
+    return new MembersCSVImporter({
+        knex: deps.knex,
+        readRows: readMemberRows,
+        spool: createRowSpool(),
+        members,
+        tiers: {
+            getDefault: deps.getDefaultTier,
+            getByName: deps.getTierByName
+        },
+        stripe: new MembersCSVImporterStripeUtils({
+            stripeAPIService: deps.stripeAPIService,
+            productRepository: deps.productRepository
+        }),
+        gifts,
+        customFields,
+        email,
+        addJob: deps.addJob,
+        getTimezone: deps.getTimezone,
+        getInlineThreshold: deps.getInlineThreshold
+    });
+}
+
+// Build the members CSV exporter. The same composition root from the other direction:
+// knex and the members id lookup are wired here, and the custom fields definitions and
+// values services are injected (boot builds them before this one). The labs flag alone
+// decides whether custom field columns appear, so nothing flag-shaped leaks into the
+// exporter itself.
+export function makeExporter({definitions, values}: CustomFieldsServices): (options?: ExportOptions) => Promise<NodeJS.ReadableStream> {
+    const exporter = new MembersCSVExporter({
+        knex: db.knex,
+
+        members: {
+            // Minimal query, only to fetch the ids of the filtered members; the stream
+            // reads their related data itself.
+            findFilteredIds: async (options) => {
+                const page = await models.Member.findPage({...options, withRelated: [], columns: ['id'], limit: 'all'});
+                return page.data.map((member: {id: string}) => member.id);
+            }
+        },
+
+        customFields: {
+            // Boot builds the definitions and values services before this one, so they
+            // are always present -- no not-initialised state to guard. The flag decides
+            // whether their columns are included at all.
+            activeDefinitions: async (): Promise<CustomFieldDefinition[]> => (labs.isSet('membersCustomFields') ? definitions.browse() : []),
+            valuesForMembers: memberIds => values.getValuesForMembers(memberIds)
+        }
+    });
+
+    return (options = {}) => exporter.export(options);
+}

package/core/server/services/members/members-api/controllers/router-controller.js

@@ -467,6 +467,13 @@ module.exports = class RouterController {
                 });
             }
 
+            if (offer.status && offer.status !== 'active') {
+                throw new BadRequestError({
+                    message: tpl(messages.offerArchived),
+                    context: 'Offer with id "' + offerId + '" is no longer active'
+                });
+            }
+
             if (!offer.tier) {
                 throw new BadRequestError({
                     message: 'Offer does not have a tier'

package/core/server/services/members/service.js

@@ -4,13 +4,12 @@ const tpl = require('@tryghost/tpl');
 const MembersSSR = require('./members-ssr');
 const db = require('../../data/db');
 const MembersConfigProvider = require('./members-config-provider');
-const makeMembersCSVImporter = require('./importer');
-const {resolveInlineThreshold} = require('./importer/config');
+const {makeImporter, makeExporter} = require('./import-export');
+const {resolveInlineThreshold} = require('./import-export/config');
 const MembersStats = require('./stats/members-stats');
 const memberJobs = require('./jobs');
 const logging = require('@tryghost/logging');
 const urlUtils = require('../../../shared/url-utils').default;
-const labsService = require('../../../shared/labs');
 const settingsCache = require('../../../shared/settings-cache');
 const config = require('../../../shared/config');
 const models = require('../../models');
@@ -47,9 +46,11 @@ const membersStats = new MembersStats({
 let membersApi;
 let verificationTrigger;
 
-const initMembersCSVImporter = ({stripeAPIService}) => {
-    return makeMembersCSVImporter({
-        storagePath: config.getContentPath('data'),
+const buildImporterDeps = ({stripeAPIService}) => {
+    // Required here, not statically: boot builds the custom fields services before this
+    // one (the exporter below relies on the same).
+    const customFields = require('../members-custom-fields');
+    return {
         getTimezone: () => settingsCache.get('timezone'),
         // A getter rather than a value because the threshold is an operator
         // setting that can change between requests
@@ -79,16 +80,16 @@ const initMembersCSVImporter = ({stripeAPIService}) => {
         },
         getGiftService: () => giftService.service,
         sendEmail: ghostMailer.send.bind(ghostMailer),
-        isSet: flag => labsService.isSet(flag),
         addJob: jobsService.addJob.bind(jobsService),
         knex: db.knex,
         urlFor: urlUtils.urlFor.bind(urlUtils),
-        context: {
-            importer: true
-        },
         stripeAPIService,
-        productRepository: membersApi.productRepository
-    });
+        productRepository: membersApi.productRepository,
+        customFields: {
+            definitions: customFields.definitions,
+            values: customFields.values
+        }
+    };
 };
 
 const initVerificationTrigger = () => {
@@ -154,19 +155,23 @@ module.exports = {
         }
         module.exports.verificationTrigger = verificationTrigger;
 
-        const membersCSVImporter = initMembersCSVImporter({stripeAPIService: stripeService.api});
+        const importerDeps = buildImporterDeps({stripeAPIService: stripeService.api});
+        const membersCSVImporter = makeImporter(importerDeps);
+        // The importer takes plain arguments and returns the domain outcome; callers build
+        // the request and shape the response. importCSV decides inline vs deferred by load
+        // (the endpoint); importInline always runs now (the Revue data import). This only
+        // supplies the verification trigger, a members-service internal.
+        module.exports.importCSV = (request) => membersCSVImporter.importCSV(request, verificationTrigger);
+        module.exports.importInline = (request) => membersCSVImporter.importInline(request, verificationTrigger);
+
         // Constructed here rather than required statically: the exporter needs the
         // custom fields services, which boot builds before this one.
         const customFields = require('../members-custom-fields');
-        module.exports.export = require('./exporter/query')({
+        module.exports.export = makeExporter({
             definitions: customFields.definitions,
             values: customFields.values
         });
 
-        module.exports.processImport = async (options) => {
-            return await membersCSVImporter.process({...options, verificationTrigger});
-        };
-
         if (!env?.startsWith('testing')) {
             const membersMigrationJobName = 'members-migrations';
             if (!(await jobsService.hasExecutedSuccessfully(membersMigrationJobName))) {
@@ -205,10 +210,10 @@ module.exports = {
 
     stripeConnect: require('./stripe-connect'),
 
-    processImport: null,
+    importCSV: null,
+    importInline: null,
 
     stats: membersStats,
-
     export: null
 };
 

package/core/server/services/members-custom-fields/values-service.js

@@ -137,8 +137,9 @@ class CustomFieldValuesService {
     /**
      * Resolve input into the writes it implies, rejecting anything invalid, and
      * writing nothing. Returned so a caller can validate before it commits to a
-     * change it would have to unwind (the member edit validates up front), then
-     * apply the same plan without re-resolving or re-validating.
+     * change it would have to unwind (the member edit and the importer both validate
+     * up front, before opening a transaction), then apply the same plan without
+     * re-resolving or re-validating.
      */
     async planWrite(input) {
         const values = this.parseValues(input);
@@ -191,14 +192,17 @@ class CustomFieldValuesService {
      * Apply a plan from `planWrite`.
      *
      * Merge, not replace: only the fields in the plan are touched, so a caller
-     * that doesn't know about a field can't erase it. The whole plan is applied in
-     * one transaction, so a mid-batch failure rolls the batch back.
+     * that doesn't know about a field can't erase it.
+     *
+     * Always transactional, so a mid-batch failure rolls the batch back. Passed an
+     * executor it joins that transaction -- the importer passes its per-member one, so a
+     * failed value write takes the member with it; passed nothing it opens its own.
      */
-    async applyWrite(memberId, writes) {
+    async applyWrite(memberId, writes, executor = this.knex) {
         if (writes.length === 0) {
             return;
         }
-        await this.knex.transaction(async (trx) => {
+        const apply = async (trx) => {
             for (const { field, value } of writes) {
                 const target = { member_id: memberId, custom_field_id: field.id };
                 if (value === undefined) {
@@ -213,7 +217,15 @@ class CustomFieldValuesService {
                     .onConflict(['member_id', 'custom_field_id'])
                     .merge({ ...valueColumns, updated_at: new Date() });
             }
-        });
+        };
+        // isTransaction is knex's marker for a transactor: join an existing transaction
+        // rather than nesting a savepoint under it, otherwise open one.
+        if (executor.isTransaction) {
+            await apply(executor);
+        }
+        else {
+            await executor.transaction(apply);
+        }
     }
 }
 exports.CustomFieldValuesService = CustomFieldValuesService;

package/core/server/services/members-custom-fields/values-service.ts

@@ -164,8 +164,9 @@ export class CustomFieldValuesService {
     /**
      * Resolve input into the writes it implies, rejecting anything invalid, and
      * writing nothing. Returned so a caller can validate before it commits to a
-     * change it would have to unwind (the member edit validates up front), then
-     * apply the same plan without re-resolving or re-validating.
+     * change it would have to unwind (the member edit and the importer both validate
+     * up front, before opening a transaction), then apply the same plan without
+     * re-resolving or re-validating.
      */
     async planWrite(input: unknown): Promise<PlannedWrite[]> {
         const values = this.parseValues(input);
@@ -225,15 +226,18 @@ export class CustomFieldValuesService {
      * Apply a plan from `planWrite`.
      *
      * Merge, not replace: only the fields in the plan are touched, so a caller
-     * that doesn't know about a field can't erase it. The whole plan is applied in
-     * one transaction, so a mid-batch failure rolls the batch back.
+     * that doesn't know about a field can't erase it.
+     *
+     * Always transactional, so a mid-batch failure rolls the batch back. Passed an
+     * executor it joins that transaction -- the importer passes its per-member one, so a
+     * failed value write takes the member with it; passed nothing it opens its own.
      */
-    async applyWrite(memberId: string, writes: PlannedWrite[]): Promise<void> {
+    async applyWrite(memberId: string, writes: PlannedWrite[], executor: Knex = this.knex): Promise<void> {
         if (writes.length === 0) {
             return;
         }
 
-        await this.knex.transaction(async (trx) => {
+        const apply = async (trx: Knex) => {
             for (const {field, value} of writes) {
                 const target = {member_id: memberId, custom_field_id: field.id};
 
@@ -250,6 +254,14 @@ export class CustomFieldValuesService {
                     .onConflict(['member_id', 'custom_field_id'])
                     .merge({...valueColumns, updated_at: new Date()});
             }
-        });
+        };
+
+        // isTransaction is knex's marker for a transactor: join an existing transaction
+        // rather than nesting a savepoint under it, otherwise open one.
+        if (executor.isTransaction) {
+            await apply(executor);
+        } else {
+            await executor.transaction(apply);
+        }
     }
 }

package/core/server/services/mentions/service.js

@@ -29,7 +29,11 @@ async function getPostData(post) {
 
 function getPostUrl(id, postData) {
     const jsonModel = {...postData};
-    outputSerializerUrlUtil.forPost(id, jsonModel, {options: {}});
+    // The URL service routes by resource type. Pages and posts share the Post
+    // model, so the page's own type must reach forPost — otherwise it defaults
+    // to 'posts', matches no post collection, and 404s under the lazy service.
+    const type = postData.type === 'page' ? 'pages' : 'posts';
+    outputSerializerUrlUtil.forPost(id, jsonModel, {options: {}}, type);
     return jsonModel.url;
 }
 

package/core/server/services/posts/posts-exporter.js

@@ -83,7 +83,7 @@ class PostsExporter {
      * @param {string|number} [options.limit]
      * @returns {Promise<Readable>}
      */
-    async export({filter, order, limit}) {
+    async export({filter, order, limit, mongoTransformer}) {
         const exportContext = await this.#getExportContext();
         const requestedLimit = parseExportLimit(limit);
         const pageLimit = Math.min(EXPORT_BATCH_SIZE, requestedLimit);
@@ -93,7 +93,8 @@ class PostsExporter {
             order: withStableExportOrder(order),
             requestedLimit,
             pageLimit,
-            exportContext
+            exportContext,
+            mongoTransformer
         }), {objectMode: true});
     }
 
@@ -119,7 +120,7 @@ class PostsExporter {
      * @param {ExportContext} options.exportContext
      * @returns {AsyncGenerator<object>}
      */
-    async *#streamPosts({filter, order, requestedLimit, pageLimit, exportContext}) {
+    async *#streamPosts({filter, order, requestedLimit, pageLimit, exportContext, mongoTransformer}) {
         let emitted = 0;
 
         for (let page = 1; emitted < requestedLimit; page++) {
@@ -129,7 +130,8 @@ class PostsExporter {
                 limit: pageLimit,
                 page,
                 skipPagination: true,
-                withRelated: EXPORT_WITH_RELATED
+                withRelated: EXPORT_WITH_RELATED,
+                mongoTransformer
             });
 
             if (posts.data.length === 0) {

package/core/server/services/posts/posts-service.js

@@ -329,8 +329,8 @@ class PostsService {
         return result;
     }
 
-    async export(frame) {
-        return await this.postsExporter.export(frame.options);
+    async export(options) {
+        return await this.postsExporter.export(options);
     }
 
     async #updatePosts(data, options) {