ghost 6.63.0 → 6.64.0

package/core/server/services/staff/staff-service-emails.js

@@ -669,7 +669,11 @@ class StaffServiceEmails {
 
     const juice = require('juice');
 
-    return juice(html, { inlinePseudoElements: true, removeStyleTags: true });
+    return juice(html, {
+      inlinePseudoElements: true,
+      removeStyleTags: true,
+      resolveCSSVariables: false,
+    });
   }
 
   async renderText(templateName, data) {

package/core/server/services/stats/stats-service.js

@@ -246,10 +246,19 @@ class StatsService {
     const request = deps.request || require('../../lib/request-external');
     const settingsCache = deps.settingsCache || require('../../../shared/settings-cache');
 
-    if (settingsCache.get('web_analytics_enabled')) {
-      // TODO: move the tinybird client to the tinybird service
-      const TinybirdServiceWrapper = require('../tinybird');
+    const labs = deps.labs || require('../../../shared/labs');
+    const webAnalyticsEnabled = settingsCache.get('web_analytics_enabled');
+    const TinybirdServiceWrapper = require('../tinybird');
+
+    if (
+      webAnalyticsEnabled ||
+      (labs.isSet('automationsTinybirdSync') && config.get('tinybird:stats'))
+    ) {
       TinybirdServiceWrapper.init();
+    }
+
+    if (webAnalyticsEnabled) {
+      // TODO: move the tinybird client to the tinybird service
       tinybirdClient = require('./utils/tinybird').create({
         config,
         request,

package/core/server/services/tier-checkout-config/service.js

@@ -8,11 +8,22 @@ const bson_objectid_1 = __importDefault(require("bson-objectid"));
 const errors_1 = __importDefault(require("@tryghost/errors"));
 const zod_1 = require("zod");
 const schema_1 = require("../members-metafields/schema");
+const members_metafields_1 = require("../members-metafields");
 const checkout_1 = require("@tryghost/checkout");
 const codec_1 = require("./codec");
 const queries_1 = require("./queries");
 const models_1 = require("./models");
 const serializers_1 = require("./serializers");
+/**
+ * What a member may do with a field this service creates for them.
+ *
+ * The opposite of the default a publisher-made field gets. These hold what the member
+ * themselves gave at checkout, and collection can be switched on while the screen for
+ * opening a field to members is not, since the two sit behind different flags. A closed
+ * default would then leave a member unable to correct their own address and nobody able
+ * to open it for them.
+ */
+const COLLECTED_FIELD_ACCESS = { member: members_metafields_1.MEMBER_ACCESS.write };
 /**
  * A request states its settings in named sections: one for shipping, one for phone. Each
  * section stands for one or more of the values Stripe hands back when a checkout is
@@ -142,7 +153,12 @@ class TierCheckoutConfigService {
                 });
             }
             if (!alreadyPlanned) {
-                create.set(key, { key, name: wants.name, type: wants.type });
+                create.set(key, {
+                    key,
+                    name: wants.name,
+                    type: wants.type,
+                    access: COLLECTED_FIELD_ACCESS,
+                });
             }
         }
         return { clear, create: [...create.values()], bind: wanted };

package/core/server/services/tinybird/tinybird-service.js

@@ -58,6 +58,7 @@ const TINYBIRD_PIPES = [
   'api_top_utm_terms',
   'api_top_devices',
   'api_gift_link_visits',
+  'api_automation_browse_stats',
   // v2 pipes (materialized view optimization)
   'api_kpis_v2',
   'api_active_visitors_v2',

package/core/server/services/tinybird-sync/get-ingest-config.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getIngestConfig = getIngestConfig;
+const zod_1 = require("zod");
+const StatsConfigSchema = zod_1.z.object({
+    id: zod_1.z.string().min(1).optional(),
+});
+const SiteUuidSchema = zod_1.z.string().min(1);
+const SyncAuthKeySchema = zod_1.z.string().min(1);
+const TrackerEndpointSchema = zod_1.z.url();
+const ANALYTICS_PATH_PREFIX = '/.ghost/analytics/';
+function getIngestConfig({ config, settingsCache, }) {
+    const trackerEndpointResult = TrackerEndpointSchema.safeParse(config.get('tinybird:tracker:endpoint'));
+    const syncAuthKeyResult = SyncAuthKeySchema.safeParse(config.get('tinybird:sync_auth_key'));
+    if (!trackerEndpointResult.success || !syncAuthKeyResult.success) {
+        return null;
+    }
+    const endpoint = new URL(trackerEndpointResult.data);
+    const analyticsPathIndex = endpoint.pathname.indexOf(ANALYTICS_PATH_PREFIX);
+    if (analyticsPathIndex === -1) {
+        return null;
+    }
+    endpoint.pathname = `${endpoint.pathname.slice(0, analyticsPathIndex + ANALYTICS_PATH_PREFIX.length)}api/v1/tinybird-sync`;
+    const statsResult = StatsConfigSchema.safeParse(config.get('tinybird:stats'));
+    const settingsSiteUuidResult = SiteUuidSchema.safeParse(settingsCache.get('site_uuid'));
+    let siteUuid = statsResult.success ? statsResult.data.id : undefined;
+    if (!siteUuid && settingsSiteUuidResult.success) {
+        siteUuid = settingsSiteUuidResult.data;
+    }
+    if (!siteUuid) {
+        return null;
+    }
+    return {
+        endpoint,
+        // secretlint-disable-next-line @secretlint/secretlint-rule-pattern
+        trafficAnalyticsAuth: syncAuthKeyResult.data,
+        siteUuid,
+    };
+}

package/core/server/services/tinybird-sync/index.js

@@ -0,0 +1,28 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.start = void 0;
+const promises_1 = require("node:timers/promises");
+const bson_objectid_1 = __importDefault(require("bson-objectid"));
+const logging_1 = __importDefault(require("@tryghost/logging"));
+const config_1 = __importDefault(require("../../../shared/config"));
+// @ts-expect-error This module lacks type definitions.
+const settings_cache_1 = __importDefault(require("../../../shared/settings-cache"));
+const db_1 = require("../../data/db");
+const tinybird_sync_service_1 = require("./tinybird-sync-service");
+const service = (0, tinybird_sync_service_1.createTinybirdSyncService)({
+    config: config_1.default,
+    settingsCache: settings_cache_1.default,
+    knex: db_1.knex,
+    logging: logging_1.default,
+    sleep: async (ms) => {
+        await (0, promises_1.setTimeout)(ms, undefined, { ref: false });
+    },
+    random: Math.random,
+    now: () => new Date(),
+    fetch: globalThis.fetch,
+    createId: () => (0, bson_objectid_1.default)().toHexString(),
+});
+exports.start = service.start;

package/core/server/services/tinybird-sync/sync-table-to-tinybird.js

@@ -0,0 +1,149 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SAFETY_LAG_MS = exports.AUTOMATION_SYNC_TARGETS = void 0;
+exports.syncTableToTinybird = syncTableToTinybird;
+const errors_1 = __importDefault(require("@tryghost/errors"));
+const date_1 = require("../../lib/db-types/date");
+// Only what the Tinybird materialized views read. Member identity and lock bookkeeping
+// stay in MySQL.
+exports.AUTOMATION_SYNC_TARGETS = [
+    {
+        table: 'automation_runs',
+        columns: ['id', 'automation_id', 'created_at', 'updated_at'],
+    },
+    {
+        table: 'automation_run_steps',
+        columns: [
+            'id',
+            'automation_run_id',
+            'automation_action_revision_id',
+            'status',
+            'step_attempts',
+            'ready_at',
+            'started_at',
+            'finished_at',
+            'created_at',
+            'updated_at',
+        ],
+    },
+];
+// A row's updated_at is assigned before its transaction commits, so a row can become
+// visible after rows with later timestamps. Holding back the newest rows keeps the
+// watermark from moving past a row whose transaction is still open.
+exports.SAFETY_LAG_MS = 60 * 1000;
+const serializeValue = (key, value) => key.endsWith('_at') && value !== null && value !== undefined
+    ? (0, date_1.fromDatabaseDate)(value).toISOString()
+    : value;
+const toEventLine = (row, siteUuid, type) => {
+    const payload = Object.fromEntries(Object.entries(row).map(([key, value]) => [key, serializeValue(key, value)]));
+    return JSON.stringify({
+        type,
+        site_uuid: siteUuid,
+        id: row.id,
+        updated_at: (0, date_1.fromDatabaseDate)(row.updated_at).toISOString(),
+        payload: { ...payload, site_uuid: siteUuid },
+    });
+};
+function* chunkByPayloadLimits(lines, maxBytes, maxMessages) {
+    let current = [];
+    let currentBytes = 0;
+    for (const line of lines) {
+        const bytes = Buffer.byteLength(line) + '\n'.length;
+        if (current.length && (currentBytes + bytes > maxBytes || current.length === maxMessages)) {
+            yield current;
+            current = [];
+            currentBytes = 0;
+        }
+        current.push(line);
+        currentBytes += bytes;
+    }
+    if (current.length) {
+        yield current;
+    }
+}
+async function postEvents(lines, { table }, { endpoint, trafficAnalyticsAuth, siteUuid, fetch, requestTimeoutMs }) {
+    const response = await fetch(endpoint, {
+        method: 'POST',
+        headers: {
+            Authorization: `Bearer ${trafficAnalyticsAuth}`,
+            'Content-Type': 'application/x-ndjson',
+            'x-site-uuid': siteUuid,
+        },
+        body: lines.join('\n'),
+        signal: AbortSignal.timeout(requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new errors_1.default.InternalServerError({
+            message: `Traffic Analytics returned ${response.status} while syncing ${table}: ${await response.text()}`,
+        });
+    }
+}
+async function readWatermark(knex, table) {
+    const row = await knex('tinybird_syncs')
+        .select('last_synced_updated_at', 'last_synced_id')
+        .where({ table_name: table })
+        .first();
+    if (!row) {
+        return null;
+    }
+    return { updatedAt: (0, date_1.fromDatabaseDate)(row.last_synced_updated_at), id: row.last_synced_id };
+}
+async function writeWatermark(knex, table, cursor, now, createId) {
+    const timestamp = (0, date_1.toDatabaseDate)(now);
+    const values = {
+        last_synced_updated_at: (0, date_1.toDatabaseDate)(cursor.updatedAt),
+        last_synced_id: cursor.id,
+        updated_at: timestamp,
+    };
+    await knex('tinybird_syncs')
+        .insert({
+        id: createId(),
+        table_name: table,
+        ...values,
+        created_at: timestamp,
+    })
+        .onConflict('table_name')
+        .merge(values);
+}
+async function readBatch({ table, columns }, { knex, batchSize }, cursor, cutoff) {
+    const query = knex(table)
+        .select(columns)
+        .where('updated_at', '<', cutoff)
+        .orderBy([{ column: 'updated_at' }, { column: 'id' }])
+        .limit(batchSize);
+    if (cursor) {
+        const updatedAt = (0, date_1.toDatabaseDate)(cursor.updatedAt);
+        query.andWhere((builder) => builder
+            .where('updated_at', '>', updatedAt)
+            .orWhere((tie) => tie.where('updated_at', updatedAt).andWhere('id', '>', cursor.id)));
+    }
+    return await query;
+}
+async function syncTableToTinybird(target, options) {
+    const { knex, siteUuid, now, createId, batchSize, maxPayloadBytes, maxPayloadMessages } = options;
+    const cutoff = (0, date_1.toDatabaseDate)(new Date(now().getTime() - exports.SAFETY_LAG_MS));
+    let cursor = await readWatermark(knex, target.table);
+    let sent = 0;
+    while (true) {
+        const rows = await readBatch(target, options, cursor, cutoff);
+        if (!rows.length) {
+            return sent;
+        }
+        const lines = rows.map((row) => toEventLine(row, siteUuid, target.table));
+        for (const chunk of chunkByPayloadLimits(lines, maxPayloadBytes, maxPayloadMessages)) {
+            await postEvents(chunk, target, options);
+        }
+        const last = rows[rows.length - 1];
+        cursor = { updatedAt: (0, date_1.fromDatabaseDate)(last.updated_at), id: last.id };
+        await writeWatermark(knex, target.table, cursor, now(), createId);
+        sent += rows.length;
+        // This is a performance optimization to prevent one needless batch read.
+        // It is not required for correctness.
+        if (rows.length < batchSize) {
+            return sent;
+        }
+    }
+}

package/core/server/services/tinybird-sync/tinybird-sync-service.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createTinybirdSyncService = createTinybirdSyncService;
+const get_ingest_config_1 = require("./get-ingest-config");
+const sync_table_to_tinybird_1 = require("./sync-table-to-tinybird");
+const INTERVAL_MS = 5 * 60 * 1000;
+const BATCH_SIZE = 5000;
+// This should be a little less than the maximum, because rows are chunked by
+// JSON line, not bytes strictly.
+const MAX_PAYLOAD_BYTES = 9 * 1024 * 1024;
+const MAX_PAYLOAD_MESSAGES = 1000;
+const REQUEST_TIMEOUT_MS = 5 * 60 * 1000;
+function createTinybirdSyncService({ config, settingsCache, knex, logging, sleep, random, now, fetch, createId, }) {
+    let started = false;
+    const syncAll = async (ingest) => {
+        const results = await Promise.allSettled(sync_table_to_tinybird_1.AUTOMATION_SYNC_TARGETS.map(async (target) => {
+            const sent = await (0, sync_table_to_tinybird_1.syncTableToTinybird)(target, {
+                knex,
+                ...ingest,
+                now,
+                fetch,
+                createId,
+                batchSize: BATCH_SIZE,
+                maxPayloadBytes: MAX_PAYLOAD_BYTES,
+                maxPayloadMessages: MAX_PAYLOAD_MESSAGES,
+                requestTimeoutMs: REQUEST_TIMEOUT_MS,
+            });
+            logging.info({ system: { event: 'tinybird.sync.completed', table: target.table, sent } }, `[Tinybird sync] ${target.table}: sent ${sent} rows`);
+        }));
+        for (const result of results) {
+            if (result.status === 'rejected') {
+                logging.error(result.reason, '[Tinybird sync] Failed to sync table');
+            }
+        }
+    };
+    const runLoop = async (ingest) => {
+        await sleep(Math.floor(random() * INTERVAL_MS));
+        while (true) {
+            await syncAll(ingest);
+            await sleep(INTERVAL_MS);
+        }
+    };
+    const start = () => {
+        if (started) {
+            return;
+        }
+        const ingest = (0, get_ingest_config_1.getIngestConfig)({ config, settingsCache });
+        if (!ingest) {
+            logging.info('[Tinybird sync] Not started: Traffic Analytics service is not configured');
+            return;
+        }
+        started = true;
+        void runLoop(ingest)
+            .then(() => {
+            logging.error('[Tinybird sync] Loop stopped unexpectedly');
+        })
+            .catch((error) => {
+            logging.error(error, '[Tinybird sync] Loop stopped');
+        });
+    };
+    return { start };
+}

package/core/server/web/api/endpoints/admin/routes.js

@@ -5,6 +5,7 @@ const auth = require('../../../../services/auth');
 const apiMw = require('../../middleware');
 const mw = require('./middleware');
 const labs = require('../../../../../shared/labs');
+const limits = require('../../../../services/limits');
 
 const shared = require('../../../shared');
 
@@ -195,40 +196,38 @@ module.exports = function apiRoutes() {
 
   router.get('/members/stripe_connect', mw.authAdminApi, http(api.membersStripeConnect.auth));
 
-  // Reading definitions is deliberately not behind the feature flag: Admin asks every site
-  // for them, and a site without the flag simply has none, so the answer is an empty list
-  // rather than a 404. Creating and changing them is flagged. Registered before /members/:id
-  // so the literal path is not captured as an id.
-  router.get('/members/metafields/:namespace', mw.authAdminApi, http(api.membersMetafields.browse));
-  router.post(
-    '/members/metafields/:namespace',
-    mw.authAdminApi,
-    labs.enabledMiddleware('membersCustomFields'),
-    http(api.membersMetafields.add),
-  );
-  router.put(
-    '/members/metafields/:namespace',
-    mw.authAdminApi,
-    labs.enabledMiddleware('membersCustomFields'),
-    http(api.membersMetafields.reorder),
-  );
-  router.get(
-    '/members/metafields/:namespace/:key',
-    mw.authAdminApi,
-    http(api.membersMetafields.read),
-  );
-  router.put(
-    '/members/metafields/:namespace/:key',
-    mw.authAdminApi,
-    labs.enabledMiddleware('membersCustomFields'),
-    http(api.membersMetafields.edit),
-  );
-  router.delete(
-    '/members/metafields/:namespace/:key',
-    mw.authAdminApi,
-    labs.enabledMiddleware('membersCustomFields'),
-    http(api.membersMetafields.destroy),
-  );
+  // Custom field definitions. Mounted rather than listed so every route under it is
+  // reached the same way, and so the guards are stated once each instead of on every
+  // route that needs them.
+  //
+  // Order carries the rule here, the way Express reads it: a request walks this stack
+  // from the top, so the reads below are answered before the guards are reached, and
+  // everything registered after them passes through both. A route added at the end is
+  // guarded by being there, which is the safer way round to forget.
+  //
+  // Mounted before /members/:id so the literal path is not captured as an id.
+  const metafieldsRouter = express.Router('admin api members metafields');
+  router.use('/members/metafields', metafieldsRouter);
+
+  metafieldsRouter.use(mw.authAdminApi);
+
+  // Reading is deliberately open: Admin asks every site for its definitions to draw
+  // screens it renders either way, and a site that has none simply answers with an empty
+  // list rather than a 404.
+  metafieldsRouter.get('/:namespace', http(api.membersMetafields.browse));
+  metafieldsRouter.get('/:namespace/:key', http(api.membersMetafields.read));
+
+  // Changing one needs the feature to exist in this build and the site's plan to include
+  // it. Two separate questions, asked once each: a 404 says the feature is not here, a 403
+  // says the plan does not cover it, and only the second is something a publisher can act
+  // on.
+  metafieldsRouter.use(labs.enabledMiddleware('membersCustomFields'));
+  metafieldsRouter.use(limits.requireFeature('limitCustomFields'));
+
+  metafieldsRouter.post('/:namespace', http(api.membersMetafields.add));
+  metafieldsRouter.put('/:namespace', http(api.membersMetafields.reorder));
+  metafieldsRouter.put('/:namespace/:key', http(api.membersMetafields.edit));
+  metafieldsRouter.delete('/:namespace/:key', http(api.membersMetafields.destroy));
 
   router.get('/members/:id', mw.authAdminApi, http(api.members.read));
   router.put('/members/:id', mw.authAdminApi, http(api.members.edit));

package/core/server/web/well-known.js

@@ -2,35 +2,31 @@ const express = require('../../shared/express');
 const settings = require('../../shared/settings-cache');
 const config = require('../../shared/config');
 const { cacheControl } = require('./shared/middleware');
+const { getPublicKeyInfo } = require('../lib/public-jwk');
 
 module.exports = function setupWellKnownApp() {
   const wellKnownApp = express('well-known');
 
-  const jose = require('node-jose');
   const dangerousPrivateKey = settings.get('ghost_private_key');
-  const keyStore = jose.JWK.createKeyStore();
-  const keyStoreReady = keyStore.add(dangerousPrivateKey, 'pem');
-
-  const getSafePublicJWKS = async () => {
-    await keyStoreReady;
-    return keyStore.toJSON();
-  };
+  const keyReady = getPublicKeyInfo(dangerousPrivateKey);
 
   const cache = cacheControl('public', { maxAge: config.get('caching:wellKnown:maxAge') });
 
   wellKnownApp.get('/jwks.json', cache, async function jwksMiddleware(req, res) {
-    const jwks = await getSafePublicJWKS();
+    const { kid, jwk } = await keyReady;
 
     // there's only one key in the store atm
     // based on this setting all of the keys to have
     // "use": "sig" property
-    const keys = jwks.keys.map((key) => ({
-      e: key.e,
-      kid: key.kid,
-      kty: key.kty,
-      n: key.n,
-      use: 'sig',
-    }));
+    const keys = [
+      {
+        e: jwk.e,
+        kid,
+        kty: jwk.kty,
+        n: jwk.n,
+        use: 'sig',
+      },
+    ];
 
     res.json({ keys });
   });

package/core/shared/config/defaults.json

@@ -305,8 +305,7 @@
         "enabled": false,
         "threshold": 500
       }
-    },
-    "openedJobLagWarningMinutes": 30
+    }
   },
   "linkClickTrackingCacheMemberUuid": false,
   "backgroundJobs": {

package/core/shared/{flush-logs.js → flush.js}

@@ -3,15 +3,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.flushLogs = flushLogs;
+exports.flushLogsAndMetrics = flushLogsAndMetrics;
 const promises_1 = require("node:timers/promises");
 const logging_1 = __importDefault(require("@tryghost/logging"));
+const metrics_1 = __importDefault(require("@tryghost/metrics"));
 const DEFAULT_TIMEOUT_MS = 2000;
 /**
  * Drain buffered log transports before the process exits, bounded so a stuck
  * transport can never block shutdown.
  */
-async function flushLogs(timeoutMs = DEFAULT_TIMEOUT_MS) {
+async function flushLogsAndMetrics(timeoutMs = DEFAULT_TIMEOUT_MS) {
     // Unreffed so the loser of the race can't hold the event loop open
-    await Promise.race([logging_1.default.flush(), (0, promises_1.setTimeout)(timeoutMs, undefined, { ref: false })]);
+    await Promise.race([
+        Promise.all([logging_1.default.flush(), metrics_1.default.flush()]),
+        (0, promises_1.setTimeout)(timeoutMs, undefined, { ref: false }),
+    ]);
 }

package/core/shared/labs.js

@@ -41,6 +41,7 @@ const PUBLIC_BETA_FEATURES = [
 // Which is only visible if the developer experiments flag is enabled
 const PRIVATE_FEATURES = [
   'automations',
+  'automationsPerTier',
   'automationRunAnalytics',
   'automationsTinybirdSync',
   'stripeAutomaticTax',
@@ -60,6 +61,7 @@ const PRIVATE_FEATURES = [
   'selfServeArchives',
   'machinePayments',
   'postsListReact',
+  'membersActivityReact',
   'editorReact',
 ];
 

package/core/shared/timing-safe-string-equal.js

@@ -0,0 +1,26 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.timingSafeStringEqual = timingSafeStringEqual;
+const crypto_1 = __importDefault(require("crypto"));
+/**
+ * Compares a value Ghost computed (an HMAC digest, say) with one taken from a
+ * request, without the response time revealing how much of it matched.
+ * Anything that is not a string never matches: a query param repeated in the
+ * URL arrives as an array, and that is as wrong as a bad key.
+ */
+function timingSafeStringEqual(expected, provided) {
+    if (typeof provided !== 'string') {
+        return false;
+    }
+    const expectedBuffer = Buffer.from(expected, 'utf16le');
+    const providedBuffer = Buffer.from(provided, 'utf16le');
+    // crypto.timingSafeEqual throws when the lengths differ. The digests compared
+    // here have a fixed, public length, so returning early gives nothing away.
+    if (expectedBuffer.length !== providedBuffer.length) {
+        return false;
+    }
+    return crypto_1.default.timingSafeEqual(expectedBuffer, providedBuffer);
+}