ghost 5.115.1 → 5.116.1

package/core/server/data/importer/handlers/ImporterContentFileHandler.js

@@ -0,0 +1,90 @@
+const _ = require('lodash');
+const path = require('path');
+
+class ImporterContentFileHandler {
+    /** @property {'media' | 'files' | 'images'} */
+    type;
+
+    /** @property {string[]} */
+    directories;
+
+    /** @property {string[]} */
+    extensions;
+
+    /** @property {string[]} */
+    contentTypes;
+
+    /**
+     * Holds path to the destination content directory
+     * @property {string} */
+    #contentPath;
+
+    /**
+     *
+     * @param {Object} deps dependencies
+     * @param {'media' | 'files' | 'images'} deps.type type of content file
+     * @param {string[]} deps.extensions file extensions to search for
+     * @param {boolean} [deps.ignoreRootFolderFiles] whether to ignore files in the root folder
+     * @param {string[]} deps.contentTypes content types to search for
+     * @param {string[]} deps.directories directories to search for content files
+     * @param {string} deps.contentPath path to the destination content directory
+     * @param {Object} deps.storage storage adapter instance
+     * @param {object} deps.urlUtils urlUtils instance
+     */
+    constructor(deps) {
+        this.type = deps.type;
+        this.directories = deps.directories;
+        this.extensions = deps.extensions;
+        this.contentTypes = deps.contentTypes;
+        this.ignoreRootFolderFiles = deps.ignoreRootFolderFiles;
+        this.storage = deps.storage;
+        this.#contentPath = deps.contentPath;
+        this.urlUtils = deps.urlUtils;
+    }
+
+    async loadFile(files, baseDir) {
+        const baseDirRegex = baseDir ? new RegExp('^' + baseDir + '/') : new RegExp('');
+
+        const contentFilesFolderRegexes = _.map(this.storage.staticFileURLPrefix.split('/'), function (dir) {
+            return new RegExp('^' + dir + '/');
+        });
+
+        if (this.ignoreRootFolderFiles) {
+            files = _.filter(files, function (file) {
+                return file.name.indexOf('/') !== -1;
+            });
+        }
+
+        // normalize the directory structure
+        const filesContentPath = this.#contentPath;
+        files = _.map(files, function (file) {
+            const noBaseDir = file.name.replace(baseDirRegex, '');
+            let noGhostDirs = noBaseDir;
+
+            _.each(contentFilesFolderRegexes, function (regex) {
+                noGhostDirs = noGhostDirs.replace(regex, '');
+            });
+
+            file.originalPath = noBaseDir;
+            file.name = noGhostDirs;
+            file.targetDir = path.join(filesContentPath, path.dirname(noGhostDirs));
+            return file;
+        });
+
+        const self = this;
+        return Promise.all(files.map(function (contentFile) {
+            return self.storage.getUniqueFileName(contentFile, contentFile.targetDir).then(function (targetFilename) {
+                contentFile.newPath = self.urlUtils.urlJoin(
+                    '/',
+                    self.urlUtils.getSubdir(),
+                    self.storage.staticFileURLPrefix,
+                    path.relative(filesContentPath, targetFilename)
+                );
+
+                return contentFile;
+            });
+        }));
+    }
+}
+
+module.exports = ImporterContentFileHandler;

package/core/server/data/importer/import-manager.js

@@ -11,7 +11,7 @@ const debug = require('@tryghost/debug')('import-manager');
 const logging = require('@tryghost/logging');
 const errors = require('@tryghost/errors');
 const ImageHandler = require('./handlers/image');
-const ImporterContentFileHandler = require('@tryghost/importer-handler-content-files');
+const ImporterContentFileHandler = require('./handlers/ImporterContentFileHandler');
 const RevueHandler = require('./handlers/revue');
 const JSONHandler = require('./handlers/json');
 const MarkdownHandler = require('./handlers/markdown');

package/core/server/data/seeders/DataGenerator.js

@@ -0,0 +1,288 @@
+const path = require('path');
+const fs = require('fs/promises');
+const JsonImporter = require('./utils/JsonImporter');
+const {getProcessRoot} = require('@tryghost/root-utils');
+const topologicalSort = require('./utils/topological-sort');
+const {faker} = require('@faker-js/faker');
+const {faker: americanFaker} = require('@faker-js/faker/locale/en_US');
+const crypto = require('crypto');
+const {Buffer} = require('node:buffer');
+const DatabaseInfo = require('@tryghost/database-info');
+const errors = require('@tryghost/errors');
+const importers = require('./importers').reduce((acc, val) => {
+    acc[val.table] = val;
+    return acc;
+}, {});
+
+class DataGenerator {
+    /**
+     *
+     * @param {object} options
+     * @param {Record<string,number>} [options.quantities] Pass in custom amounts for specific tables
+     * @param {number} [options.seed] If you pass the same seed, the same data will be generated if you used the same options too and if the data generation code remained the same.
+     */
+    constructor({
+        knex,
+        tables,
+        schemaTables,
+        clearDatabase = false,
+        baseDataPack = '',
+        baseUrl,
+        logger,
+        printDependencies,
+        withDefault,
+        seed,
+        quantities = {},
+        useTransaction = true
+    }) {
+        this.knex = knex;
+        this.tableList = tables || [];
+        this.schemaTables = schemaTables;
+        this.willClearData = clearDatabase;
+        this.useBaseDataPack = baseDataPack !== '';
+        this.baseDataPack = baseDataPack;
+        this.baseUrl = baseUrl;
+        this.logger = logger;
+        this.withDefault = withDefault;
+        this.printDependencies = printDependencies;
+        this.seed = seed;
+        this.quantities = quantities;
+        this.useTransaction = useTransaction;
+    }
+
+    sortTableList() {
+        // Add missing dependencies
+        for (const table of this.tableList) {
+            table.importer = importers[table.name];
+
+            // eslint-disable-next-line no-unused-vars
+            table.dependencies = Object.entries(this.schemaTables[table.name]).reduce((acc, [_col, data]) => {
+                if (data.references) {
+                    const referencedTable = data.references.split('.')[0];
+                    // The ghost_subscriptions_id property has a foreign key to the subscriptions table, but we don't use that table yet atm, so don't add it as a dependency
+                    if (!acc.includes(referencedTable) && referencedTable !== 'subscriptions') {
+                        acc.push(referencedTable);
+                    }
+                }
+                return acc;
+            }, table.importer.dependencies);
+
+            for (const dependency of table.dependencies) {
+                if (!this.tableList.find(t => t.name === dependency)) {
+                    this.tableList.push({
+                        name: dependency,
+                        importer: importers[dependency]
+                    });
+                }
+            }
+        }
+
+        // Order to ensure dependencies are created before dependants
+        this.tableList = topologicalSort(this.tableList);
+    }
+
+    /**
+     * TODO: This needs to reverse through all dependency chains to clear data from all tables
+     * @param {import('knex/types').Knex.Transaction} transaction
+     */
+    async clearData(transaction) {
+        const tables = this.tableList.map(t => t.name).reverse();
+
+        // TODO: Remove this once we import posts_meta
+        tables.unshift('posts_meta');
+
+        // Clear data from any tables that are being imported
+        for (const table of tables) {
+            this.logger.debug(`Clearing table ${table}`);
+
+            if (table === 'roles_users') {
+                await transaction(table).del().whereNot('user_id', '1');
+            } else if (table === 'users') {
+                // Avoid deleting the admin user
+                await transaction(table).del().whereNot('id', '1');
+            } else {
+                await transaction(table).truncate();
+            }
+        }
+    }
+
+    async importBasePack(transaction) {
+        let baseDataPack = this.baseDataPack;
+        if (!path.isAbsolute(this.baseDataPack)) {
+            baseDataPack = path.join(getProcessRoot(), baseDataPack);
+        }
+        let baseData = {};
+        try {
+            baseData = JSON.parse((await fs.readFile(baseDataPack)).toString());
+            this.logger.info('Read base data pack');
+        } catch (error) {
+            this.logger.error('Failed to read data pack: ', error);
+            throw error;
+        }
+
+        this.logger.info('Starting base data import');
+        const jsonImporter = new JsonImporter(transaction);
+
+        // Clear settings table
+        await transaction('settings').del();
+
+        // Hard-coded for order
+        const tablesToImport = [
+            'newsletters',
+            'posts',
+            'tags',
+            'products',
+            'benefits',
+            'products_benefits',
+            'stripe_products',
+            'stripe_prices',
+            'settings',
+            'custom_theme_settings'
+        ];
+        for (const table of tablesToImport) {
+            this.logger.info(`Importing content for table ${table} from base data pack`);
+            await jsonImporter.import({
+                name: table,
+                data: baseData[table]
+            });
+            const tableIndex = this.tableList.findIndex(t => t.name === table);
+            if (tableIndex !== -1) {
+                this.tableList.splice(tableIndex, 1);
+            }
+        }
+
+        this.logger.info('Completed base data import');
+    }
+
+    async importData() {
+        const start = Date.now();
+
+        // Add default tables if none are specified
+        if (this.tableList.length === 0) {
+            this.tableList = Object.keys(importers).map(name => ({name}));
+        } else if (this.withDefault) {
+        // Add default tables to the end of the list
+            const defaultTables = Object.keys(importers).map(name => ({name}));
+            for (const table of defaultTables) {
+                if (!this.tableList.find(t => t.name === table.name)) {
+                    this.tableList.push(table);
+                }
+            }
+        }
+
+        // Error if we have an unknown table
+        for (const table of this.tableList) {
+            if (importers[table.name] === undefined) {
+                throw new errors.IncorrectUsageError({message: `Unknown table: ${table.name}`});
+            }
+        }
+
+        this.sortTableList();
+
+        if (this.printDependencies) {
+            this.logger.info('Table dependencies:');
+            for (const table of this.tableList) {
+                this.logger.info(`\t${table.name}: ${table.dependencies.join(', ')}`);
+            }
+            process.exit(0);
+        }
+
+        if (this.useTransaction) {
+            await this.knex.transaction(async (transaction) => {
+                if (!DatabaseInfo.isSQLite(this.knex)) {
+                    await transaction.raw('SET autocommit=0;');
+                }
+
+                await this.#run(transaction);
+            }, {isolationLevel: 'read committed'});
+        } else {
+            await this.#run(this.knex);
+        }
+
+        this.logger.info(`Completed data import in ${((Date.now() - start) / 1000).toFixed(1)}s`);
+    }
+
+    async #run(transaction) {
+        if (!DatabaseInfo.isSQLite(this.knex)) {
+            if (process.env.DISABLE_FAST_IMPORT) {
+                await transaction.raw('SET FOREIGN_KEY_CHECKS=0;');
+                await transaction.raw('SET unique_checks=0;');
+            } else {
+                await transaction.raw('ALTER INSTANCE DISABLE INNODB REDO_LOG;');
+                await transaction.raw('SET FOREIGN_KEY_CHECKS=0;');
+                await transaction.raw('SET unique_checks=0;');
+                await transaction.raw('SET GLOBAL local_infile=1;');
+            }
+        }
+
+        if (this.willClearData) {
+            await this.clearData(transaction);
+        }
+
+        if (this.useBaseDataPack) {
+            await this.importBasePack(transaction);
+        }
+
+        // Set quantities for tables
+        for (const table of this.tableList) {
+            if (this.quantities[table.name] !== undefined) {
+                table.quantity = this.quantities[table.name];
+            }
+        }
+
+        const cryptoRandomBytes = crypto.randomBytes;
+
+        if (this.seed) {
+            // The probality distributions library uses crypto.randomBytes, which we can't seed, so we need to override it
+            crypto.randomBytes = (size) => {
+                const buffer = Buffer.alloc(size);
+                for (let i = 0; i < size; i++) {
+                    buffer[i] = Math.floor(faker.datatype.number({min: 0, max: 255}));
+                }
+                return buffer;
+            };
+        }
+
+        try {
+            for (const table of this.tableList) {
+                if (this.seed) {
+                    // We reset the seed for every table, so the chosen tables don't affect the data and changes in one importer don't affect the others
+                    faker.seed(this.seed);
+                    americanFaker.seed(this.seed);
+                }
+
+                // Add all common options to every importer, whether they use them or not
+                const tableImporter = new table.importer(this.knex, transaction, {
+                    baseUrl: this.baseUrl
+                });
+
+                const amount = table.quantity ?? tableImporter.defaultQuantity;
+                this.logger.info('Importing content for table', table.name, amount ? `(${amount} records)` : '');
+
+                await tableImporter.import(table.quantity ?? undefined);
+            }
+        } finally {
+            if (this.seed) {
+                // Revert crypto.randomBytes to the original function
+                crypto.randomBytes = cryptoRandomBytes;
+            }
+        }
+
+        // Finalise all tables - uses new table importer objects to avoid keeping all data in memory
+        for (const table of this.tableList) {
+            const tableImporter = new table.importer(this.knex, transaction, {
+                baseUrl: this.baseUrl
+            });
+            await tableImporter.finalise();
+        }
+
+        // Re-enable the redo log because it's a persisted global
+        // Leaving it disabled can break the database in the event of an unexpected shutdown
+        // See https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-disable-redo-logging
+        if (!DatabaseInfo.isSQLite(this.knex) && !process.env.DISABLE_FAST_IMPORT) {
+            await transaction.raw('ALTER INSTANCE ENABLE INNODB REDO_LOG;');
+        }
+    }
+}
+
+module.exports = DataGenerator;

package/core/server/data/seeders/importers/BenefitsImporter.js

@@ -0,0 +1,28 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const {slugify} = require('@tryghost/string');
+const {blogStartDate} = require('../utils/blog-info');
+
+class BenefitsImporter extends TableImporter {
+    static table = 'benefits';
+    static dependencies = [];
+    defaultQuantity = 5;
+
+    constructor(knex, transaction) {
+        super(BenefitsImporter.table, knex, transaction);
+    }
+
+    generate() {
+        const name = faker.company.catchPhrase();
+        const sixMonthsLater = new Date(blogStartDate);
+        sixMonthsLater.setMonth(sixMonthsLater.getMonth() + 6);
+        return {
+            id: this.fastFakeObjectId(),
+            name: name,
+            slug: `${slugify(name)}-${faker.random.numeric(3)}`,
+            created_at: faker.date.between(blogStartDate, sixMonthsLater)
+        };
+    }
+}
+
+module.exports = BenefitsImporter;

package/core/server/data/seeders/importers/CommentsImporter.js

@@ -0,0 +1,73 @@
+const {faker} = require('@faker-js/faker');
+const TableImporter = require('./TableImporter');
+const {luck} = require('../utils/random');
+const generateEvents = require('../utils/event-generator');
+const dateToDatabaseString = require('../utils/database-date');
+
+class CommentsImporter extends TableImporter {
+    static table = 'comments';
+    static dependencies = ['posts'];
+
+    constructor(knex, transaction) {
+        super(CommentsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const posts = await this.transaction.select('id', 'published_at').from('posts')
+            .where('status', 'published');
+        this.members = await this.transaction.select('id', 'created_at').from('members');
+
+        this.commentsPerPost = quantity ? quantity / posts.length : 10;
+
+        await this.importForEach(posts, this.commentsPerPost);
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
+
+        this.commentIds = [];
+
+        this.timestamps = generateEvents({
+            shape: 'ease-out',
+            trend: 'negative',
+            // Steady readers login more, readers who lose interest read less overall.
+            // ceil because members will all have logged in at least once
+            total: faker.datatype.number({min: 0, max: this.commentsPerPost}),
+            startTime: new Date(model.published_at),
+            endTime: new Date()
+        });
+
+        this.possibleMembers = this.members.filter(member => new Date(member.created_at) < new Date(model.published_at));
+    }
+
+    generate() {
+        const timestamp = this.timestamps.pop();
+        if (!timestamp) {
+            // Out of events for this post
+            return null;
+        }
+
+        if (this.possibleMembers.length === 0) {
+            return null;
+        }
+
+        const isReply = luck(30) && this.commentIds.length > 0; // 30% of comments are replies
+
+        const event = {
+            id: this.fastFakeObjectId(),
+            post_id: this.model.id,
+            member_id: this.possibleMembers[faker.datatype.number(this.possibleMembers.length - 1)].id,
+            parent_id: isReply ? this.commentIds[faker.datatype.number(this.commentIds.length - 1)] : undefined,
+            status: 'published',
+            created_at: dateToDatabaseString(timestamp),
+            updated_at: dateToDatabaseString(timestamp),
+            html: `<p>${faker.lorem.sentence().replace(/[&<>"']/g, c => `&#${c.charCodeAt(0)};`)}</p>`
+        };
+
+        this.commentIds.push(event.id);
+
+        return event;
+    }
+}
+
+module.exports = CommentsImporter;

package/core/server/data/seeders/importers/EmailBatchesImporter.js

@@ -0,0 +1,38 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const dateToDatabaseString = require('../utils/database-date');
+
+class EmailBatchesImporter extends TableImporter {
+    static table = 'email_batches';
+    static dependencies = ['emails'];
+
+    constructor(knex, transaction) {
+        super(EmailBatchesImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const emails = await this.transaction.select('id', 'created_at', 'email_count').from('emails');
+
+        // 1 batch per 1000 recipients
+        await this.importForEach(emails, quantity ?? (() => {
+            return Math.ceil(this.model.email_count / 1000);
+        }));
+    }
+
+    generate() {
+        const emailSentDate = new Date(this.model.created_at);
+        const latestUpdatedDate = new Date(this.model.created_at);
+        latestUpdatedDate.setHours(latestUpdatedDate.getHours() + 1);
+
+        return {
+            id: this.fastFakeObjectId(),
+            email_id: this.model.id,
+            provider_id: `${new Date().toISOString().split('.')[0].replace(/[^0-9]/g, '')}.${faker.datatype.hexadecimal({length: 16, prefix: '', case: 'lower'})}@m.example.com`,
+            status: 'submitted', // TODO: introduce failures
+            created_at: this.model.created_at,
+            updated_at: dateToDatabaseString(faker.date.between(emailSentDate, latestUpdatedDate))
+        };
+    }
+}
+
+module.exports = EmailBatchesImporter;

package/core/server/data/seeders/importers/EmailRecipientFailuresImporter.js

@@ -0,0 +1,67 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+
+class EmailRecipientFailuresImporter extends TableImporter {
+    static table = 'email_recipient_failures';
+    static dependencies = ['email_recipients'];
+
+    constructor(knex, transaction) {
+        super(EmailRecipientFailuresImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const recipients = await this.transaction
+            .select(
+                'id',
+                'email_id',
+                'member_id',
+                'failed_at')
+            .from('email_recipients')
+            .whereNotNull('failed_at');
+
+        await this.importForEach(recipients, quantity ? quantity / recipients.length : 1);
+    }
+
+    generate() {
+        const errors = [
+            {
+                severity: 'permanent',
+                code: 605,
+                enhanced_code: null,
+                message: 'Not delivering to previously bounced address'
+            },
+            {
+                severity: 'permanent',
+                code: 451,
+                enhanced_code: '4.7.652',
+                message: '4.7.652 The mail server [xxx.xxx.xxx.xxx] has exceeded the maximum number of connections.'
+            },
+            {
+                message: 'No MX for example.com',
+                code: 498,
+                enhanced_code: null,
+                severity: 'permanent'
+            },
+            {
+                severity: 'temporary',
+                code: 552,
+                enhanced_code: null,
+                message: '5.2.2 <xxxxxxxx@example.com>: user is over quota'
+            }
+        ];
+
+        const error = faker.helpers.arrayElement(errors);
+
+        return {
+            id: this.fastFakeObjectId(),
+            email_id: this.model.email_id,
+            member_id: this.model.member_id,
+            email_recipient_id: this.model.id,
+            event_id: faker.random.alphaNumeric(20),
+            ...error,
+            failed_at: this.model.failed_at
+        };
+    }
+}
+
+module.exports = EmailRecipientFailuresImporter;