ghost 5.115.1 → 5.116.1

package/core/server/data/seeders/importers/RecommendationSubscribeEventsImporter.js

@@ -0,0 +1,32 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const {luck} = require('../utils/random');
+
+class RecommendationSubscribeEventsImporter extends TableImporter {
+    static table = 'recommendation_subscribe_events';
+    static dependencies = ['recommendations', 'members'];
+
+    constructor(knex, transaction) {
+        super(RecommendationSubscribeEventsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const recommendations = await this.transaction.select('id', 'created_at').from('recommendations').where('one_click_subscribe', true);
+        this.members = await this.transaction.select('id').from('members').limit(500);
+
+        await this.importForEach(recommendations, quantity ? quantity / recommendations.length : () => faker.datatype.number({min: 0, max: 50}));
+    }
+
+    generate() {
+        // Not unique
+        const member = luck(1) ? null : faker.helpers.arrayElement(this.members);
+        return {
+            id: this.fastFakeObjectId(),
+            recommendation_id: this.model.id,
+            member_id: member?.id ?? null,
+            created_at: faker.date.past()
+        };
+    }
+}
+
+module.exports = RecommendationSubscribeEventsImporter;

package/core/server/data/seeders/importers/RecommendationsImporter.js

@@ -0,0 +1,34 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+
+function capitalize(str) {
+    return str.charAt(0).toUpperCase() + str.slice(1);
+}
+
+class RecommendationsImporter extends TableImporter {
+    static table = 'recommendations';
+    static dependencies = [];
+    defaultQuantity = 15;
+
+    constructor(knex, transaction) {
+        super(RecommendationsImporter.table, knex, transaction);
+    }
+
+    generate() {
+        const id = this.fastFakeObjectId();
+        return {
+            id,
+            url: faker.internet.url(),
+            title: capitalize(`${faker.word.adjective()} ${faker.word.noun()}`),
+            excerpt: faker.lorem.sentence(),
+            featured_image: `https://api.dicebear.com/5.x/shapes/png?size=256&seed=${id}`,
+            favicon: `https://api.dicebear.com/5.x/shapes/png?size=32&seed=${id}`,
+            description: faker.lorem.sentence(),
+            one_click_subscribe: faker.datatype.boolean(),
+            created_at: faker.date.past(),
+            updated_at: faker.date.past()
+        };
+    }
+}
+
+module.exports = RecommendationsImporter;

package/core/server/data/seeders/importers/RedirectsImporter.js

@@ -0,0 +1,49 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+
+class RedirectsImporter extends TableImporter {
+    static table = 'redirects';
+    static dependencies = ['posts'];
+
+    constructor(knex, transaction) {
+        super(RedirectsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const posts = await this.transaction
+            .select('id', 'published_at')
+            .from('posts')
+            .where('type', 'post')
+            .andWhere('status', 'published');
+
+        this.quantity = quantity ? quantity / posts.length : 10;
+        await this.importForEach(posts, this.quantity);
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
+
+        // Reset the amount for each model
+        this.amount = faker.datatype.number({
+            min: 0,
+            max: this.quantity
+        });
+    }
+
+    generate() {
+        if (this.amount <= 0) {
+            return;
+        }
+        this.amount -= 1;
+        return {
+            id: this.fastFakeObjectId(),
+            from: `/r/${faker.datatype.hexadecimal({length: 8, prefix: '', case: 'lower'})}`,
+            to: `${faker.internet.url()}/${faker.helpers.slugify(`${faker.word.adjective()} ${faker.word.noun()}`).toLowerCase()}`,
+            post_id: this.model.id,
+            created_at: this.model.published_at,
+            updated_at: this.model.published_at
+        };
+    }
+}
+
+module.exports = RedirectsImporter;

package/core/server/data/seeders/importers/RolesUsersImporter.js

@@ -0,0 +1,42 @@
+const {faker} = require('@faker-js/faker');
+const TableImporter = require('./TableImporter');
+
+class RolesUsersImporter extends TableImporter {
+    static table = 'roles_users';
+    // No roles imorter, since roles are statically defined in database
+    static dependencies = ['users'];
+
+    constructor(knex, transaction) {
+        super(RolesUsersImporter.table, knex, transaction);
+    }
+
+    /**
+     * Ignore overriden quantity for 1:1 relationship
+     */
+    async import() {
+        const users = await this.transaction.select('id').from('users').whereNot('id', 1);
+        this.roles = await this.transaction.select('id', 'name').from('roles');
+
+        await this.importForEach(users, 1);
+    }
+
+    generate() {
+        const userRoles = ['Editor', 'Contributor', 'Author'];
+        const userRole = userRoles[faker.datatype.number({
+            min: 0,
+            max: userRoles.length - 1
+        })];
+        const actualRole = this.roles.find(role => role.name === userRole);
+        if (!actualRole) {
+            // No roles defined in database, don't bother creating user role
+            return;
+        }
+        return {
+            id: this.fastFakeObjectId(),
+            role_id: actualRole.id,
+            user_id: this.model.id
+        };
+    }
+}
+
+module.exports = RolesUsersImporter;

package/core/server/data/seeders/importers/StripePricesImporter.js

@@ -0,0 +1,69 @@
+const {faker} = require('@faker-js/faker');
+const TableImporter = require('./TableImporter');
+const {blogStartDate} = require('../utils/blog-info');
+
+const sixWeeksLater = new Date(blogStartDate);
+sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6));
+
+class StripePricesImporter extends TableImporter {
+    static table = 'stripe_prices';
+    static dependencies = ['products', 'stripe_products'];
+
+    constructor(knex, transaction) {
+        super(StripePricesImporter.table, knex, transaction);
+    }
+
+    async import() {
+        const stripeProducts = await this.transaction.select('id', 'stripe_product_id', 'product_id').from('stripe_products');
+        this.products = await this.transaction.select('id', 'monthly_price', 'yearly_price').from('products');
+
+        await this.importForEach(stripeProducts, 2);
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
+        this.count = 0;
+    }
+
+    generate() {
+        const count = this.count;
+        this.count = this.count + 1;
+
+        const relatedProduct = this.products.find(product => product.id === this.model.product_id);
+
+        if (count === 1 && relatedProduct.monthly_price === null) {
+            // Only single complimentary price (yearly)
+            return null;
+        }
+
+        const billingCycle = {
+            nickname: 'Monthly',
+            interval: 'month',
+            type: 'recurring',
+            currency: 'usd',
+            amount: relatedProduct.monthly_price
+        };
+        if (count === 1) {
+            billingCycle.nickname = 'Yearly';
+            billingCycle.interval = 'year';
+            billingCycle.amount = relatedProduct.yearly_price;
+        } else if (relatedProduct.monthly_price === null) {
+            billingCycle.nickname = 'Complimentary';
+            billingCycle.interval = 'year';
+            billingCycle.amount = 0;
+        }
+
+        return Object.assign({}, {
+            id: this.fastFakeObjectId(),
+            stripe_price_id: faker.datatype.hexadecimal({
+                length: 64,
+                prefix: ''
+            }),
+            stripe_product_id: this.model.stripe_product_id,
+            active: true,
+            created_at: faker.date.between(blogStartDate, sixWeeksLater)
+        }, billingCycle);
+    }
+}
+
+module.exports = StripePricesImporter;

package/core/server/data/seeders/importers/StripeProductsImporter.js

@@ -0,0 +1,34 @@
+const {faker} = require('@faker-js/faker');
+const TableImporter = require('./TableImporter');
+const {blogStartDate} = require('../utils/blog-info');
+
+const sixWeeksLater = new Date(blogStartDate);
+sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6));
+
+class StripeProductsImporter extends TableImporter {
+    static table = 'stripe_products';
+    static dependencies = ['products'];
+
+    constructor(knex, transaction) {
+        super(StripeProductsImporter.table, knex, transaction);
+    }
+
+    async import() {
+        const products = await this.transaction.select('id').from('products');
+        await this.importForEach(products, 1);
+    }
+
+    generate() {
+        return {
+            id: this.fastFakeObjectId(),
+            product_id: this.model.id,
+            stripe_product_id: faker.datatype.hexadecimal({
+                length: 64,
+                prefix: ''
+            }),
+            created_at: faker.date.between(blogStartDate, sixWeeksLater)
+        };
+    }
+}
+
+module.exports = StripeProductsImporter;

package/core/server/data/seeders/importers/TableImporter.js

@@ -0,0 +1,187 @@
+const debug = require('@tryghost/debug')('TableImporter');
+const dateToDatabaseString = require('../utils/database-date');
+const path = require('path');
+const createCsvWriter = require('csv-writer').createObjectCsvWriter;
+const fs = require('fs');
+const {luck} = require('../utils/random');
+const os = require('os');
+const crypto = require('crypto');
+const logging = require('@tryghost/logging');
+const errors = require('@tryghost/errors');
+
+class TableImporter {
+    /**
+     * @type {object|undefined} model Referenced model when generating data
+     */
+    model;
+
+    /**
+     * @type {number|undefined} defaultQuantity Default number of records to import
+     */
+    defaultQuantity;
+
+    /**
+     * Transaction and knex need to be separate since we're using the batchInsert helper
+     * @param {string} name Name of the table to be generated
+     * @param {import('knex/types').Knex} knex Database connection
+     * @param {import('knex/types').Knex.Transaction} transaction Transaction to be used for import
+     */
+    constructor(name, knex, transaction) {
+        this.name = name;
+        this.knex = knex;
+        this.transaction = transaction;
+    }
+
+    fastFakeObjectId() {
+        // It is important that IDs are generated for a timestamp < NOW (for email batch sending) and that
+        // generating the ids is fast.
+        return `00000000` + crypto.randomBytes(8).toString('hex');
+    }
+
+    async #generateData(amount = this.defaultQuantity) {
+        let data = [];
+
+        for (let i = 0; i < amount; i++) {
+            const model = await this.generate();
+            if (model) {
+                data.push(model);
+            }
+        }
+
+        return data;
+    }
+
+    async import(amount = this.defaultQuantity) {
+        const generateNow = Date.now();
+        const data = await this.#generateData(amount);
+        debug(`${this.name} generated ${data.length} records in ${Date.now() - generateNow}ms`);
+
+        if (data.length > 0) {
+            await this.batchInsert(data);
+        }
+    }
+
+    /**
+     * @param {Array<Object>} models List of models to reference
+     * @param {Number|function} amount Number of records to import per model
+     */
+    async importForEach(models = [], amount) {
+        const data = [];
+
+        debug (`Generating data for ${models.length} models x ${amount} for ${this.name}`);
+        const now = Date.now();
+        let settingReferenceModel = 0;
+
+        for (const model of models) {
+            let s = Date.now();
+            this.setReferencedModel(model);
+            settingReferenceModel += Date.now() - s;
+
+            let currentAmount = (typeof amount === 'function') ? amount() : amount;
+            if (!Number.isInteger(currentAmount)) {
+                currentAmount = Math.floor(currentAmount) + luck((currentAmount % 1) * 100);
+            }
+
+            const generatedData = await this.#generateData(currentAmount);
+            if (generatedData.length > 0) {
+                data.push(...generatedData);
+            }
+        }
+
+        debug(`${this.name} generated ${data.length} records in ${Date.now() - now}ms (${settingReferenceModel}ms setting reference model)`);
+
+        if (data.length > 0) {
+            await this.batchInsert(data);
+        }
+    }
+
+    async batchInsert(data) {
+        // Write to CSV file
+        const rootFolder = os.tmpdir();
+        const filePath = path.join(rootFolder, `${this.name}.csv`);
+        let now = Date.now();
+
+        if (data.length > 5000 && !process.env.DISABLE_FAST_IMPORT) {
+            try {
+                await fs.promises.unlink(filePath);
+            } catch (e) {
+                // Ignore: file doesn't exist
+            }
+
+            const csvWriter = createCsvWriter({
+                path: filePath,
+                header: Object.keys(data[0]).map((key) => {
+                    return {id: key, title: key};
+                })
+            });
+
+            // Loop the data in chunks of 50.000 items
+            const batchSize = 50000;
+
+            // Otherwise we get a out of range error because csvWriter tries to create a string that is too long
+            for (let i = 0; i < data.length; i += batchSize) {
+                const slicedData = data.slice(i, i + batchSize);
+
+                // Map data to what MySQL expects in the CSV for values like booleans, null and dates
+                for (let j = 0; j < slicedData.length; j++) {
+                    const obj = slicedData[j];
+
+                    for (const [key, value] of Object.entries(obj)) {
+                        if (typeof value === 'boolean') {
+                            obj[key] = value ? 1 : 0;
+                        } else if (value instanceof Date) {
+                            obj[key] = dateToDatabaseString(value);
+                        } else if (value === null) {
+                            obj[key] = '\\N';
+                        }
+                    }
+                }
+                await csvWriter.writeRecords(slicedData);
+            }
+
+            debug(`${this.name} saved CSV import file in ${Date.now() - now}ms`);
+            now = Date.now();
+
+            // Import from CSV file
+            const [result] = await this.transaction.raw(`LOAD DATA LOCAL INFILE '${filePath}' INTO TABLE \`${this.name}\` FIELDS TERMINATED BY ',' ENCLOSED BY '"' IGNORE 1 LINES (${Object.keys(data[0]).map(d => '`' + d + '`').join(',')});`);
+            if (result.affectedRows !== data.length) {
+                if (Math.abs(result.affectedRows - data.length) > 0.01 * data.length) {
+                    throw new errors.InternalServerError({
+                        message: `CSV import failed: expected ${data.length} imported rows, got ${result.affectedRows}`
+                    });
+                }
+                logging.warn(`CSV import warning: expected ${data.length} imported rows, got ${result.affectedRows}.`);
+            }
+        } else {
+            await this.knex.batchInsert(this.name, data).transacting(this.transaction);
+        }
+
+        debug(`${this.name} imported ${data.length} records in ${Date.now() - now}ms`);
+    }
+
+    /**
+     * Finalise the imported data, e.g. adding summary records based on a table's dependents
+     */
+    async finalise() {
+        // No-op by default
+    }
+
+    /**
+     * Sets the model which newly generated data will reference
+     * @param {Object} model Model to reference when generating data
+     */
+    setReferencedModel(model) {
+        this.model = model;
+    }
+
+    /**
+     * Generates the data for a single model to be imported
+     * @returns {Object|null} Data to import, optional
+     */
+    generate() {
+        // Should never be called
+        return false;
+    }
+}
+
+module.exports = TableImporter;

package/core/server/data/seeders/importers/TagsImporter.js

@@ -0,0 +1,41 @@
+const {faker} = require('@faker-js/faker');
+const {slugify} = require('@tryghost/string');
+const TableImporter = require('./TableImporter');
+const dateToDatabaseString = require('../utils/database-date');
+
+class TagsImporter extends TableImporter {
+    static table = 'tags';
+    static dependencies = ['users'];
+    defaultQuantity = faker.datatype.number({
+        min: 16,
+        max: 24
+    });
+
+    constructor(knex, transaction) {
+        super(TagsImporter.table, knex, transaction);
+    }
+
+    async import(quantity = this.defaultQuantity) {
+        this.users = await this.transaction.select('id').from('users');
+        await super.import(quantity);
+    }
+
+    generate() {
+        let name = `${faker.color.human()} ${faker.name.jobType()}`;
+        name = `${name[0].toUpperCase()}${name.slice(1)}`;
+        const threeYearsAgo = new Date();
+        threeYearsAgo.setFullYear(threeYearsAgo.getFullYear() - 3);
+        const twoYearsAgo = new Date();
+        twoYearsAgo.setFullYear(twoYearsAgo.getFullYear() - 2);
+        return {
+            id: this.fastFakeObjectId(),
+            name: name,
+            slug: `${slugify(name)}-${faker.random.numeric(3)}`,
+            description: faker.lorem.sentence(),
+            created_at: dateToDatabaseString(faker.date.between(threeYearsAgo, twoYearsAgo)),
+            created_by: this.users[faker.datatype.number(this.users.length - 1)].id
+        };
+    }
+}
+
+module.exports = TagsImporter;

package/core/server/data/seeders/importers/UsersImporter.js

@@ -0,0 +1,31 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const {slugify} = require('@tryghost/string');
+const security = require('@tryghost/security');
+const dateToDatabaseString = require('../utils/database-date');
+
+class UsersImporter extends TableImporter {
+    static table = 'users';
+    static dependencies = [];
+    defaultQuantity = 8;
+
+    constructor(knex, transaction) {
+        super(UsersImporter.table, knex, transaction);
+    }
+
+    async generate() {
+        const name = `${faker.name.firstName()} ${faker.name.lastName()}`;
+        return {
+            id: this.fastFakeObjectId(),
+            name: name,
+            slug: slugify(name),
+            password: await security.password.hash(faker.color.human()),
+            email: faker.internet.email(name),
+            profile_image: faker.internet.avatar(),
+            created_at: dateToDatabaseString(faker.date.between(new Date(2016, 0), new Date())),
+            created_by: 'unused'
+        };
+    }
+}
+
+module.exports = UsersImporter;

package/core/server/data/seeders/importers/WebMentionsImporter.js

@@ -0,0 +1,42 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const dateToDatabaseString = require('../utils/database-date');
+
+class WebMentionsImporter extends TableImporter {
+    static table = 'mentions';
+    static dependencies = [];
+    defaultQuantity = 23;
+
+    constructor(knex, transaction, {baseUrl}) {
+        super(WebMentionsImporter.table, knex, transaction);
+
+        this.baseUrl = baseUrl;
+    }
+
+    generate() {
+        const id = this.fastFakeObjectId();
+
+        const author = `${faker.name.fullName()}`;
+
+        // Generating only incoming recommendations for now, since we don't use webmentions for other things atm
+        return {
+            id,
+            source: `${faker.internet.url()}/.well-known/recommendations.json`,
+            source_title: faker.lorem.sentence(5),
+            source_site_title: `${author}'s ${faker.word.noun()}`,
+            source_excerpt: faker.lorem.paragraph(),
+            source_author: author,
+            source_featured_image: `https://api.dicebear.com/5.x/shapes/png?size=256&seed=${id}`,
+            source_favicon: `https://api.dicebear.com/5.x/bottts/png?size=32&seed=${id}`,
+            target: `${this.baseUrl}`,
+            resource_id: null,
+            resource_type: null,
+            created_at: dateToDatabaseString(faker.date.past()),
+            payload: JSON.stringify({}),
+            deleted: 0,
+            verified: 1
+        };
+    }
+}
+
+module.exports = WebMentionsImporter;

package/core/server/data/seeders/importers/index.js

@@ -0,0 +1,41 @@
+module.exports = [
+    require('./NewslettersImporter'),
+    require('./PostsImporter'),
+    require('./UsersImporter'),
+    require('./TagsImporter'),
+    require('./ProductsImporter'),
+    require('./MembersImporter'),
+    require('./BenefitsImporter'),
+    require('./WebMentionsImporter'),
+    require('./PostsAuthorsImporter'),
+    require('./PostsTagsImporter'),
+    require('./ProductsBenefitsImporter'),
+    require('./MembersProductsImporter'),
+    require('./PostsProductsImporter'),
+    require('./MembersNewslettersImporter'),
+    require('./StripeProductsImporter'),
+    require('./StripePricesImporter'),
+    require('./EmailsImporter'),
+    require('./EmailBatchesImporter'),
+    require('./EmailRecipientsImporter'),
+    require('./EmailRecipientFailuresImporter'),
+    require('./RedirectsImporter'),
+    require('./MembersClickEventsImporter'),
+    require('./OffersImporter'),
+    require('./MembersCreatedEventsImporter'),
+    require('./MembersLoginEventsImporter'),
+    require('./MembersStatusEventsImporter'),
+    require('./MembersStripeCustomersImporter'),
+    require('./MembersStripeCustomersSubscriptionsImporter'),
+    require('./MembersPaidSubscriptionEventsImporter'),
+    require('./MembersSubscriptionCreatedEventsImporter'),
+    require('./MembersSubscribeEventsImporter'),
+    require('./LabelsImporter'),
+    require('./MembersLabelsImporter'),
+    require('./RolesUsersImporter'),
+    require('./MembersFeedbackImporter'),
+    require('./RecommendationsImporter'),
+    require('./RecommendationClickEventsImporter'),
+    require('./RecommendationSubscribeEventsImporter'),
+    require('./CommentsImporter')
+];

package/core/server/data/seeders/utils/JsonImporter.js

@@ -0,0 +1,39 @@
+const TableImporter = require('../importers/TableImporter');
+
+class JsonImporter extends TableImporter {
+    constructor(knex, transaction) {
+        super();
+        this.knex = knex;
+        this.transaction = transaction;
+    }
+
+    /**
+     * @typedef {Object} JsonImportOptions
+     * @property {string} name Name of the table to import
+     * @property {Object} data Models without ids to be imported
+     * @property {Array<string>} [rows] Set of rows to be returned
+     */
+
+    /**
+     * Import a dataset to the database
+     * @param {JsonImportOptions} options
+     * @returns {Promise}
+     */
+    async import({
+        name,
+        data,
+        rows = []
+    }) {
+        for (const obj of data) {
+            if (!('id' in obj)) {
+                obj.id = this.fastFakeObjectId();
+            }
+        }
+        if (rows.findIndex(row => row === 'id') === -1) {
+            rows.unshift('id');
+        }
+        await this.knex.batchInsert(name, data, 500).transacting(this.transaction);
+    }
+}
+
+module.exports = JsonImporter;

package/core/server/data/seeders/utils/blog-info.js

@@ -0,0 +1,3 @@
+module.exports = {
+    blogStartDate: new Date(2018, 5, 4)
+};

package/core/server/data/seeders/utils/database-date.js

@@ -0,0 +1,7 @@
+module.exports = function dateToDatabaseString(date) {
+    if (typeof date === 'string') {
+        // SQLite fix when reusing other dates from the db
+        return date;
+    }
+    return date.toISOString().replace('Z','').replace('T', ' ');
+};