ghost 5.115.0 → 5.116.0

package/core/server/data/seeders/importers/BenefitsImporter.js

@@ -0,0 +1,28 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const {slugify} = require('@tryghost/string');
+const {blogStartDate} = require('../utils/blog-info');
+
+class BenefitsImporter extends TableImporter {
+    static table = 'benefits';
+    static dependencies = [];
+    defaultQuantity = 5;
+
+    constructor(knex, transaction) {
+        super(BenefitsImporter.table, knex, transaction);
+    }
+
+    generate() {
+        const name = faker.company.catchPhrase();
+        const sixMonthsLater = new Date(blogStartDate);
+        sixMonthsLater.setMonth(sixMonthsLater.getMonth() + 6);
+        return {
+            id: this.fastFakeObjectId(),
+            name: name,
+            slug: `${slugify(name)}-${faker.random.numeric(3)}`,
+            created_at: faker.date.between(blogStartDate, sixMonthsLater)
+        };
+    }
+}
+
+module.exports = BenefitsImporter;

package/core/server/data/seeders/importers/CommentsImporter.js

@@ -0,0 +1,73 @@
+const {faker} = require('@faker-js/faker');
+const TableImporter = require('./TableImporter');
+const {luck} = require('../utils/random');
+const generateEvents = require('../utils/event-generator');
+const dateToDatabaseString = require('../utils/database-date');
+
+class CommentsImporter extends TableImporter {
+    static table = 'comments';
+    static dependencies = ['posts'];
+
+    constructor(knex, transaction) {
+        super(CommentsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const posts = await this.transaction.select('id', 'published_at').from('posts')
+            .where('status', 'published');
+        this.members = await this.transaction.select('id', 'created_at').from('members');
+
+        this.commentsPerPost = quantity ? quantity / posts.length : 10;
+
+        await this.importForEach(posts, this.commentsPerPost);
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
+
+        this.commentIds = [];
+
+        this.timestamps = generateEvents({
+            shape: 'ease-out',
+            trend: 'negative',
+            // Steady readers login more, readers who lose interest read less overall.
+            // ceil because members will all have logged in at least once
+            total: faker.datatype.number({min: 0, max: this.commentsPerPost}),
+            startTime: new Date(model.published_at),
+            endTime: new Date()
+        });
+
+        this.possibleMembers = this.members.filter(member => new Date(member.created_at) < new Date(model.published_at));
+    }
+
+    generate() {
+        const timestamp = this.timestamps.pop();
+        if (!timestamp) {
+            // Out of events for this post
+            return null;
+        }
+
+        if (this.possibleMembers.length === 0) {
+            return null;
+        }
+
+        const isReply = luck(30) && this.commentIds.length > 0; // 30% of comments are replies
+
+        const event = {
+            id: this.fastFakeObjectId(),
+            post_id: this.model.id,
+            member_id: this.possibleMembers[faker.datatype.number(this.possibleMembers.length - 1)].id,
+            parent_id: isReply ? this.commentIds[faker.datatype.number(this.commentIds.length - 1)] : undefined,
+            status: 'published',
+            created_at: dateToDatabaseString(timestamp),
+            updated_at: dateToDatabaseString(timestamp),
+            html: `<p>${faker.lorem.sentence().replace(/[&<>"']/g, c => `&#${c.charCodeAt(0)};`)}</p>`
+        };
+
+        this.commentIds.push(event.id);
+
+        return event;
+    }
+}
+
+module.exports = CommentsImporter;

package/core/server/data/seeders/importers/EmailBatchesImporter.js

@@ -0,0 +1,38 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const dateToDatabaseString = require('../utils/database-date');
+
+class EmailBatchesImporter extends TableImporter {
+    static table = 'email_batches';
+    static dependencies = ['emails'];
+
+    constructor(knex, transaction) {
+        super(EmailBatchesImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const emails = await this.transaction.select('id', 'created_at', 'email_count').from('emails');
+
+        // 1 batch per 1000 recipients
+        await this.importForEach(emails, quantity ?? (() => {
+            return Math.ceil(this.model.email_count / 1000);
+        }));
+    }
+
+    generate() {
+        const emailSentDate = new Date(this.model.created_at);
+        const latestUpdatedDate = new Date(this.model.created_at);
+        latestUpdatedDate.setHours(latestUpdatedDate.getHours() + 1);
+
+        return {
+            id: this.fastFakeObjectId(),
+            email_id: this.model.id,
+            provider_id: `${new Date().toISOString().split('.')[0].replace(/[^0-9]/g, '')}.${faker.datatype.hexadecimal({length: 16, prefix: '', case: 'lower'})}@m.example.com`,
+            status: 'submitted', // TODO: introduce failures
+            created_at: this.model.created_at,
+            updated_at: dateToDatabaseString(faker.date.between(emailSentDate, latestUpdatedDate))
+        };
+    }
+}
+
+module.exports = EmailBatchesImporter;

package/core/server/data/seeders/importers/EmailRecipientFailuresImporter.js

@@ -0,0 +1,67 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+
+class EmailRecipientFailuresImporter extends TableImporter {
+    static table = 'email_recipient_failures';
+    static dependencies = ['email_recipients'];
+
+    constructor(knex, transaction) {
+        super(EmailRecipientFailuresImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const recipients = await this.transaction
+            .select(
+                'id',
+                'email_id',
+                'member_id',
+                'failed_at')
+            .from('email_recipients')
+            .whereNotNull('failed_at');
+
+        await this.importForEach(recipients, quantity ? quantity / recipients.length : 1);
+    }
+
+    generate() {
+        const errors = [
+            {
+                severity: 'permanent',
+                code: 605,
+                enhanced_code: null,
+                message: 'Not delivering to previously bounced address'
+            },
+            {
+                severity: 'permanent',
+                code: 451,
+                enhanced_code: '4.7.652',
+                message: '4.7.652 The mail server [xxx.xxx.xxx.xxx] has exceeded the maximum number of connections.'
+            },
+            {
+                message: 'No MX for example.com',
+                code: 498,
+                enhanced_code: null,
+                severity: 'permanent'
+            },
+            {
+                severity: 'temporary',
+                code: 552,
+                enhanced_code: null,
+                message: '5.2.2 <xxxxxxxx@example.com>: user is over quota'
+            }
+        ];
+
+        const error = faker.helpers.arrayElement(errors);
+
+        return {
+            id: this.fastFakeObjectId(),
+            email_id: this.model.email_id,
+            member_id: this.model.member_id,
+            email_recipient_id: this.model.id,
+            event_id: faker.random.alphaNumeric(20),
+            ...error,
+            failed_at: this.model.failed_at
+        };
+    }
+}
+
+module.exports = EmailRecipientFailuresImporter;

package/core/server/data/seeders/importers/EmailRecipientsImporter.js

@@ -0,0 +1,212 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const generateEvents = require('../utils/event-generator');
+const dateToDatabaseString = require('../utils/database-date');
+const debug = require('@tryghost/debug')('EmailRecipientsImporter');
+
+const emailStatus = {
+    delivered: Symbol(),
+    opened: Symbol(),
+    failed: Symbol(),
+    none: Symbol()
+};
+
+function findFirstHigherIndex(arr, target) {
+    let start = 0;
+    let end = arr.length - 1;
+    let result = -1;
+
+    while (start <= end) {
+        let mid = Math.floor((start + end) / 2);
+
+        if (arr[mid] >= target) {
+            result = mid;
+            end = mid - 1; // Continue searching in the left half
+        } else {
+            start = mid + 1; // Continue searching in the right half
+        }
+    }
+
+    return result; // Return -1 if no element is higher than target
+}
+
+class EmailRecipientsImporter extends TableImporter {
+    static table = 'email_recipients';
+    static dependencies = ['emails', 'email_batches', 'members', 'members_subscribe_events'];
+
+    constructor(knex, transaction) {
+        super(EmailRecipientsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        if (quantity === 0) {
+            return;
+        }
+
+        const now = Date.now();
+        const emails = await this.transaction
+            .select(
+                'id',
+                'newsletter_id',
+                'email_count',
+                'delivered_count',
+                'opened_count',
+                'failed_count')
+            .from('emails');
+        this.emails = new Map();
+
+        for (const email of emails) {
+            this.emails.set(email.id, email);
+        }
+
+        this.emailBatches = await this.transaction.select('id', 'email_id', 'updated_at').from('email_batches').orderBy('email_id');
+        const members = await this.transaction.select('id', 'uuid', 'email', 'name').from('members');
+        this.membersSubscribeEvents = await this.transaction.select('id', 'newsletter_id', 'created_at', 'member_id').from('members_subscribe_events').orderBy('created_at', 'asc'); // Order required for better performance in setReferencedModel
+
+        // Create a map for fast lookups
+        this.members = new Map();
+        for (const member of members) {
+            this.members.set(member.id, member);
+        }
+
+        // Save indexes of each batch for performance (remarkable faster than doing findIndex on each generate call)
+        let lastEmailId = null;
+        let lastIndex = 0;
+        for (const batch of this.emailBatches) {
+            if (batch.email_id !== lastEmailId) {
+                lastIndex = 0;
+                lastEmailId = batch.email_id;
+            }
+            batch.index = lastIndex;
+            lastIndex += 1;
+        }
+
+        // Now reorder by email id
+
+        debug (`Prepared data for ${this.name} in ${Date.now() - now}ms`);
+
+        // We use the same event curve for all emails to speed up the generation
+        // Spread over 14 days
+        this.eventStartTimeUsed = new Date();
+        const endTime = new Date(this.eventStartTimeUsed.getTime() + 1000 * 60 * 60 * 24 * 14);
+        this.eventCurve = generateEvents({
+            shape: 'ease-out',
+            trend: 'negative',
+            total: 1000,
+            startTime: this.eventStartTimeUsed,
+            endTime
+        });
+
+        this.membersSubscribeEventsByNewsletterId = new Map();
+        this.membersSubscribeEventsCreatedAtsByNewsletterId = new Map();
+        for (const memberSubscribeEvent of this.membersSubscribeEvents) {
+            if (!this.membersSubscribeEventsByNewsletterId.has(memberSubscribeEvent.newsletter_id)) {
+                this.membersSubscribeEventsByNewsletterId.set(memberSubscribeEvent.newsletter_id, []);
+            }
+            this.membersSubscribeEventsByNewsletterId.get(memberSubscribeEvent.newsletter_id).push(memberSubscribeEvent);
+
+            if (!this.membersSubscribeEventsCreatedAtsByNewsletterId.has(memberSubscribeEvent.newsletter_id)) {
+                this.membersSubscribeEventsCreatedAtsByNewsletterId.set(memberSubscribeEvent.newsletter_id, []);
+            }
+
+            if (!(memberSubscribeEvent.created_at instanceof Date)) {
+                // SQLite fix
+                memberSubscribeEvent.created_at = new Date(memberSubscribeEvent.created_at);
+            }
+            this.membersSubscribeEventsCreatedAtsByNewsletterId.get(memberSubscribeEvent.newsletter_id).push(memberSubscribeEvent.created_at.getTime());
+        }
+
+        await this.importForEach(this.emailBatches, quantity ? quantity / emails.length : 1000);
+    }
+
+    setReferencedModel(model) {
+        this.batch = model;
+        this.model = this.emails.get(this.batch.email_id);
+        this.batchIndex = this.batch.index;
+
+        // Shallow clone members list so we can shuffle and modify it
+        const earliestOpenTime = new Date(this.batch.updated_at);
+        const latestOpenTime = new Date(this.batch.updated_at);
+        latestOpenTime.setDate(latestOpenTime.getDate() + 14);
+
+        // Get all members that were subscribed to this newsletter BEFORE the batch was sent
+        // We use binary search to speed up it up
+        const lastIndex = findFirstHigherIndex(this.membersSubscribeEventsCreatedAtsByNewsletterId.get(this.model.newsletter_id), earliestOpenTime);
+
+        this.membersList = this.membersSubscribeEventsByNewsletterId.get(this.model.newsletter_id).slice(0, Math.max(0, lastIndex - 1))
+            .slice(this.batchIndex * 1000, (this.batchIndex + 1) * 1000)
+            .map(memberSubscribeEvent => memberSubscribeEvent.member_id);
+
+        this.events = faker.helpers.shuffle(this.eventCurve.slice(0, this.membersList.length));
+        this.eventIndex = 0;
+
+        this.emailMeta = {
+            // delievered and not opened
+            deliveredCount: this.model.delivered_count - this.model.opened_count,
+            openedCount: this.model.opened_count,
+            failedCount: this.model.failed_count
+        };
+
+        let offset = this.batchIndex * 1000;
+
+        // We always first create the failures, then the opened, then the delivered, so we need to remove those from the meta so we don't generate them multiple times
+        this.emailMeta = {
+            failedCount: Math.max(0, this.emailMeta.failedCount - offset),
+            openedCount: Math.max(0, this.emailMeta.openedCount - Math.max(0, offset - this.emailMeta.failedCount)),
+            deliveredCount: Math.max(0, this.emailMeta.deliveredCount - Math.max(0, offset - this.emailMeta.failedCount - this.emailMeta.openedCount))
+        };
+    }
+
+    generate() {
+        let timestamp = this.events.pop();
+        if (!timestamp) {
+            return;
+        }
+
+        // The events are generated for a different time, so we need to move them to the batch time
+        timestamp = new Date(timestamp.getTime() - this.eventStartTimeUsed.getTime() + new Date(this.batch.updated_at).getTime());
+
+        if (timestamp > new Date()) {
+            timestamp = new Date();
+        }
+
+        const memberId = this.membersList[this.events.length];
+        const member = this.members.get(memberId);
+
+        let status = emailStatus.none;
+        if (this.emailMeta.failedCount > 0) {
+            status = emailStatus.failed;
+            this.emailMeta.failedCount -= 1;
+        } else if (this.emailMeta.openedCount > 0) {
+            status = emailStatus.opened;
+            this.emailMeta.openedCount -= 1;
+        } else if (this.emailMeta.deliveredCount > 0) {
+            status = emailStatus.delivered;
+            this.emailMeta.deliveredCount -= 1;
+        }
+
+        let deliveredTime;
+        if (status === emailStatus.opened) {
+            const startDate = this.batch.updated_at;
+            const endDate = timestamp;
+            deliveredTime = faker.date.between(startDate, endDate);
+        }
+
+        return {
+            // Using sorted ids are much much faster (35% as far as my testing goes) for huge imports
+            id: this.fastFakeObjectId(),
+            email_id: this.model.id,
+            batch_id: this.batch.id,
+            member_id: member.id,
+            processed_at: dateToDatabaseString(this.batch.updated_at),
+            delivered_at: status === emailStatus.opened ? dateToDatabaseString(deliveredTime) : status === emailStatus.delivered ? dateToDatabaseString(timestamp) : null,
+            opened_at: status === emailStatus.opened ? dateToDatabaseString(timestamp) : null,
+            failed_at: status === emailStatus.failed ? dateToDatabaseString(timestamp) : null,
+            member_uuid: member.uuid,
+            member_email: member.email,
+            member_name: member.name
+        };
+    }
+}
+
+module.exports = EmailRecipientsImporter;

package/core/server/data/seeders/importers/EmailsImporter.js

@@ -0,0 +1,99 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const generateEvents = require('../utils/event-generator');
+const {luck} = require('../utils/random');
+const dateToDatabaseString = require('../utils/database-date');
+
+class EmailsImporter extends TableImporter {
+    static table = 'emails';
+    static dependencies = ['posts', 'newsletters', 'members_subscribe_events'];
+
+    constructor(knex, transaction) {
+        super(EmailsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        if (quantity === 0) {
+            return;
+        }
+
+        const posts = await this.transaction.select('id', 'title', 'published_at').from('posts').where('type', 'post').where('status', 'published').orderBy('published_at', 'desc');
+        this.newsletters = await this.transaction.select('id').from('newsletters').orderBy('sort_order');
+        this.membersSubscribeEvents = await this.transaction.select('id', 'newsletter_id', 'created_at').from('members_subscribe_events');
+
+        // Only generate emails for last 25% of posts, and only generate emails for 50% of those
+        await this.importForEach(posts.slice(0, Math.ceil(posts.length / 4)), quantity ? quantity / posts.length : 0.5);
+    }
+
+    generate() {
+        const id = this.fastFakeObjectId();
+
+        let newsletter;
+        if (this.newsletters.length === 0) {
+            return null;
+        } else if (this.newsletters.length === 1) {
+            newsletter = this.newsletters[0];
+        } else {
+            // Choose between first two newsletters
+            newsletter = luck(90)
+                // Regular premium
+                ? this.newsletters[0]
+                // Occasional freebie
+                : this.newsletters[1];
+        }
+
+        const timestamp = luck(60)
+            ? new Date(this.model.published_at)
+            : generateEvents({
+                shape: 'ease-out',
+                trend: 'negative',
+                total: 1,
+                startTime: new Date(this.model.published_at),
+                endTime: new Date()
+            })[0];
+
+        const recipientCount = this.membersSubscribeEvents
+            .filter(entry => entry.newsletter_id === newsletter.id)
+            .filter(entry => new Date(entry.created_at) < timestamp).length;
+        const deliveredCount = Math.ceil(recipientCount * faker.datatype.float({
+            max: 1,
+            min: 0.9,
+            precision: 0.001
+        }));
+        const openedCount = Math.ceil(deliveredCount * faker.datatype.float({
+            max: 0.95,
+            min: 0.6,
+            precision: 0.001
+        }));
+        const failedCount = Math.floor((recipientCount - deliveredCount) * faker.datatype.float({
+            max: 0.05,
+            min: 0,
+            precision: 0.001
+        }));
+
+        return {
+            id,
+            uuid: faker.datatype.uuid(),
+            post_id: this.model.id,
+            status: 'submitted',
+            recipient_filter: '', // TODO: Add recipient filter?
+            email_count: recipientCount,
+            delivered_count: deliveredCount,
+            opened_count: openedCount,
+            failed_count: failedCount,
+            subject: this.model.title,
+            source_type: 'html',
+            track_opens: true,
+            track_clicks: true,
+            feedback_enabled: true,
+            submitted_at: dateToDatabaseString(timestamp),
+            newsletter_id: newsletter.id,
+            created_at: dateToDatabaseString(timestamp),
+            created_by: 'unused',
+            updated_at: dateToDatabaseString(timestamp),
+            updated_by: 'unused'
+        };
+    }
+}
+
+module.exports = EmailsImporter;

package/core/server/data/seeders/importers/LabelsImporter.js

@@ -0,0 +1,41 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const {slugify} = require('@tryghost/string');
+const {blogStartDate} = require('../utils/blog-info');
+const dateToDatabaseString = require('../utils/database-date');
+
+class LabelsImporter extends TableImporter {
+    static table = 'labels';
+    static dependencies = [];
+    defaultQuantity = 10;
+
+    constructor(knex, transaction) {
+        super(LabelsImporter.table, knex, transaction);
+        this.generatedNames = new Set();
+    }
+
+    generateName() {
+        let name;
+        do {
+            name = `${faker.color.human()} ${faker.name.jobType()}`;
+            name = `${name[0].toUpperCase()}${name.slice(1)}`;
+        } while (this.generatedNames.has(name));
+        this.generatedNames.add(name);
+        return name;
+    }
+
+    generate() {
+        const name = this.generateName();
+        return {
+            id: this.fastFakeObjectId(),
+            name: name,
+            slug: `${slugify(name)}`,
+            created_at: dateToDatabaseString(blogStartDate),
+            created_by: '1',
+            updated_at: dateToDatabaseString(blogStartDate),
+            updated_by: '1'
+        };
+    }
+}
+
+module.exports = LabelsImporter;

package/core/server/data/seeders/importers/MembersClickEventsImporter.js

@@ -0,0 +1,69 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const {luck} = require('../utils/random');
+const dateToDatabaseString = require('../utils/database-date');
+
+class MembersClickEventsImporter extends TableImporter {
+    static table = 'members_click_events';
+    static dependencies = ['email_recipients', 'redirects', 'emails'];
+
+    constructor(knex, transaction) {
+        super(MembersClickEventsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const emailRecipients = await this.transaction.select('id', 'opened_at', 'email_id', 'member_id').from('email_recipients').whereNotNull('opened_at');
+        const redirects = await this.transaction.select('id', 'post_id').from('redirects');
+        const emails = await this.transaction.select('id', 'post_id').from('emails');
+        this.quantity = quantity ? quantity / emailRecipients.length : 2;
+
+        // Create maps for faster lookups (this does make a difference for large data generation)
+        this.emails = new Map();
+        for (const email of emails) {
+            this.emails.set(email.id, email);
+        }
+
+        this.redirects = new Map();
+        for (const redirect of redirects) {
+            if (!this.redirects.has(redirect.post_id)) {
+                this.redirects.set(redirect.post_id, []);
+            }
+            this.redirects.get(redirect.post_id).push(redirect);
+        }
+
+        await this.importForEach(emailRecipients, this.quantity);
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
+        this.amount = model.opened_at === null ? 0 : luck(40) ? faker.datatype.number({
+            min: 0,
+            max: this.quantity
+        }) : 0;
+        const email = this.emails.get(model.email_id);
+        this.redirectList = this.redirects.get(email.post_id) ?? [];
+    }
+
+    generate() {
+        if (this.amount <= 0 || this.redirectList.length === 0 || !this.model.opened_at) {
+            return;
+        }
+        this.amount -= 1;
+
+        const openedAt = new Date(this.model.opened_at);
+        const laterOn = new Date(openedAt.getTime() + 1000 * 60 * 15);
+        const clickTime = faker.date.between(openedAt.getTime(), laterOn.getTime()); //added getTime here because it threw random errors
+
+        return {
+            id: this.fastFakeObjectId(),
+            member_id: this.model.member_id,
+            redirect_id: this.redirectList[this.redirectList.length === 1 ? 0 : (faker.datatype.number({
+                min: 0,
+                max: this.redirectList.length - 1
+            }))].id,
+            created_at: dateToDatabaseString(clickTime)
+        };
+    }
+}
+
+module.exports = MembersClickEventsImporter;