ghost 5.115.0 → 5.116.0

package/core/server/data/seeders/importers/MembersCreatedEventsImporter.js

@@ -0,0 +1,103 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const {luck} = require('../utils/random');
+const dateToDatabaseString = require('../utils/database-date');
+
+class MembersCreatedEventsImporter extends TableImporter {
+    static table = 'members_created_events';
+    static dependencies = ['members', 'posts', 'mentions'];
+
+    constructor(knex, transaction) {
+        super(MembersCreatedEventsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const members = await this.transaction.select('id', 'created_at').from('members');
+        this.posts = await this.transaction.select('id', 'published_at', 'visibility', 'type', 'slug').from('posts').orderBy('published_at', 'desc');
+        this.incomingRecommendations = await this.transaction.select('id', 'source', 'created_at').from('mentions');
+
+        await this.importForEach(members, quantity ? quantity / members.length : 1);
+    }
+
+    generateSource() {
+        let source = 'member';
+        if (luck(10)) {
+            source = 'admin';
+        } else if (luck(5)) {
+            source = 'api';
+        } else if (luck(5)) { // eslint-disable-line no-dupe-else-if
+            source = 'import';
+        }
+        return source;
+    }
+
+    generate() {
+        const source = this.generateSource();
+
+        // We need to add all properties here already otherwise CSV imports won't know all the columns
+        let attribution = {
+            attribution_id: null,
+            attribution_type: null,
+            attribution_url: null
+        };
+        let referrer = {
+            referrer_source: null,
+            referrer_url: null,
+            referrer_medium: null
+        };
+
+        if (source === 'member' && luck(30)) {
+            const post = this.posts.find(p => p.visibility === 'public' && new Date(p.published_at) < new Date(this.model.created_at));
+            if (post) {
+                attribution = {
+                    attribution_id: post.id,
+                    attribution_type: post.type,
+                    attribution_url: post.slug
+                };
+            }
+        }
+
+        if (source === 'member' && luck(40)) {
+            if (luck(20)) {
+                // Ghost network
+                referrer = {
+                    referrer_source: luck(20) ? 'Ghost.org' : 'Ghost Explore',
+                    referrer_url: 'ghost.org',
+                    referrer_medium: 'Ghost Network'
+                };
+            } else {
+                // Incoming recommendation
+                const incomingRecommendation = faker.helpers.arrayElement(this.incomingRecommendations);
+
+                const hostname = new URL(incomingRecommendation.source).hostname;
+                referrer = {
+                    referrer_source: hostname,
+                    referrer_url: hostname,
+                    referrer_medium: faker.helpers.arrayElement([null, 'Email'])
+                };
+            }
+        }
+
+        if (source === 'import') {
+            referrer.referrer_source = 'Imported';
+            referrer.referrer_medium = 'Member Importer';
+        } else if (source === 'admin') {
+            referrer.referrer_source = 'Created manually';
+            referrer.referrer_medium = 'Ghost Admin';
+        } else if (source === 'api') {
+            referrer.referrer_source = 'Created via API';
+            referrer.referrer_medium = 'Admin API';
+        }
+
+        return {
+            id: this.fastFakeObjectId(),
+            created_at: dateToDatabaseString(this.model.created_at),
+            member_id: this.model.id,
+            source,
+            ...attribution,
+            ...referrer
+        };
+    }
+}
+
+module.exports = MembersCreatedEventsImporter;

package/core/server/data/seeders/importers/MembersFeedbackImporter.js

@@ -0,0 +1,45 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const {luck} = require('../utils/random');
+const dateToDatabaseString = require('../utils/database-date');
+
+class MembersFeedbackImporter extends TableImporter {
+    static table = 'members_feedback';
+    static dependencies = ['emails', 'email_recipients'];
+
+    constructor(knex, transaction, {emails}) {
+        super(MembersFeedbackImporter.table, knex, transaction);
+        this.emails = emails;
+    }
+
+    async import(quantity) {
+        const emailRecipients = await this.transaction.select('id', 'opened_at', 'email_id', 'member_id').from('email_recipients');
+        this.emails = await this.transaction.select('id', 'post_id').from('emails');
+
+        await this.importForEach(emailRecipients, quantity ? quantity / emailRecipients.length : 1);
+    }
+
+    generate() {
+        // ~10% of people who opened the email will leave feedback
+        if (!this.model.opened_at || luck(90)) {
+            return null;
+        }
+
+        const openedAt = new Date(this.model.opened_at);
+        const laterOn = new Date(this.model.opened_at);
+        laterOn.setMinutes(laterOn.getMinutes() + 60);
+        const feedbackTime = faker.date.between(openedAt, laterOn);
+
+        const postId = this.emails.find(email => email.id === this.model.email_id).post_id;
+        return {
+            id: this.fastFakeObjectId(),
+            score: luck(70) ? 1 : 0,
+            member_id: this.model.member_id,
+            post_id: postId,
+            created_at: dateToDatabaseString(feedbackTime),
+            updated_at: dateToDatabaseString(feedbackTime)
+        };
+    }
+}
+
+module.exports = MembersFeedbackImporter;

package/core/server/data/seeders/importers/MembersImporter.js

@@ -0,0 +1,111 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const {faker: americanFaker} = require('@faker-js/faker/locale/en_US');
+const {blogStartDate: startTime} = require('../utils/blog-info');
+const generateEvents = require('../utils/event-generator');
+const {luck} = require('../utils/random');
+const dateToDatabaseString = require('../utils/database-date');
+const debug = require('@tryghost/debug')('MembersImporter');
+
+class MembersImporter extends TableImporter {
+    static table = 'members';
+    static dependencies = [];
+    defaultQuantity = faker.datatype.number({
+        min: 7000,
+        max: 8000
+    });
+
+    constructor(knex, transaction) {
+        super(MembersImporter.table, knex, transaction);
+    }
+
+    async import(quantity = this.defaultQuantity) {
+        const generateNow = Date.now();
+
+        this.timestamps = generateEvents({
+            shape: 'ease-in',
+            trend: 'positive',
+            total: quantity,
+            startTime,
+            endTime: new Date()
+        }).sort();
+        debug(`${this.name} generated ${this.timestamps.length} timestamps in ${Date.now() - generateNow}ms`);
+
+        await super.import(quantity);
+    }
+
+    /**
+     * Add open rate data to members table
+     */
+    async finalise() {
+        const emailRecipients = await this.transaction.select('id', 'member_id', 'opened_at').from('email_recipients');
+
+        const memberData = {};
+        for (const emailRecipient of emailRecipients) {
+            if (!(emailRecipient.member_id in memberData)) {
+                memberData[emailRecipient.member_id] = {
+                    emailCount: 1,
+                    openedCount: emailRecipient.opened_at ? 1 : 0
+                };
+            } else {
+                memberData[emailRecipient.member_id].emailCount += 1;
+                if (emailRecipient.opened_at) {
+                    memberData[emailRecipient.member_id].openedCount += 1;
+                }
+            }
+        }
+
+        for (const [memberId, emailInfo] of Object.entries(memberData)) {
+            const openRate = Math.round(100 * (emailInfo.openedCount / emailInfo.emailCount));
+            await this.transaction('members').update({
+                email_count: emailInfo.emailCount,
+                email_opened_count: emailInfo.openedCount,
+                email_open_rate: emailInfo.emailCount >= 5 ? openRate : null
+            }).where({id: memberId});
+        }
+    }
+
+    generate() {
+        const id = this.fastFakeObjectId();
+        // Use name from American locale to reflect an English-speaking audience
+        const name = `${americanFaker.name.firstName()} ${americanFaker.name.lastName()}`;
+        const timestamp = this.timestamps.pop();
+
+        return {
+            id,
+            uuid: faker.datatype.uuid(),
+            transient_id: faker.datatype.uuid(),
+            email: `${name.replace(' ', '.').replace(/[^a-zA-Z0-9]/g, '').toLowerCase()}${faker.datatype.number({min: 0, max: 999999})}@example.com`,
+            status: luck(5) ? 'comped' : luck(15) ? 'paid' : 'free',
+            name: name,
+            expertise: luck(30) ? faker.name.jobTitle() : undefined,
+            geolocation: JSON.stringify({
+                organization_name: faker.company.name(),
+                region: faker.address.state(),
+                accuracy: 50,
+                asn: parseInt(faker.random.numeric(4)),
+                organization: `${faker.random.alpha({count: 2, casing: 'upper'})}${faker.random.numeric(4)} ${faker.company.name()}`,
+                timezone: faker.address.timeZone(),
+                longitude: faker.address.longitude(),
+                country_code3: faker.address.countryCode('alpha-3'),
+                area_code: '0',
+                ip: faker.internet.ipv4(),
+                city: faker.address.cityName(),
+                country: faker.address.country(),
+                continent_code: 'EU',
+                country_code: faker.address.countryCode('alpha-2'),
+                latitude: faker.address.latitude()
+            }),
+            email_count: 0, // Depends on number of emails sent since created_at, the newsletter they're a part of and subscription status
+            email_opened_count: 0,
+            email_open_rate: null,
+            // 40% of users logged in within a week, 60% sometime since registering
+            last_seen_at: luck(40) ? dateToDatabaseString(faker.date.recent(7)) : dateToDatabaseString(faker.date.between(timestamp, new Date())),
+            created_at: dateToDatabaseString(timestamp),
+            created_by: id,
+            updated_at: dateToDatabaseString(timestamp)
+        };
+    }
+}
+
+module.exports = MembersImporter;

package/core/server/data/seeders/importers/MembersLabelsImporter.js

@@ -0,0 +1,39 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const {luck} = require('../utils/random');
+
+class MembersLabelsImporter extends TableImporter {
+    static table = 'members_labels';
+    static dependencies = ['labels', 'members'];
+
+    constructor(knex, transaction, {labels}) {
+        super(MembersLabelsImporter.table, knex, transaction);
+        this.labels = labels;
+    }
+
+    async import(quantity) {
+        const members = await this.transaction.select('id').from('members');
+        this.labels = await this.transaction.select('id').from('labels');
+
+        await this.importForEach(members, quantity ? quantity / members.length : 1);
+    }
+
+    generate() {
+        if (luck(90)) {
+            // 90% of members don't have labels
+            return;
+        }
+        // TODO: Ensure we don't generate the same member label twice
+        return {
+            id: this.fastFakeObjectId(),
+            member_id: this.model.id,
+            label_id: this.labels[faker.datatype.number({
+                min: 0,
+                max: this.labels.length - 1
+            })].id,
+            sort_order: 0
+        };
+    }
+}
+
+module.exports = MembersLabelsImporter;

package/core/server/data/seeders/importers/MembersLoginEventsImporter.js

@@ -0,0 +1,69 @@
+const TableImporter = require('./TableImporter');
+const {luck} = require('../utils/random');
+const generateEvents = require('../utils/event-generator');
+const dateToDatabaseString = require('../utils/database-date');
+
+class MembersLoginEventsImporter extends TableImporter {
+    static table = 'members_login_events';
+    static dependencies = ['members'];
+
+    constructor(knex, transaction) {
+        super(MembersLoginEventsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        if (quantity === 0) {
+            return;
+        }
+
+        let offset = 0;
+        let limit = 100000;
+
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const members = await this.transaction.select('id', 'created_at').from('members').limit(limit).offset(offset);
+
+            if (members.length === 0) {
+                break;
+            }
+
+            await this.importForEach(members, quantity ? quantity / members.length : 5);
+
+            offset += limit;
+        }
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
+
+        const endDate = new Date();
+        const daysBetween = Math.ceil((endDate.valueOf() - new Date(model.created_at).valueOf()) / (1000 * 60 * 60 * 24));
+
+        // Assuming most people either subscribe and lose interest, or maintain steady readership
+        const shape = luck(40) ? 'ease-out' : 'flat';
+        this.timestamps = generateEvents({
+            shape,
+            trend: 'negative',
+            // Steady readers login more, readers who lose interest read less overall.
+            // ceil because members will all have logged in at least once
+            total: Math.min(5, shape === 'flat' ? Math.ceil(daysBetween / 3) : Math.ceil(daysBetween / 7)),
+            startTime: new Date(model.created_at),
+            endTime: endDate
+        });
+    }
+
+    generate() {
+        const timestamp = this.timestamps.pop();
+        if (!timestamp) {
+            // Out of events for this user
+            return null;
+        }
+        return {
+            id: this.fastFakeObjectId(),
+            created_at: dateToDatabaseString(timestamp),
+            member_id: this.model.id
+        };
+    }
+}
+
+module.exports = MembersLoginEventsImporter;

package/core/server/data/seeders/importers/MembersNewslettersImporter.js

@@ -0,0 +1,38 @@
+const TableImporter = require('./TableImporter');
+
+class MembersNewslettersImporter extends TableImporter {
+    static table = 'members_newsletters';
+    static dependencies = ['members_subscribe_events'];
+
+    constructor(knex, transaction) {
+        super(MembersNewslettersImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        let offset = 0;
+        let limit = 100000;
+
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const membersSubscribeEvents = await this.transaction.select('member_id', 'newsletter_id').from('members_subscribe_events').limit(limit).offset(offset);
+
+            if (membersSubscribeEvents.length === 0) {
+                break;
+            }
+
+            await this.importForEach(membersSubscribeEvents, quantity ? quantity / membersSubscribeEvents.length : 1);
+
+            offset += limit;
+        }
+    }
+
+    generate() {
+        return {
+            id: this.fastFakeObjectId(),
+            member_id: this.model.member_id,
+            newsletter_id: this.model.newsletter_id
+        };
+    }
+}
+
+module.exports = MembersNewslettersImporter;

package/core/server/data/seeders/importers/MembersPaidSubscriptionEventsImporter.js

@@ -0,0 +1,99 @@
+const TableImporter = require('./TableImporter');
+
+class MembersPaidSubscriptionEventsImporter extends TableImporter {
+    static table = 'members_paid_subscription_events';
+    static dependencies = ['members_stripe_customers_subscriptions'];
+
+    constructor(knex, transaction) {
+        super(MembersPaidSubscriptionEventsImporter.table, knex, transaction);
+    }
+
+    async import() {
+        let offset = 0;
+        let limit = 1000;
+
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const subscriptions = await this.transaction.select('id', 'customer_id', 'plan_currency', 'plan_amount', 'created_at', 'plan_id', 'status', 'cancel_at_period_end', 'current_period_end').from('members_stripe_customers_subscriptions').limit(limit).offset(offset);
+
+            if (subscriptions.length === 0) {
+                break;
+            }
+            const membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers').whereIn('customer_id', subscriptions.map(subscription => subscription.customer_id));
+
+            this.membersStripeCustomers = new Map();
+            for (const customer of membersStripeCustomers) {
+                this.membersStripeCustomers.set(customer.customer_id, customer);
+            }
+            await this.importForEach(subscriptions, 2);
+
+            offset += limit;
+        }
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
+        this.count = 0;
+    }
+
+    isActiveSubscriptionStatus(status) {
+        return ['active', 'trialing', 'unpaid', 'past_due'].includes(status);
+    }
+
+    getStatus(modelToCheck) {
+        const status = modelToCheck.status;
+        const canceled = modelToCheck.cancel_at_period_end;
+
+        if (status === 'canceled') {
+            return 'expired';
+        }
+
+        if (canceled) {
+            return 'canceled';
+        }
+
+        if (this.isActiveSubscriptionStatus(status)) {
+            return 'active';
+        }
+
+        return 'inactive';
+    }
+
+    generate() {
+        this.count += 1;
+
+        const isActive = this.isActiveSubscriptionStatus(this.model.status);
+        if (this.count > 1 && isActive) {
+            // We only need one event, because the MRR is still here
+            return;
+        }
+
+        if (this.model.status === 'incomplete' || this.model.status === 'incomplete_expired') {
+            // Not a paid subscription
+            return;
+        }
+
+        const memberCustomer = this.membersStripeCustomers.get(this.model.customer_id);
+        const isMonthly = this.model.plan_interval === 'month';
+
+        // Note that we need to recalculate the MRR, because it will be zero for inactive subscrptions
+        const mrr = isMonthly ? this.model.plan_amount : Math.floor(this.model.plan_amount / 12);
+
+        // todo: implement + MRR and -MRR in case of inactive subscriptions
+        return {
+            id: this.fastFakeObjectId(),
+            // TODO: Support expired / updated / cancelled events too
+            type: this.count === 1 ? 'created' : this.getStatus(this.model),
+            member_id: memberCustomer.member_id,
+            subscription_id: this.model.id,
+            from_plan: this.count === 1 ? null : this.model.plan_id,
+            to_plan: this.count === 1 ? this.model.plan_id : null,
+            currency: this.model.plan_currency,
+            source: 'stripe',
+            mrr_delta: this.count === 1 ? mrr : -mrr,
+            created_at: this.count === 1 ? this.model.created_at : this.model.current_period_end
+        };
+    }
+}
+
+module.exports = MembersPaidSubscriptionEventsImporter;

package/core/server/data/seeders/importers/MembersProductsImporter.js

@@ -0,0 +1,42 @@
+const {faker} = require('@faker-js/faker');
+const TableImporter = require('./TableImporter');
+const {luck} = require('../utils/random');
+const dateToDatabaseString = require('../utils/database-date');
+
+class MembersProductsImporter extends TableImporter {
+    static table = 'members_products';
+    static dependencies = ['products', 'members'];
+
+    constructor(knex, transaction) {
+        super(MembersProductsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const members = await this.transaction.select('id').from('members').whereNot('status', 'free');
+        this.products = await this.transaction.select('id').from('products').whereNot('type', 'fee');
+
+        await this.importForEach(members, quantity ? quantity / members.length : 1);
+    }
+
+    getProduct() {
+        if (this.products.length > 1) {
+            return luck(10) ? this.products[2]
+                : luck(50) ? this.products[1]
+                    : this.products[0];
+        } else {
+            return this.products[0];
+        }
+    }
+
+    generate() {
+        return {
+            id: this.fastFakeObjectId(),
+            member_id: this.model.id,
+            product_id: this.getProduct().id,
+            sort_order: 0,
+            expiry_at: this.model.status === 'paid' ? null : (luck(50) ? null : dateToDatabaseString(faker.date.future()))
+        };
+    }
+}
+
+module.exports = MembersProductsImporter;

package/core/server/data/seeders/importers/MembersStatusEventsImporter.js

@@ -0,0 +1,58 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const dateToDatabaseString = require('../utils/database-date');
+
+class MembersStatusEventsImporter extends TableImporter {
+    static table = 'members_status_events';
+    static dependencies = ['members'];
+
+    constructor(knex, transaction) {
+        super(MembersStatusEventsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        let offset = 0;
+        let limit = 100000;
+
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const members = await this.transaction.select('id', 'created_at', 'status').from('members').limit(limit).offset(offset);
+
+            if (members.length === 0) {
+                break;
+            }
+
+            await this.importForEach(members, quantity ? quantity / members.length : 2);
+            offset += limit;
+        }
+    }
+
+    setReferencedModel(model) {
+        this.events = [{
+            id: this.fastFakeObjectId(),
+            member_id: model.id,
+            from_status: null,
+            to_status: 'free',
+            created_at: dateToDatabaseString(model.created_at)
+        }];
+        if (model.status !== 'free') {
+            this.events.push({
+                id: this.fastFakeObjectId(),
+                member_id: model.id,
+                from_status: 'free',
+                to_status: model.status,
+                created_at: dateToDatabaseString(faker.date.between(new Date(model.created_at), new Date()))
+            });
+        }
+    }
+
+    generate() {
+        const event = this.events.pop();
+        if (!event) {
+            return null;
+        }
+        return event;
+    }
+}
+
+module.exports = MembersStatusEventsImporter;

package/core/server/data/seeders/importers/MembersStripeCustomersImporter.js

@@ -0,0 +1,60 @@
+const {faker} = require('@faker-js/faker');
+const TableImporter = require('./TableImporter');
+
+class MembersStripeCustomersImporter extends TableImporter {
+    static table = 'members_stripe_customers';
+    static dependencies = ['members'];
+
+    constructor(knex, transaction) {
+        super(MembersStripeCustomersImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        if (quantity === 0) {
+            return;
+        }
+
+        let offset = 0;
+        let limit = 100000;
+
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const members = await this.transaction.select('id', 'name', 'email', 'created_at', 'status').from('members').limit(limit).offset(offset);
+
+            if (members.length === 0) {
+                break;
+            }
+
+            await this.importForEach(members, quantity ? quantity / members.length : 1);
+            offset += limit;
+        }
+    }
+
+    generate() {
+        if (this.model.status !== 'paid') {
+            // Only 30% of free members should have a stripe customer = have had a subscription in the past or tried to subscribe
+            // The number should increase the older the member is
+
+            const daysSinceMemberCreated = Math.floor((new Date() - new Date(this.model.created_at)) / (1000 * 60 * 60 * 24));
+            const shouldHaveStripeCustomer = faker.datatype.number({min: 0, max: 100}) < Math.max(Math.min(daysSinceMemberCreated / 60, 15), 2);
+
+            if (!shouldHaveStripeCustomer) {
+                return;
+            }
+        }
+
+        return {
+            id: this.fastFakeObjectId(),
+            member_id: this.model.id,
+            customer_id: `cus_${faker.random.alphaNumeric(14, {
+                casing: 'mixed'
+            })}`,
+            name: this.model.name,
+            email: this.model.email,
+            created_at: this.model.created_at,
+            created_by: 'unused'
+        };
+    }
+}
+
+module.exports = MembersStripeCustomersImporter;