ghost 5.115.1 → 5.116.1

package/core/server/data/seeders/importers/MembersLabelsImporter.js

@@ -0,0 +1,39 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const {luck} = require('../utils/random');
+
+class MembersLabelsImporter extends TableImporter {
+    static table = 'members_labels';
+    static dependencies = ['labels', 'members'];
+
+    constructor(knex, transaction, {labels}) {
+        super(MembersLabelsImporter.table, knex, transaction);
+        this.labels = labels;
+    }
+
+    async import(quantity) {
+        const members = await this.transaction.select('id').from('members');
+        this.labels = await this.transaction.select('id').from('labels');
+
+        await this.importForEach(members, quantity ? quantity / members.length : 1);
+    }
+
+    generate() {
+        if (luck(90)) {
+            // 90% of members don't have labels
+            return;
+        }
+        // TODO: Ensure we don't generate the same member label twice
+        return {
+            id: this.fastFakeObjectId(),
+            member_id: this.model.id,
+            label_id: this.labels[faker.datatype.number({
+                min: 0,
+                max: this.labels.length - 1
+            })].id,
+            sort_order: 0
+        };
+    }
+}
+
+module.exports = MembersLabelsImporter;

package/core/server/data/seeders/importers/MembersLoginEventsImporter.js

@@ -0,0 +1,69 @@
+const TableImporter = require('./TableImporter');
+const {luck} = require('../utils/random');
+const generateEvents = require('../utils/event-generator');
+const dateToDatabaseString = require('../utils/database-date');
+
+class MembersLoginEventsImporter extends TableImporter {
+    static table = 'members_login_events';
+    static dependencies = ['members'];
+
+    constructor(knex, transaction) {
+        super(MembersLoginEventsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        if (quantity === 0) {
+            return;
+        }
+
+        let offset = 0;
+        let limit = 100000;
+
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const members = await this.transaction.select('id', 'created_at').from('members').limit(limit).offset(offset);
+
+            if (members.length === 0) {
+                break;
+            }
+
+            await this.importForEach(members, quantity ? quantity / members.length : 5);
+
+            offset += limit;
+        }
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
+
+        const endDate = new Date();
+        const daysBetween = Math.ceil((endDate.valueOf() - new Date(model.created_at).valueOf()) / (1000 * 60 * 60 * 24));
+
+        // Assuming most people either subscribe and lose interest, or maintain steady readership
+        const shape = luck(40) ? 'ease-out' : 'flat';
+        this.timestamps = generateEvents({
+            shape,
+            trend: 'negative',
+            // Steady readers login more, readers who lose interest read less overall.
+            // ceil because members will all have logged in at least once
+            total: Math.min(5, shape === 'flat' ? Math.ceil(daysBetween / 3) : Math.ceil(daysBetween / 7)),
+            startTime: new Date(model.created_at),
+            endTime: endDate
+        });
+    }
+
+    generate() {
+        const timestamp = this.timestamps.pop();
+        if (!timestamp) {
+            // Out of events for this user
+            return null;
+        }
+        return {
+            id: this.fastFakeObjectId(),
+            created_at: dateToDatabaseString(timestamp),
+            member_id: this.model.id
+        };
+    }
+}
+
+module.exports = MembersLoginEventsImporter;

package/core/server/data/seeders/importers/MembersNewslettersImporter.js

@@ -0,0 +1,38 @@
+const TableImporter = require('./TableImporter');
+
+class MembersNewslettersImporter extends TableImporter {
+    static table = 'members_newsletters';
+    static dependencies = ['members_subscribe_events'];
+
+    constructor(knex, transaction) {
+        super(MembersNewslettersImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        let offset = 0;
+        let limit = 100000;
+
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const membersSubscribeEvents = await this.transaction.select('member_id', 'newsletter_id').from('members_subscribe_events').limit(limit).offset(offset);
+
+            if (membersSubscribeEvents.length === 0) {
+                break;
+            }
+
+            await this.importForEach(membersSubscribeEvents, quantity ? quantity / membersSubscribeEvents.length : 1);
+
+            offset += limit;
+        }
+    }
+
+    generate() {
+        return {
+            id: this.fastFakeObjectId(),
+            member_id: this.model.member_id,
+            newsletter_id: this.model.newsletter_id
+        };
+    }
+}
+
+module.exports = MembersNewslettersImporter;

package/core/server/data/seeders/importers/MembersPaidSubscriptionEventsImporter.js

@@ -0,0 +1,99 @@
+const TableImporter = require('./TableImporter');
+
+class MembersPaidSubscriptionEventsImporter extends TableImporter {
+    static table = 'members_paid_subscription_events';
+    static dependencies = ['members_stripe_customers_subscriptions'];
+
+    constructor(knex, transaction) {
+        super(MembersPaidSubscriptionEventsImporter.table, knex, transaction);
+    }
+
+    async import() {
+        let offset = 0;
+        let limit = 1000;
+
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const subscriptions = await this.transaction.select('id', 'customer_id', 'plan_currency', 'plan_amount', 'created_at', 'plan_id', 'status', 'cancel_at_period_end', 'current_period_end').from('members_stripe_customers_subscriptions').limit(limit).offset(offset);
+
+            if (subscriptions.length === 0) {
+                break;
+            }
+            const membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers').whereIn('customer_id', subscriptions.map(subscription => subscription.customer_id));
+
+            this.membersStripeCustomers = new Map();
+            for (const customer of membersStripeCustomers) {
+                this.membersStripeCustomers.set(customer.customer_id, customer);
+            }
+            await this.importForEach(subscriptions, 2);
+
+            offset += limit;
+        }
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
+        this.count = 0;
+    }
+
+    isActiveSubscriptionStatus(status) {
+        return ['active', 'trialing', 'unpaid', 'past_due'].includes(status);
+    }
+
+    getStatus(modelToCheck) {
+        const status = modelToCheck.status;
+        const canceled = modelToCheck.cancel_at_period_end;
+
+        if (status === 'canceled') {
+            return 'expired';
+        }
+
+        if (canceled) {
+            return 'canceled';
+        }
+
+        if (this.isActiveSubscriptionStatus(status)) {
+            return 'active';
+        }
+
+        return 'inactive';
+    }
+
+    generate() {
+        this.count += 1;
+
+        const isActive = this.isActiveSubscriptionStatus(this.model.status);
+        if (this.count > 1 && isActive) {
+            // We only need one event, because the MRR is still here
+            return;
+        }
+
+        if (this.model.status === 'incomplete' || this.model.status === 'incomplete_expired') {
+            // Not a paid subscription
+            return;
+        }
+
+        const memberCustomer = this.membersStripeCustomers.get(this.model.customer_id);
+        const isMonthly = this.model.plan_interval === 'month';
+
+        // Note that we need to recalculate the MRR, because it will be zero for inactive subscrptions
+        const mrr = isMonthly ? this.model.plan_amount : Math.floor(this.model.plan_amount / 12);
+
+        // todo: implement + MRR and -MRR in case of inactive subscriptions
+        return {
+            id: this.fastFakeObjectId(),
+            // TODO: Support expired / updated / cancelled events too
+            type: this.count === 1 ? 'created' : this.getStatus(this.model),
+            member_id: memberCustomer.member_id,
+            subscription_id: this.model.id,
+            from_plan: this.count === 1 ? null : this.model.plan_id,
+            to_plan: this.count === 1 ? this.model.plan_id : null,
+            currency: this.model.plan_currency,
+            source: 'stripe',
+            mrr_delta: this.count === 1 ? mrr : -mrr,
+            created_at: this.count === 1 ? this.model.created_at : this.model.current_period_end
+        };
+    }
+}
+
+module.exports = MembersPaidSubscriptionEventsImporter;

package/core/server/data/seeders/importers/MembersProductsImporter.js

@@ -0,0 +1,42 @@
+const {faker} = require('@faker-js/faker');
+const TableImporter = require('./TableImporter');
+const {luck} = require('../utils/random');
+const dateToDatabaseString = require('../utils/database-date');
+
+class MembersProductsImporter extends TableImporter {
+    static table = 'members_products';
+    static dependencies = ['products', 'members'];
+
+    constructor(knex, transaction) {
+        super(MembersProductsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        const members = await this.transaction.select('id').from('members').whereNot('status', 'free');
+        this.products = await this.transaction.select('id').from('products').whereNot('type', 'fee');
+
+        await this.importForEach(members, quantity ? quantity / members.length : 1);
+    }
+
+    getProduct() {
+        if (this.products.length > 1) {
+            return luck(10) ? this.products[2]
+                : luck(50) ? this.products[1]
+                    : this.products[0];
+        } else {
+            return this.products[0];
+        }
+    }
+
+    generate() {
+        return {
+            id: this.fastFakeObjectId(),
+            member_id: this.model.id,
+            product_id: this.getProduct().id,
+            sort_order: 0,
+            expiry_at: this.model.status === 'paid' ? null : (luck(50) ? null : dateToDatabaseString(faker.date.future()))
+        };
+    }
+}
+
+module.exports = MembersProductsImporter;

package/core/server/data/seeders/importers/MembersStatusEventsImporter.js

@@ -0,0 +1,58 @@
+const TableImporter = require('./TableImporter');
+const {faker} = require('@faker-js/faker');
+const dateToDatabaseString = require('../utils/database-date');
+
+class MembersStatusEventsImporter extends TableImporter {
+    static table = 'members_status_events';
+    static dependencies = ['members'];
+
+    constructor(knex, transaction) {
+        super(MembersStatusEventsImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        let offset = 0;
+        let limit = 100000;
+
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const members = await this.transaction.select('id', 'created_at', 'status').from('members').limit(limit).offset(offset);
+
+            if (members.length === 0) {
+                break;
+            }
+
+            await this.importForEach(members, quantity ? quantity / members.length : 2);
+            offset += limit;
+        }
+    }
+
+    setReferencedModel(model) {
+        this.events = [{
+            id: this.fastFakeObjectId(),
+            member_id: model.id,
+            from_status: null,
+            to_status: 'free',
+            created_at: dateToDatabaseString(model.created_at)
+        }];
+        if (model.status !== 'free') {
+            this.events.push({
+                id: this.fastFakeObjectId(),
+                member_id: model.id,
+                from_status: 'free',
+                to_status: model.status,
+                created_at: dateToDatabaseString(faker.date.between(new Date(model.created_at), new Date()))
+            });
+        }
+    }
+
+    generate() {
+        const event = this.events.pop();
+        if (!event) {
+            return null;
+        }
+        return event;
+    }
+}
+
+module.exports = MembersStatusEventsImporter;

package/core/server/data/seeders/importers/MembersStripeCustomersImporter.js

@@ -0,0 +1,60 @@
+const {faker} = require('@faker-js/faker');
+const TableImporter = require('./TableImporter');
+
+class MembersStripeCustomersImporter extends TableImporter {
+    static table = 'members_stripe_customers';
+    static dependencies = ['members'];
+
+    constructor(knex, transaction) {
+        super(MembersStripeCustomersImporter.table, knex, transaction);
+    }
+
+    async import(quantity) {
+        if (quantity === 0) {
+            return;
+        }
+
+        let offset = 0;
+        let limit = 100000;
+
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const members = await this.transaction.select('id', 'name', 'email', 'created_at', 'status').from('members').limit(limit).offset(offset);
+
+            if (members.length === 0) {
+                break;
+            }
+
+            await this.importForEach(members, quantity ? quantity / members.length : 1);
+            offset += limit;
+        }
+    }
+
+    generate() {
+        if (this.model.status !== 'paid') {
+            // Only 30% of free members should have a stripe customer = have had a subscription in the past or tried to subscribe
+            // The number should increase the older the member is
+
+            const daysSinceMemberCreated = Math.floor((new Date() - new Date(this.model.created_at)) / (1000 * 60 * 60 * 24));
+            const shouldHaveStripeCustomer = faker.datatype.number({min: 0, max: 100}) < Math.max(Math.min(daysSinceMemberCreated / 60, 15), 2);
+
+            if (!shouldHaveStripeCustomer) {
+                return;
+            }
+        }
+
+        return {
+            id: this.fastFakeObjectId(),
+            member_id: this.model.id,
+            customer_id: `cus_${faker.random.alphaNumeric(14, {
+                casing: 'mixed'
+            })}`,
+            name: this.model.name,
+            email: this.model.email,
+            created_at: this.model.created_at,
+            created_by: 'unused'
+        };
+    }
+}
+
+module.exports = MembersStripeCustomersImporter;

package/core/server/data/seeders/importers/MembersStripeCustomersSubscriptionsImporter.js

@@ -0,0 +1,259 @@
+const {faker} = require('@faker-js/faker');
+const TableImporter = require('./TableImporter');
+const dateToDatabaseString = require('../utils/database-date');
+const generateEvents = require('../utils/event-generator');
+const {luck} = require('../utils/random');
+
+class MembersStripeCustomersSubscriptionsImporter extends TableImporter {
+    static table = 'members_stripe_customers_subscriptions';
+    static dependencies = ['members', 'members_products', 'members_stripe_customers', 'products', 'stripe_products', 'stripe_prices'];
+
+    constructor(knex, transaction) {
+        super(MembersStripeCustomersSubscriptionsImporter.table, knex, transaction);
+    }
+
+    async import() {
+        let offset = 0;
+        let limit = 5000;
+        this.products = await this.transaction.select('id', 'name').from('products').whereNot('type', 'free');
+        this.stripeProducts = await this.transaction.select('id', 'product_id', 'stripe_product_id').from('stripe_products');
+        this.stripePrices = await this.transaction.select('id', 'nickname', 'stripe_product_id', 'stripe_price_id', 'amount', 'interval', 'currency').from('stripe_prices');
+
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers').limit(limit).offset(offset);
+
+            if (membersStripeCustomers.length === 0) {
+                break;
+            }
+
+            this.members = await this.transaction.select('id', 'status', 'created_at').from('members').whereIn('id', membersStripeCustomers.map(m => m.member_id));
+
+            if (this.members.length === 0) {
+                continue;
+            }
+
+            const membersProducts = await this.transaction.select('member_id', 'product_id').from('members_products').whereIn('member_id', this.members.map(member => member.id));
+            //const membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers').whereIn('member_id', this.members.map(member => member.id));
+
+            this.membersStripeCustomers = new Map();
+            for (const customer of membersStripeCustomers) {
+                this.membersStripeCustomers.set(customer.member_id, customer);
+            }
+
+            this.membersProducts = new Map();
+
+            for (const product of membersProducts) {
+                this.membersProducts.set(product.member_id, product);
+            }
+
+            await this.importForEach(this.members, 1.2);
+
+            offset += limit;
+        }
+    }
+
+    setReferencedModel(model) {
+        this.model = model;
+        this.count = 0;
+        this.lastSubscriptionStart = null;
+    }
+
+    generate() {
+        this.count += 1;
+
+        const member = this.model;
+        const customer = this.membersStripeCustomers.get(this.model.id);
+
+        if (!customer) {
+            // This is a requirement, so skip if we don't have a customer
+            return;
+        }
+
+        if (this.count > 1 && member.status !== 'paid') {
+            return;
+        }
+
+        const memberProduct = this.membersProducts.get(this.model.id);
+        let ghostProduct = memberProduct ? this.products.find(product => product.id === memberProduct.product_id) : null;
+
+        // Whether we should create a valid subscription or not
+        // We'll only create one valid subscription for each member if they are currently paid
+        let createValid = this.count === 1 && member.status === 'paid';
+
+        if (!ghostProduct) {
+            // Generate canceled, incomplete, incomplete_expired or unpaid subscriptions
+            // Choose a random paid product
+            ghostProduct = faker.helpers.arrayElement(this.products);
+            createValid = false;
+        }
+
+        const isMonthly = luck(70);
+        const stripeProduct = this.stripeProducts.find(product => product.product_id === ghostProduct.id);
+        const stripePrice = this.stripePrices.find((price) => {
+            return price.stripe_product_id === stripeProduct.stripe_product_id &&
+                (isMonthly ? price.interval === 'month' : price.interval === 'year');
+        });
+        const mrr = createValid ? (isMonthly ? stripePrice.amount : Math.floor(stripePrice.amount / 12)) : 0;
+
+        const referenceEndDate = this.lastSubscriptionStart ?? new Date();
+
+        if (!createValid) {
+            if (isMonthly) {
+                referenceEndDate.setMonth(referenceEndDate.getMonth() - 1);
+            } else {
+                referenceEndDate.setFullYear(referenceEndDate.getFullYear() - 1);
+            }
+        }
+
+        if (referenceEndDate < member.created_at) {
+            // Not possible to create an invalid subscription here
+            return;
+        }
+
+        const [startDate] = generateEvents({
+            total: 1,
+            trend: 'negative',
+            startTime: new Date(member.created_at),
+            endTime: referenceEndDate,
+            shape: 'ease-out'
+        });
+        this.lastSubscriptionStart = startDate;
+        const endDate = new Date(startDate);
+
+        if (createValid) {
+            // End date should be in the future
+
+            if (isMonthly) {
+                endDate.setFullYear(new Date().getFullYear());
+                endDate.setMonth(new Date().getMonth());
+                if (endDate < new Date()) {
+                    endDate.setMonth(endDate.getMonth() + 1);
+                }
+            } else {
+                endDate.setFullYear(new Date().getFullYear());
+                if (endDate < new Date()) {
+                    endDate.setFullYear(endDate.getFullYear() + 1);
+                }
+            }
+        } else {
+            // End date should be in the past
+            if (isMonthly) {
+                // What is the month difference between startDate and now? Pick a random number in between
+                const monthDiff = (new Date().getFullYear() - startDate.getFullYear()) * 12 + (new Date().getMonth() - startDate.getMonth());
+                if (monthDiff === 0) {
+                    // Not possible to create an invalid subscription here
+                    return;
+                }
+
+                const randomMonthDiff = faker.datatype.number({min: 1, max: monthDiff});
+                endDate.setMonth(startDate.getMonth() + randomMonthDiff);
+            } else {
+                // What is the year difference between startDate and now? Pick a random number in between
+                const yearDiff = new Date().getFullYear() - startDate.getFullYear();
+
+                if (yearDiff === 0) {
+                    // Not possible to create an invalid subscription here
+                    return;
+                }
+                const randomYearDiff = faker.datatype.number({min: 1, max: yearDiff});
+
+                endDate.setFullYear(startDate.getFullYear() + randomYearDiff);
+            }
+        }
+
+        // Simulate some different statusses here:
+        // - active, not ending (cancel_at_period_end  = false)
+        // - active, ending (cancel_at_period_end = true)
+        // - canceled -> current_period_end can be in both past or present, cancel_at_period_end can be both true or false
+        // - incomplete_expired -> user tried to pay but 3D secure expired
+        // - incomplete -> waiting on 3D secure
+        // - trialing -> need to set trial_end_at to a date in the future
+        // - past_due -> last paymet failed, but subscription still active until tried a couple of times
+        // - unpaid -> all payment attempts failed - but keep the subscription active (special setting in Stripe)
+
+        const validStatusses = new Array(10).fill({
+            status: 'active',
+            cancel_at_period_end: false
+        });
+
+        // Trialing only possible when the startDate > 1 month ago
+        const monthAgo = new Date();
+
+        if (!isMonthly) {
+            // Year ago
+            monthAgo.setFullYear(monthAgo.getFullYear() - 1);
+        } else {
+            // Month ago
+            monthAgo.setMonth(monthAgo.getMonth() - 1);
+        }
+
+        if (startDate > monthAgo) {
+            validStatusses.push({
+                status: 'trialing',
+                cancel_at_period_end: false,
+                trial_end_at: dateToDatabaseString(endDate),
+                trial_start_at: dateToDatabaseString(startDate)
+            });
+        }
+
+        // Past due only possible if startDate < 1 month ago
+        if (startDate < monthAgo) {
+            validStatusses.push({
+                status: 'past_due',
+                cancel_at_period_end: false
+            });
+            validStatusses.push({
+                status: 'unpaid',
+                cancel_at_period_end: false
+            });
+        }
+
+        const invalidStatusses = [
+            {
+                status: 'canceled',
+                cancel_at_period_end: true
+            },
+            {
+                status: 'canceled',
+                cancel_at_period_end: false
+            },
+            {
+                status: 'incomplete_expired',
+                cancel_at_period_end: false
+            },
+            {
+                status: 'incomplete',
+                cancel_at_period_end: false
+            }
+        ];
+
+        const status = createValid ? faker.helpers.arrayElement(validStatusses) : faker.helpers.arrayElement(invalidStatusses);
+
+        return {
+            id: this.fastFakeObjectId(),
+            customer_id: customer.customer_id,
+            subscription_id: `sub_${faker.random.alphaNumeric(14)}`,
+            stripe_price_id: stripePrice.stripe_price_id,
+            start_date: dateToDatabaseString(startDate),
+            created_at: dateToDatabaseString(startDate),
+            created_by: 'unused',
+            mrr,
+            plan_id: stripeProduct.stripe_product_id,
+            plan_nickname: `${ghostProduct.name} - ${stripePrice.nickname}`,
+            plan_interval: stripePrice.interval,
+            plan_amount: stripePrice.amount,
+            plan_currency: stripePrice.currency,
+
+            // Defaults
+            status: 'active',
+            cancel_at_period_end: false,
+            current_period_end: dateToDatabaseString(endDate),
+
+            // Override
+            ...status
+        };
+    }
+}
+
+module.exports = MembersStripeCustomersSubscriptionsImporter;