make-mp-data 2.1.6 → 3.0.1

package/dungeons/kurby.js

@@ -0,0 +1,211 @@
+
+const SEED = "kurby-retention";
+import dayjs from 'dayjs';
+import utc from 'dayjs/plugin/utc.js';
+dayjs.extend(utc);
+import * as u from "../lib/utils/utils.js";
+import * as v from 'ak-tools';
+const chance = u.initChance(SEED);
+
+const num_users = 100;
+const days = 360;
+
+/** @typedef {import("../types.js").Dungeon} Config */
+
+// Churn rates by favorite color per behavior type (percentage likelihood)
+const CHURN_RATES = {
+	red:    { hourly: 15, daily: 40, weekly: 15, monthly: 10 },
+	blue:   { hourly: 15, daily: 15, weekly: 40, monthly: 10 },
+	green:  { hourly: 5,  daily: 5,  weekly: 5,  monthly: 5 },
+	yellow: { hourly: 15, daily: 15, weekly: 15, monthly: 40 },
+	purple: { hourly: 40, daily: 15, weekly: 15, monthly: 10 },
+};
+
+// Behavior cadence definitions: [intervalAmount, intervalUnit, jitterMax, jitterUnit]
+const BEHAVIORS = {
+	hourly:  { interval: 1,  unit: 'hour',  jitterMax: 10, jitterUnit: 'minute' },
+	daily:   { interval: 1,  unit: 'day',   jitterMax: 2,  jitterUnit: 'hour' },
+	weekly:  { interval: 7,  unit: 'day',   jitterMax: 12, jitterUnit: 'hour' },
+	monthly: { interval: 30, unit: 'day',   jitterMax: 2,  jitterUnit: 'day' },
+};
+
+function generateCadencedEvents(behaviorName, cadence, startTime, endTime, churnTime, userId) {
+	const events = [];
+	let cursor = dayjs(startTime);
+	const end = dayjs(endTime);
+
+	while (cursor.isBefore(end)) {
+		if (churnTime && cursor.isAfter(churnTime)) break;
+
+		const jitter = chance.integer({ min: -cadence.jitterMax, max: cadence.jitterMax });
+		const eventTime = cursor.add(jitter, cadence.jitterUnit);
+
+		if (eventTime.isBefore(end) && eventTime.isAfter(startTime)) {
+			events.push({
+				event: `${behaviorName} behavior`,
+				time: eventTime.toISOString(),
+				user_id: userId,
+				insert_id: v.uid(12),
+			});
+		}
+
+		cursor = cursor.add(cadence.interval, cadence.unit);
+	}
+
+	// Insert churn event if user churns on this behavior
+	if (churnTime && dayjs(churnTime).isBefore(end)) {
+		events.push({
+			event: `churn - ${behaviorName} behavior`,
+			time: dayjs(churnTime).toISOString(),
+			user_id: userId,
+			insert_id: v.uid(12),
+		});
+	}
+
+	return events;
+}
+
+/** @type {Config} */
+const config = {
+	token: "ed63e7c6227b1990d2b1b847d7d8e0d9",
+	seed: "lets go",
+	numDays: days,
+	numEvents: num_users * 50,
+	numUsers: num_users,
+	hasAnonIds: false,
+	hasSessionIds: false,
+	format: "json",
+	alsoInferFunnels: false,
+	hasLocation: false,
+	hasAndroidDevices: false,
+	hasIOSDevices: false,
+	hasDesktopDevices: false,
+	hasBrowser: false,
+	hasCampaigns: false,
+	isAnonymous: false,
+	hasAdSpend: false,
+	hasAvatar: false,
+	makeChart: false,
+	batchSize: 5_500_000,
+	concurrency: 10,
+	writeToDisk: false,
+	percentUsersBornInDataset: 100,
+
+	funnels: [],
+
+	events: [
+		{
+			event: "first behavior",
+			weight: 1,
+			isFirstEvent: true,
+			properties: {},
+		},
+		{
+			event: "hourly behavior",
+			weight: 40,
+			properties: {},
+		},
+		{
+			event: "daily behavior",
+			weight: 30,
+			properties: {},
+		},
+		{
+			event: "weekly behavior",
+			weight: 15,
+			properties: {},
+		},
+		{
+			event: "monthly behavior",
+			weight: 10,
+			properties: {},
+		},
+		{
+			event: "placeholder",
+			weight: 50,
+			properties: {},
+		},
+	],
+
+	superProps: {},
+
+	userProps: {
+		"favorite color": ["red", "blue", "green", "yellow", "purple"],
+	},
+
+	scdProps: {},
+	mirrorProps: {},
+	groupKeys: [],
+	groupProps: {},
+	lookupTables: [],
+
+	hook: function (record, type, meta) {
+		if (type === "everything") {
+			if (!record.length) return record;
+
+			const userId = record[0].user_id || record[0].device_id;
+			if (!userId) return record;
+
+			const favoriteColor = meta.profile?.["favorite color"] || "green";
+			const colorChurnRates = CHURN_RATES[favoriteColor] || CHURN_RATES.green;
+
+			// Find the time range from the raw events
+			const times = record.map(e => new Date(e.time).getTime()).filter(t => !isNaN(t));
+			if (!times.length) return record;
+			const startTime = dayjs(Math.min(...times));
+			const endTime = dayjs(Math.max(...times));
+			const totalSpanHours = endTime.diff(startTime, 'hour');
+
+			if (totalSpanHours < 1) return record;
+
+			// Keep the "first behavior" event from the original stream
+			const firstBehavior = record.find(e => e.event === "first behavior");
+			const newEvents = [];
+
+			if (firstBehavior) {
+				newEvents.push(firstBehavior);
+			} else {
+				// Create a first behavior event at the start
+				newEvents.push({
+					event: "first behavior",
+					time: startTime.toISOString(),
+					user_id: userId,
+					insert_id: v.uid(12),
+				});
+			}
+
+			// For each behavior type, determine churn and generate cadenced events
+			for (const [behaviorKey, cadence] of Object.entries(BEHAVIORS)) {
+				const churnRate = colorChurnRates[behaviorKey];
+				let churnTime = null;
+
+				if (chance.bool({ likelihood: churnRate })) {
+					// Pick churn time between 20% and 80% into the active period
+					const churnOffsetHours = chance.integer({
+						min: Math.floor(totalSpanHours * 0.2),
+						max: Math.floor(totalSpanHours * 0.8),
+					});
+					churnTime = startTime.add(churnOffsetHours, 'hour');
+				}
+
+				const behaviorEvents = generateCadencedEvents(
+					behaviorKey,
+					cadence,
+					startTime,
+					endTime,
+					churnTime,
+					userId
+				);
+				newEvents.push(...behaviorEvents);
+			}
+
+			// Sort all events by time
+			newEvents.sort((a, b) => new Date(a.time) - new Date(b.time));
+			return newEvents;
+		}
+
+		return record;
+	},
+};
+
+export default config;

package/dungeons/media.js

@@ -4,7 +4,7 @@ import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc.js';
 dayjs.extend(utc);
 import 'dotenv/config';
-import * as u from '../lib/utils/utils.js';
+import * as u from "../lib/utils/utils.js";
 import * as v from 'ak-tools';
 const chance = u.initChance(SEED);
 const num_users = 10_000;
@@ -25,14 +25,14 @@ const channelIds = genIds(100);
 
 /** @type {Config} */
 const config = {
-	token: "",
+	// token: process.env.MASTER_PROJECT_TOKEN || "",
 	seed: `LFG!`, //,
 	numDays: days,
 	numEvents: num_users * 63,
 	numUsers: num_users,
 	hasAnonIds: false,
 	hasSessionIds: false,
-	format: "json",
+	format: "csv",
 	alsoInferFunnels: false,
 	hasLocation: false,
 	hasAndroidDevices: true,
@@ -44,8 +44,9 @@ const config = {
 	hasAdSpend: false,
 
 	hasAvatar: false,
+	makeChart: false,
 
-	batchSize: 1_500_000,
+	batchSize: 2_500_000,
 	concurrency: 10,
 	writeToDisk: true,
 

package/dungeons/mil.js

@@ -4,7 +4,7 @@ import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc.js';
 dayjs.extend(utc);
 import 'dotenv/config';
-import * as u from '../lib/utils/utils.js';
+import * as u from "../lib/utils/utils.js";
 import * as v from 'ak-tools';
 const chance = u.initChance(SEED);
 const num_users = 10_000;
@@ -25,7 +25,7 @@ const channelIds = genIds(100);
 
 /** @type {Config} */
 const config = {
-	token: "",
+	token: process.env.MASTER_PROJECT_TOKEN || "",
 	seed: `LFG!`, //,
 	numDays: days,
 	numEvents: num_users * 62,
@@ -44,8 +44,9 @@ const config = {
 	hasAdSpend: false,
 
 	hasAvatar: false,
+	makeChart: false,
 
-	batchSize: 1_500_000,
+	batchSize: 2_500_000,
 	concurrency: 10,
 	writeToDisk: true,
 

package/dungeons/mirror.js

@@ -14,12 +14,12 @@ import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 dayjs.extend(utc);
 import { uid, comma } from 'ak-tools';
-import { pickAWinner, weighNumRange, date, integer } from '../lib/utils/utils.js';
+import { pickAWinner, weighNumRange, date, integer } from "../lib/utils/utils.js";
 
 
 /** @type {import('../types').Dungeon} */
 const config = {
-	token: "",
+	token: process.env.MASTER_PROJECT_TOKEN || "",
 	seed: "mirror me",
 	numDays: 30, //how many days worth of data
 	numEvents: 10000, //how many events

package/dungeons/money2020-ed.js

@@ -5,17 +5,17 @@ import utc from "dayjs/plugin/utc.js";
 import "dotenv/config";
 import { weighNumRange, range, date, initChance, exhaust, choose, integer, decimal } from "../lib/utils/utils.js";
 
-const SEED = "thfdgdfgdfgank sup hello 2020 money!!!";
+const SEED = "thank sup hello 2020 money!!!";
 dayjs.extend(utc);
 const chance = initChance(SEED);
-const num_users = 4_200;
-const days = 35;
+const num_users = 10;
+const days = 100;
 
 /** @typedef  {import("../types.js").Dungeon} Dungeon */
 
 /** @type {Dungeon} */
 const dungeon = {
-	token: "",
+	token: process.env.MASTER_PROJECT_TOKEN || "",
 	seed: SEED,
 	numDays: days,
 	numEvents: num_users * 120,
@@ -31,11 +31,12 @@ const dungeon = {
 	hasBrowser: true,
 	hasCampaigns: true,
 	isAnonymous: false,
-	hasAdSpend: false,
-	percentUsersBornInDataset: 40,
+	hasAdSpend: true,
+
 	hasAvatar: true,
+	makeChart: false,
 
-	batchSize: 1_500_000,
+	batchSize: 2_500_000,
 	concurrency: 1,
 	writeToDisk: false,
 	superProps: {

package/dungeons/sanity.js

@@ -13,12 +13,12 @@ const chance = new Chance();
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc.js';
 dayjs.extend(utc);
-import { weighNumRange, integer } from '../lib/utils/utils.js';
+import { weighNumRange, integer } from "../lib/utils/utils.js";
 
 
 /** @type {import('../types.js').Dungeon} */
 const config = {
-	token: "",
+	token: process.env.MASTER_PROJECT_TOKEN || "",
 	seed: "foo bar",
 	numDays: 90, //how many days worth of data
 	numEvents: 50_000, //how many events
@@ -28,6 +28,7 @@ const config = {
 	hasAnonIds: false, //if true, anonymousIds are created for each user
 	hasSessionIds: false, //if true, hasSessionIds are created for each user
 	alsoInferFunnels: true, //if true, infer funnels from events
+	makeChart: true,
 	writeToDisk: true,
 	concurrency: 25,
 	funnels: [

package/dungeons/scd.js

@@ -14,7 +14,7 @@ import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 dayjs.extend(utc);
 import { uid, comma } from 'ak-tools';
-import { pickAWinner, weighNumRange, date, integer, weighChoices } from '../lib/utils/utils.js';
+import { pickAWinner, weighNumRange, date, integer, weighChoices } from "../lib/utils/utils.js";
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 
@@ -22,7 +22,7 @@ const videoCategories = ["funny", "educational", "inspirational", "music", "news
 
 /** @type {import('../types').Dungeon} */
 const config = {
-	token: "",
+	token: process.env.MASTER_PROJECT_TOKEN || "",
 	seed: "simple is best",
 	numDays: 30, //how many days worth1 of data
 	numEvents: 50000, //how many events
@@ -32,6 +32,7 @@ const config = {
 	hasAnonIds: false, //if true, anonymousIds are created for each user
 	hasSessionIds: false, //if true, hasSessionIds are created for each user
 	hasAdSpend: false,
+	makeChart: false,
 	hasLocation: true,
 	hasAndroidDevices: true,
 	hasIOSDevices: true,

package/dungeons/simple.js

@@ -14,7 +14,7 @@ import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 dayjs.extend(utc);
 import { uid, comma } from 'ak-tools';
-import { pickAWinner, weighNumRange, date, integer, weighChoices } from '../lib/utils/utils.js';
+import { pickAWinner, weighNumRange, date, integer, weighChoices } from "../lib/utils/utils.js";
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 
@@ -22,29 +22,44 @@ const videoCategories = ["funny", "educational", "inspirational", "music", "news
 
 /** @type {import('../types').Dungeon} */
 const config = {
-	token: "",
-	seed: "simple is best!",
+	token: process.env.MASTER_PROJECT_TOKEN || "",
+	seed: "simple is best",
 	numDays: 100, //how many days worth1 of data
-	numEvents: 250_000, //how many events
-	numUsers: 1_000, //how many users
-	format: 'csv', //csv or json
+	numEvents: 1_000_000, //how many events
+	numUsers: 10_000, //how many users
+	format: 'json', //csv or json
 	region: "US",
-	hasAnonIds: false, //if true, anonymousIds are created for each user
-	hasSessionIds: false, //if true, hasSessionIds are created for each user
+	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasSessionIds: true, //if true, hasSessionIds are created for each user
 	hasAdSpend: true,
+	makeChart: false,
 	hasLocation: true,
-	hasAndroidDevices: false,
-	hasIOSDevices: false,
-	hasDesktopDevices: false,
-	hasBrowser: false,
+	hasAndroidDevices: true,
+	hasIOSDevices: true,
+	hasDesktopDevices: true,
+	hasBrowser: true,
 	hasCampaigns: true,
 	isAnonymous: false,
 	alsoInferFunnels: true,
-	concurrency: 1,
+	concurrency: 10,
 	batchSize: 2_500_000,
 
 
 	events: [
+		{
+			event: "ad impression",
+			weight: 15,
+			properties: {
+				partner: pickAWinner(["google", "facebook", "twitter", "linkedin", "bing", "taboola", "outbrain", "quora", "pinterest"]),
+			}
+		},
+		{
+			event: "ad click",
+			weight: 10,
+			properties: {
+				partner: pickAWinner(["google", "facebook", "twitter", "linkedin", "bing", "taboola", "outbrain", "quora", "pinterest"]),
+			}
+		},
 		{
 			event: "checkout",
 			weight: 2,
@@ -184,7 +199,7 @@ const config = {
 
 		if (type === "everything") {
 
-			//custom themers purchase more:
+			//custom themes purchase more:
 			const numCustomMode = record.filter(a => a.theme === 'custom').length;
 			const numLightMode = record.filter(a => a.theme === 'light').length;
 			const numDarkMode = record.filter(a => a.theme === 'dark').length;

package/dungeons/strict-event-test.js

@@ -0,0 +1,30 @@
+export default {
+	name: "strict-event-count-test",
+	seed: "test-123",
+
+	// We want exactly 1000 events
+	numEvents: 1000,
+
+	// Even though we specify 500 users, we should stop early when we hit 1000 events
+	numUsers: 500,
+
+	numDays: 7,
+
+	// Enable strict event count - this will bail out at exactly numEvents
+	strictEventCount: true,
+
+	verbose: true,
+	writeToDisk: true,
+	format: 'json',
+
+	events: [
+		{ event: "page_view", weight: 5 },
+		{ event: "click", weight: 3 },
+		{ event: "purchase", weight: 1 }
+	],
+
+	userProps: {
+		plan: ["free", "pro", "enterprise"],
+		region: ["US", "EU", "APAC"]
+	}
+};

package/dungeons/student-teacher.js

@@ -15,7 +15,7 @@ const days = 100;
 
 /** @type {Config} */
 const config = {
-	token: "",
+	token: process.env.MASTER_PROJECT_TOKEN || "",
 	seed: SEED,
 	numDays: days,
 	numEvents: num_users * 100,
@@ -34,8 +34,9 @@ const config = {
 	hasAdSpend: true,
 
 	hasAvatar: true,
+	makeChart: false,
 
-	batchSize: 1_500_000,
+	batchSize: 2_500_000,
 	concurrency: 1,
 	writeToDisk: false,