make-mp-data 3.0.1 → 3.0.2

package/dungeons/mil.js

@@ -25,7 +25,7 @@ const channelIds = genIds(100);
 
 /** @type {Config} */
 const config = {
-	token: process.env.MASTER_PROJECT_TOKEN || "",
+	token: "",
 	seed: `LFG!`, //,
 	numDays: days,
 	numEvents: num_users * 62,
@@ -48,7 +48,7 @@ const config = {
 
 	batchSize: 2_500_000,
 	concurrency: 10,
-	writeToDisk: true,
+	writeToDisk: false,
 
 	funnels: [],
 
@@ -290,6 +290,33 @@ const config = {
 	groupProps: {},
 	lookupTables: [],
 	hook: function (record, type, meta) {
+		// event hook: tag events by frequency tier and enrich booleans
+		if (type === "event") {
+			const highFreq = ["foo", "bar", "baz"];
+			const lowFreq = ["crumn", "yak"];
+			if (highFreq.includes(record.event)) {
+				record.frequency_tier = "high";
+			} else if (lowFreq.includes(record.event)) {
+				record.frequency_tier = "low";
+			} else {
+				record.frequency_tier = "medium";
+			}
+			// boolean super prop drives a derived field
+			if (record.boolean === true) {
+				record.flag_status = "active";
+			}
+		}
+
+		// everything hook: users with very few events are marked as churned
+		if (type === "everything") {
+			if (record.length <= 3) {
+				for (const e of record) {
+					e.is_churned_user = true;
+				}
+			}
+			return record;
+		}
+
 		return record;
 	}
 };

package/dungeons/mirror.js

@@ -19,7 +19,7 @@ import { pickAWinner, weighNumRange, date, integer } from "../lib/utils/utils.js
 
 /** @type {import('../types').Dungeon} */
 const config = {
-	token: process.env.MASTER_PROJECT_TOKEN || "",
+	token: "",
 	seed: "mirror me",
 	numDays: 30, //how many days worth of data
 	numEvents: 10000, //how many events
@@ -120,6 +120,38 @@ const config = {
 	groupProps: {},
 	lookupTables: [],
 	hook: function (record, type, meta) {
+		// --- user hook: segment users by spirit animal ---
+		if (type === "user") {
+			const aquatic = ["whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "seahorse", "crab", "lobster", "shrimp"];
+			record.segment = aquatic.includes(record.spiritAnimal) ? "aquatic" : "terrestrial";
+			return record;
+		}
+
+		// --- event hook: color-based property boost ---
+		if (type === "event") {
+			// warm-colored events get a higher updateMe value
+			const warmColors = ["red", "orange", "yellow"];
+			if (warmColors.includes(record.color)) {
+				record.updateMe = (record.updateMe || 5) + 10;
+			}
+			return record;
+		}
+
+		// --- everything hook: inject a "milestone" event for users with 8+ events ---
+		if (type === "everything") {
+			if (record.length >= 8) {
+				const lastEvent = record[record.length - 1];
+				record.push({
+					event: "milestone_reached",
+					time: lastEvent.time,
+					user_id: lastEvent.user_id,
+					color: lastEvent.color,
+					milestone: record.length
+				});
+			}
+			return record;
+		}
+
 		return record;
 	}
 };

package/dungeons/{kurby.js → retention-cadence.js}

@@ -67,7 +67,7 @@ function generateCadencedEvents(behaviorName, cadence, startTime, endTime, churn
 
 /** @type {Config} */
 const config = {
-	token: "ed63e7c6227b1990d2b1b847d7d8e0d9",
+	token: "",
 	seed: "lets go",
 	numDays: days,
 	numEvents: num_users * 50,

package/dungeons/{harness/harness-gaming.js → rpg.js}

@@ -1,7 +1,7 @@
 import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 import "dotenv/config";
-import * as u from "../../lib/utils/utils.js";
+import * as u from "../lib/utils/utils.js";
 import * as v from "ak-tools";
 
 const SEED = "harness-gaming";
@@ -110,7 +110,7 @@ const itemIds = v.range(1, 301).map(n => `item_${v.uid(7)}`);
 
 /** @type {Config} */
 const config = {
-	token: "7958c5407b34dbbeef4db2c9d1f032d4",
+	token: "",
 	seed: SEED,
 	numDays: days,
 	numEvents: num_users * 120,
@@ -585,8 +585,8 @@ const config = {
 				const eventTime = dayjs(event.time);
 				const daysSinceStart = firstEventTime ? eventTime.diff(firstEventTime, 'days', true) : 0;
 
-				// Hook #8: Capture subscription tier from first event
-				if (idx === 0 && event.subscription_tier) {
+				// Hook #8: Capture subscription tier from any event that has it
+				if (event.subscription_tier) {
 					subscriptionTier = event.subscription_tier;
 				}
 
@@ -1133,7 +1133,7 @@ export default config;
  *   import config from './dungeons/harness-gaming.js';
  *   const results = await generate(config);
  *
- * OUTPUT FILES (with writeToDisk: true, format: "parquet", gzip: true):
+ * OUTPUT FILES (with writeToDisk: false, format: "parquet", gzip: true):
  *
  *   - needle-in-haystack__events.parquet.gz - All event data
  *   - needle-in-haystack__user_profiles.parquet.gz - User profiles

package/dungeons/sanity.js

@@ -18,7 +18,7 @@ import { weighNumRange, integer } from "../lib/utils/utils.js";
 
 /** @type {import('../types.js').Dungeon} */
 const config = {
-	token: process.env.MASTER_PROJECT_TOKEN || "",
+	token: "",
 	seed: "foo bar",
 	numDays: 90, //how many days worth of data
 	numEvents: 50_000, //how many events
@@ -29,7 +29,7 @@ const config = {
 	hasSessionIds: false, //if true, hasSessionIds are created for each user
 	alsoInferFunnels: true, //if true, infer funnels from events
 	makeChart: true,
-	writeToDisk: true,
+	writeToDisk: false,
 	concurrency: 25,
 	funnels: [
 		{
@@ -104,6 +104,35 @@ const config = {
 		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
 	},
 	hook: function (record, type, meta) {
+		// --- user hook: tag power users based on luckyNumber ---
+		if (type === "user") {
+			record.userTier = record.luckyNumber > 300 ? "power" : "regular";
+			return record;
+		}
+
+		// --- event hook: add an engagement score based on event type ---
+		if (type === "event") {
+			const highEngagement = ["fonk", "crumn", "yak"];
+			const midEngagement = ["garply", "durtle", "linny"];
+			if (highEngagement.includes(record.event)) {
+				record.engagement_score = chance.integer({ min: 70, max: 100 });
+			} else if (midEngagement.includes(record.event)) {
+				record.engagement_score = chance.integer({ min: 30, max: 69 });
+			} else {
+				record.engagement_score = chance.integer({ min: 1, max: 29 });
+			}
+			return record;
+		}
+
+		// --- everything hook: low-activity users lose their last few events (churn simulation) ---
+		if (type === "everything") {
+			if (record.length > 3 && record.length < 8) {
+				// users with few events lose the tail end — simulates churn
+				return record.slice(0, Math.ceil(record.length * 0.6));
+			}
+			return record;
+		}
+
 		return record;
 	}
 };

package/dungeons/{harness/harness-sass.js → sass.js}

@@ -1,7 +1,7 @@
 import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 import "dotenv/config";
-import * as u from "../../lib/utils/utils.js";
+import * as u from "../lib/utils/utils.js";
 import * as v from "ak-tools";
 
 const SEED = "harness-sass";
@@ -78,7 +78,7 @@ const failedDeployUsers = new Map();
 
 /** @type {Config} */
 const config = {
-	token: "495b9a9bbfc06c0e65b98827e07a4b87",
+	token: "",
 	seed: SEED,
 	numDays: days,
 	numEvents: num_users * 120,

package/dungeons/scd.js

@@ -22,7 +22,7 @@ const videoCategories = ["funny", "educational", "inspirational", "music", "news
 
 /** @type {import('../types').Dungeon} */
 const config = {
-	token: process.env.MASTER_PROJECT_TOKEN || "",
+	token: "",
 	seed: "simple is best",
 	numDays: 30, //how many days worth1 of data
 	numEvents: 50000, //how many events
@@ -196,6 +196,51 @@ const config = {
 		}
 	},
 	hook: function (record, type, meta) {
+		// --- user hook: classify users into spending tiers ---
+		if (type === "user") {
+			record.spendTier = record.luckyNumber > 250 ? "high_spender" : "budget";
+			return record;
+		}
+
+		// --- event hook: coupon users get discounted checkout amounts ---
+		if (type === "event") {
+			if (record.event === "checkout" && record.coupon && record.coupon !== "none") {
+				const discountPct = parseInt(record.coupon) || 10;
+				record.amount = Math.round(record.amount * (1 - discountPct / 100));
+				record.discount_applied = true;
+			}
+			// weekend watchers get longer watch times
+			if (record.event === "watch video" && record.time) {
+				const day = dayjs(record.time).day();
+				if (day === 0 || day === 6) {
+					record.watchTimeSec = Math.round((record.watchTimeSec || 60) * 1.5);
+					record.is_weekend = true;
+				}
+			}
+			return record;
+		}
+
+		// --- everything hook: simulate cart abandonment ---
+		if (type === "everything") {
+			const hasAddToCart = record.some(e => e.event === "add to cart");
+			const hasCheckout = record.some(e => e.event === "checkout");
+			// users who added to cart but never checked out: remove checkout events (if any slipped through)
+			// and mark them as abandoned
+			if (hasAddToCart && !hasCheckout && record.length > 2) {
+				const lastAdd = record.filter(e => e.event === "add to cart").pop();
+				if (lastAdd) {
+					record.push({
+						event: "cart_abandoned",
+						time: dayjs(lastAdd.time).add(30, "minute").toISOString(),
+						user_id: lastAdd.user_id,
+						platform: lastAdd.platform,
+						amount: lastAdd.amount
+					});
+				}
+			}
+			return record;
+		}
+
 		return record;
 	}
 };

package/dungeons/simple.js

@@ -22,7 +22,7 @@ const videoCategories = ["funny", "educational", "inspirational", "music", "news
 
 /** @type {import('../types').Dungeon} */
 const config = {
-	token: process.env.MASTER_PROJECT_TOKEN || "",
+	token: "",
 	seed: "simple is best",
 	numDays: 100, //how many days worth1 of data
 	numEvents: 1_000_000, //how many events

package/dungeons/{harness/harness-social.js → social.js}

@@ -1,7 +1,7 @@
 import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 import "dotenv/config";
-import * as u from "../../lib/utils/utils.js";
+import * as u from "../lib/utils/utils.js";
 import * as v from "ak-tools";
 
 const SEED = "harness-social";
@@ -98,7 +98,7 @@ const postIds = v.range(1, 1001).map(n => `post_${v.uid(8)}`);
 
 /** @type {Config} */
 const config = {
-	token: "5ad61e3f75003dfe4faad0e9e5f41030",
+	token: "",
 	seed: SEED,
 	numDays: days,
 	numEvents: num_users * 120,

package/dungeons/streaming.js

@@ -0,0 +1,373 @@
+
+const SEED = "my-seed";
+import dayjs from 'dayjs';
+import utc from 'dayjs/plugin/utc.js';
+dayjs.extend(utc);
+import 'dotenv/config';
+import * as u from "../lib/utils/utils.js";
+import * as v from 'ak-tools';
+const chance = u.initChance(SEED);
+const num_users = 10_000;
+const days = 125;
+
+/** @typedef  {import("../types.js").Dungeon} Config */
+
+function genIds(numIds = 1000) {
+	const ids = [];
+	for (let i = 0; i < numIds; i++) {
+		ids.push(v.uid());
+	}
+	return ids;
+}
+
+const videoIds = genIds();
+const channelIds = genIds(100);
+
+/** @type {Config} */
+const config = {
+	// token: "",
+	seed: `LFG!`, //,
+	numDays: days,
+	numEvents: num_users * 63,
+	numUsers: num_users,
+	hasAnonIds: false,
+	hasSessionIds: false,
+	format: "csv",
+	alsoInferFunnels: false,
+	hasLocation: false,
+	hasAndroidDevices: true,
+	hasIOSDevices: true,
+	hasDesktopDevices: true,
+	hasBrowser: false,
+	hasCampaigns: false,
+	isAnonymous: false,
+	hasAdSpend: false,
+
+	hasAvatar: false,
+	makeChart: false,
+
+	batchSize: 2_500_000,
+	concurrency: 10,
+	writeToDisk: false,
+
+	funnels: [],
+
+	events: [
+		{
+			event: "watch video",
+			weight: 55,
+			properties: {
+				video_id: u.pickAWinner(videoIds),
+				"watch percent": u.pickAWinner([
+					25,
+					50,
+					75,
+					100,
+				]),
+				"watch time": u.weighNumRange(1, 65, .89, 100),
+
+				"category": u.pickAWinner([
+					"comedy",
+					"educational",
+					"music",
+					"sports",
+					"news",
+					"gaming",
+					"travel",
+				]),
+				quality: u.pickAWinner([
+					"240p",
+					"360p",
+					"480p",
+					"720p",
+					"1080p",
+					"4k",
+				], 4),
+				autoplay: [
+					true,
+					false,
+				],
+				fullscreen: [
+					true,
+					false,
+				],
+				"ads?": [
+					true, true,
+					false,
+				],
+			},
+		},
+		{
+			event: "like",
+			weight: 10,
+			properties: {
+				video_id: u.pickAWinner(videoIds),
+			},
+		},
+		{
+			event: "comment",
+			weight: 5,
+			properties: {
+				video_id: u.pickAWinner(videoIds),
+				comment_length: [
+					"short",
+					"medium",
+					"long",
+				],
+			},
+		},
+		{
+			event: "share",
+			weight: 3,
+			properties: {
+				video_id: u.pickAWinner(videoIds),
+				"share network": u.pickAWinner([
+					"facebook",
+					"twitter",
+					"reddit",
+					"email",
+					"whatsapp",
+				]),
+			},
+		},
+		{
+			event: "search",
+			weight: 25,
+			properties: {
+				search_term: [
+					"cats",
+					"dogs",
+					"tutorial",
+					"news",
+					"music",
+				],
+				"results count": u.pickAWinner([
+					0,
+					1,
+					2,
+					3,
+					4,
+					5,
+					6, 7, 8, 9, 10
+				], 5),
+				"search category": [
+					"all",
+					"channels",
+					"playlists",
+				],
+			},
+		},
+		{
+			event: "subscribe",
+			weight: 7,
+			properties: {
+				channel_id: u.pickAWinner(genIds()),
+			},
+		},
+		{
+			event: "unsubscribe",
+			weight: 2,
+			properties: {
+				channel_id: u.pickAWinner(genIds()),
+			},
+		},
+		{
+			event: "create playlist",
+			weight: 4,
+			properties: {
+				"play list name": u.pickAWinner([
+					"favorites",
+					"watch later",
+					"my music",
+					"funny videos",
+					"educational",
+				]),
+				privacy: u.pickAWinner([
+					"public",
+					"private",
+					"unlisted",
+				]),
+			},
+		},
+		{
+			event: "account signup",
+			weight: 1,
+			isFirstEvent: true,
+			properties: {
+				"sign up method": [
+					"email",
+					"google",
+					"facebook",
+				],
+			},
+		},
+		{
+			event: "account login",
+			weight: 9,
+			properties: {
+				"log in method": u.pickAWinner([
+					"email",
+					"google",
+					"facebook",
+				]),
+				success: [
+					true,
+					false,
+				],
+				error_message: [
+					"incorrect password",
+					"user not found",
+					"account locked",
+				],
+			},
+		},
+		{
+			event: "$experiment_started",
+			weight: 5,
+			isSessionStartEvent: true,
+			properties: {
+				"$experiment_type": "ak_ad_hoc",
+				"Experiment name": "show results on empty search",
+				"Variant name": ["feature enabled", "feature disabled"],
+			}
+		}
+	],
+	superProps: {
+		platform: u.pickAWinner([
+			"web",
+			"ios",
+			"android",
+		]),
+		network_type: [
+			"wifi",
+			"cellular",
+		],
+	},
+	userProps: {
+		subscription_status: [
+			"free",
+			"free",
+			"premium",
+		],
+		age_range: [
+			"13-17",
+			"18-24",
+			"25-34",
+			"35-44",
+			"45-54",
+			"55+",
+		],
+		preferred_genre: u.pickAWinner([
+			"comedy",
+			"action",
+			"drama",
+			"sci-fi",
+			"horror",
+		]),
+		upload_count: [
+			0,
+			1,
+			5,
+			10,
+			20,
+		],
+		following_count: [
+			0,
+			10,
+			50,
+			100,
+			500,
+		],
+		dark_mode_enabled: [
+			true,
+			false,
+		],
+	},
+
+	scdProps: {},
+	mirrorProps: {},
+	groupKeys: [],
+	groupProps: {},
+	lookupTables: [],
+	hook: function (record, type, meta) {
+		const NOW = dayjs();
+		const TIME_WHEN_SEARCH_GOT_BAD = NOW.subtract(21, 'days');
+		const TIME_WE_EXPERIMENTED = NOW.subtract(14, 'days');
+
+		if (type === "event") {
+			const EVENT_TIME = dayjs(record.time);
+			//when search got bad, people started searching less
+			//and got fewer results
+			if (EVENT_TIME.isAfter(TIME_WHEN_SEARCH_GOT_BAD)) {
+				if (chance.bool({ likelihood: 50 })) {
+					if (record.event === "search") {
+						record["results count"] = 0;
+					}
+				}
+
+				if (chance.bool({ likelihood: 18 })) {
+					record._drop = true;
+				}
+			}
+
+			if (EVENT_TIME.isBefore(TIME_WE_EXPERIMENTED)) {
+				if (record.event === "$experiment_started") {
+					record._drop = true;
+				}
+			}
+		}
+
+
+
+		if (type === "everything") {
+			// Filter out events tagged for dropping by the event hook
+			record = record.filter(e => !e._drop);
+
+			const hadFeatureEnabled = record.some(event =>
+				event.event === "$experiment_started" &&
+				event["Variant name"] === "feature enabled"
+			);
+
+			const hadFeatureDisabled = record.some(event =>
+				event.event === "$experiment_started" &&
+				event["Variant name"] === "feature disabled"
+			);
+
+			record.forEach((event, idx) => {
+				const EVENT_TIME = dayjs(event.time);
+
+				if (EVENT_TIME.isAfter(TIME_WE_EXPERIMENTED)) {
+					if (hadFeatureEnabled) {
+						// Users with feature enabled variant have a higher likelihood of subscribing.
+						// Add an extra subscribe event 50% of the time immediately after watching a video.
+						if (event.event === "watch video" && chance.bool({ likelihood: 75 })) {
+							// watch time goes up 
+							event["watch time"] = v.round(event["watch time"] * 1.7);
+							const subscribeEvent = {
+								event: "subscribe",
+								time: dayjs(event.time).add(1, 'minute').toISOString(),	
+								user_id: event.user_id,						
+							};
+							record.splice(idx + 1, 0, subscribeEvent);
+						}
+					} else if (hadFeatureDisabled) {
+						// Users with feature disabled variant have lower likelihood of subscribing.
+						// Drop subscribe events 50% of the time.						
+						if (event.event === "subscribe" && chance.bool({ likelihood: 75 })) {
+							record.splice(idx, 1);
+						}
+
+						// watch time goes down
+						if (event.event === "watch video") {
+							event["watch time"] = v.round(event["watch time"] * 0.5);
+						}
+					}
+				}
+			});
+		}
+
+		return record;
+	}
+};
+
+export default config;

package/dungeons/strict-event-test.js

@@ -14,7 +14,7 @@ export default {
 	strictEventCount: true,
 
 	verbose: true,
-	writeToDisk: true,
+	writeToDisk: false,
 	format: 'json',
 
 	events: [