make-mp-data 3.0.2 → 3.0.4

package/types.d.ts

@@ -12,75 +12,123 @@ export type ValueValid = Primitives | ValueValid[] | (() => ValueValid);
  * main config object for the entire data generation
  */
 export interface Dungeon {
-    // constants
+    // ── Core Parameters ──
+    /** Mixpanel project token. If provided, data will be imported to Mixpanel after generation. */
     token?: string;
+    /** RNG seed for reproducible output. Same seed + concurrency=1 = identical data. */
     seed?: string;
+    /** Number of days the dataset spans (from "now" looking backward). Default: 30 */
     numDays?: number;
+    /** Explicit start of dataset window (unix seconds). Alternative to numDays. */
     epochStart?: number;
+    /** Explicit end of dataset window (unix seconds). Defaults to FIXED_NOW. */
     epochEnd?: number;
+    /** Target total number of events to generate across all users. */
     numEvents?: number;
+    /** Number of unique users to generate. */
     numUsers?: number;
+    /** Output format for files written to disk. */
     format?: "csv" | "json" | "parquet" | string;
+    /** Mixpanel data residency region. */
     region?: "US" | "EU";
+    /** User generation concurrency. Default: 1. Values > 1 break seed reproducibility and provide no performance benefit (CPU-bound). */
     concurrency?: number;
+    /** Number of records before auto-flushing to disk. Prevents OOM for large datasets. Default: 1,000,000 */
     batchSize?: number;
 
+    // ── Mixpanel Import Credentials (for SCD import) ──
     serviceAccount?: string;
     serviceSecret?: string;
     projectId?: string;
 
-    // ids
+    // ── Identifiers ──
+    /** Dataset name prefix for output files. Auto-generated if not set. */
     name?: string;
 
-    //switches
+    // ── Feature Switches ──
+    /** If true, users have no distinct_id (anonymous-only tracking). */
     isAnonymous?: boolean;
+    /** If true, user profiles include avatar URLs. */
     hasAvatar?: boolean;
+    /** If true, events include geo properties (city, region, country, lat/lng). */
     hasLocation?: boolean;
+    /** If true, events include UTM campaign properties. */
     hasCampaigns?: boolean;
+    /** If true, generates ad spend data (impressions, clicks, cost). */
     hasAdSpend?: boolean;
+    /** If true, device pool includes iOS devices. */
     hasIOSDevices?: boolean;
+    /** If true, device pool includes Android devices. */
     hasAndroidDevices?: boolean;
+    /** If true, device pool includes desktop devices. */
     hasDesktopDevices?: boolean;
+    /** If true, events include browser properties. */
     hasBrowser?: boolean;
+    /** If true (default), writes output files to ./data/. Can also be a directory path string. */
     writeToDisk?: boolean | string;
+    /** If true, gzip-compresses output files. */
     gzip?: boolean;
+    /** If true, prints progress to stdout during generation. */
     verbose?: boolean;
+    /** If true, users get anonymous device IDs in addition to distinct_id. */
     hasAnonIds?: boolean;
+    /** If true, users get session IDs attached to events. */
     hasSessionIds?: boolean;
+    /** If true, auto-generates funnels from the events array in addition to any explicit funnels. */
     alsoInferFunnels?: boolean;
-    makeChart?: boolean | string;
+    /** Restrict all location data to a single country (e.g., "US", "GB"). */
     singleCountry?: string;
+    /** If true, stops generation at exactly numEvents (forces concurrency=1). Without this, event count is approximate. */
     strictEventCount?: boolean;
+    /** Internal flag for UI-triggered jobs (affects SCD credential handling). */
     isUIJob?: boolean;
 
-    //models
-    events?: EventConfig[]; //| string[]; //can also be a array of strings
+    // ── Data Models ──
+    /** Event definitions: names, weights, properties, and behavioral flags. */
+    events?: EventConfig[];
+    /** Properties that appear on EVERY event (e.g., platform, app_version). */
     superProps?: Record<string, ValueValid>;
+    /** Funnel definitions: conversion sequences, rates, ordering strategies. */
     funnels?: Funnel[];
+    /** User profile properties set once per user. */
     userProps?: Record<string, ValueValid>;
+    /** Slowly Changing Dimension properties: time-series mutations of user/group attributes. */
     scdProps?: Record<string, SCDProp>;
+    /** Mirror dataset definitions: create transformed copies of event data. */
     mirrorProps?: Record<string, MirrorProps>;
-    groupKeys?: [string, number][] | [string, number, string[]][]; // [key, numGroups, [events]]
+    /** Group analytics keys. Format: [key, numGroups] or [key, numGroups, [associatedEvents]]. */
+    groupKeys?: [string, number][] | [string, number, string[]][];
+    /** Properties for each group key's entities. */
     groupProps?: Record<string, Record<string, ValueValid>>;
+    /** Group-level events (stub — not yet implemented). */
     groupEvents?: GroupEventConfig[];
+    /** Lookup table definitions for dimension tables. */
     lookupTables?: LookupTableSchema[];
+    /** TimeSoup configuration: controls the temporal distribution of events (peaks, deviation, mean). */
     soup?: soup;
+    /** Hook function called on every data point. The primary mechanism for engineering deliberate trends and patterns. */
     hook?: Hook<any>;
 
-    //allow anything to be on the config
+    /** Allow arbitrary additional properties on the config. */
     [key: string]: any;
 
-    //probabilities
+    // ── Distribution Controls ──
+    /** Percentage of users whose account creation falls within the dataset window (vs. pre-existing). Default: 15 */
     percentUsersBornInDataset?: number;
     /** Bias toward recent birth dates for users born in dataset (0 = uniform, 1 = heavily recent). Default: 0.3 */
     bornRecentBias?: number;
 }
 
 export type SCDProp = {
+    /** Entity type this SCD applies to. "user" for user profiles; use a group key (e.g., "company_id") for group SCDs. Default: "user" */
     type?: string | "user" | "company_id" | "team_id" | "department_id";
+    /** How often the property mutates. Default: "day" */
     frequency?: "day" | "week" | "month" | "year";
+    /** Array of possible values, or a function that returns values. */
     values: ValueValid;
+    /** "fixed" = mutations at clean boundaries (start of day/week/month/year). "fuzzy" = mutations at any time. Default: "fuzzy" */
     timing?: "fixed" | "fuzzy";
+    /** Maximum number of mutations per entity. Default: 10 */
     max?: number;
 };
 
@@ -88,13 +136,25 @@ export type SCDProp = {
  * the soup is a set of parameters that determine the distribution of events over time
  */
 type soup = {
+    /** Controls clustering tightness. Higher = tighter peaks. Default: 2 */
     deviation?: number;
+    /** Number of time clusters to distribute events across. Default: dynamic (numDays/7, minimum 5) */
     peaks?: number;
+    /** Offset for the normal distribution center within each peak. Default: 0 */
     mean?: number;
 };
 
 /**
- * the types of hooks that can be used
+ * Hook types and when they fire (in order per user):
+ * - "user"        — user profile object (mutate in-place, return ignored)
+ * - "scd-pre"     — array of SCD entries (mutate in-place OR return new array to replace)
+ * - "funnel-pre"  — funnel config object (mutate conversionRate, timeToConvert, etc. in-place)
+ * - "event"       — single event with FLAT properties (return value replaces event)
+ * - "funnel-post" — array of generated funnel events (mutate in-place, splice to inject)
+ * - "everything"  — array of ALL events for one user (return array to replace; meta.profile available)
+ *
+ * Storage-only hooks (fire during hookPush, not in generators):
+ * - "ad-spend", "group", "mirror", "lookup"
  */
 export type hookTypes =
     | "event"
@@ -113,7 +173,10 @@ export type hookTypes =
     | "";
 
 /**
- * a hook is a function that can be called before each entity is created, and can be used to modify attributes
+ * A hook function that receives every piece of data as it flows through the pipeline.
+ * @param record - The data being processed (event, profile, array of events, etc.)
+ * @param type - Which hook type is firing
+ * @param meta - Contextual metadata (varies by type; "everything" includes meta.profile and meta.scd)
  */
 export type Hook<T> = (record: any, type: hookTypes, meta: any) => T;
 
@@ -199,6 +262,7 @@ export interface Context {
     FIXED_NOW: number;
     FIXED_BEGIN?: number;
     TIME_SHIFT_SECONDS: number;
+    MAX_TIME: number;
 
     // State update methods
     incrementOperations(): void;
@@ -223,14 +287,24 @@ export interface Context {
  * how we define events and their properties
  */
 export interface EventConfig {
+    /** The event name (e.g., "page viewed", "purchase completed"). */
     event?: string;
+    /** Relative frequency weight (1-10). Higher = more likely to be selected. Used for both standalone event selection and funnel sequence building. Default: 1 */
     weight?: number;
+    /** Properties to attach to this event type. Values can be arrays (random pick), functions, or primitives. */
     properties?: Record<string, ValueValid>;
+    /** If true, this is the user's first-ever event (e.g., "sign up"). Used to create onboarding funnels. */
     isFirstEvent?: boolean;
+    /** If true, generating this event signals the user has churned. The user stops producing further events unless returnLikelihood allows them to come back. */
     isChurnEvent?: boolean;
+    /** Probability (0-1) that a churned user returns and continues generating events. 0 = permanent churn, 1 = always returns. Only used when isChurnEvent is true. Default: 0 */
+    returnLikelihood?: number;
+    /** If true, this event is automatically prepended 15 seconds before each funnel sequence (e.g., "$session_started"). */
     isSessionStartEvent?: boolean;
+    /** Internal: timing offset in milliseconds (set by funnel system, not user-configured). */
     relativeTimeMs?: number;
-	isStrictEvent?: boolean;
+    /** If true, this event is excluded from auto-generated funnels (inferFunnels and catch-all). Use for system events that shouldn't appear in conversion sequences. */
+    isStrictEvent?: boolean;
 }
 
 export interface GroupEventConfig extends EventConfig {
@@ -297,13 +371,6 @@ export interface Funnel {
         | "interrupted"
         | string;
 
-    /**
-     * todo: implement this
-     * if set, the funnel might be the last thing the user does
-     * ^ the numerical value is the likelihood that the user will churn
-     * todo: allow for users to be resurrected
-     */
-    isChurnFunnel?: void | number;
     /**
      * the likelihood that a user will convert (0-100%)
      */

package/lib/templates/verbose-schema.js

@@ -1,272 +0,0 @@
-/**
- * @fileoverview this is a highly verbose schema for a dungeon that shows all the options available
- * and how they might be implemented with extensive comments so an AI can understand it
- * it is not meant to be used as a template, but rather as a reference for how to create a dungeon
- * it is also used as a test for the AI to see if it can generate a dungeon with the same structure
- *
- * IMPORTANT: This file uses the NEW JSON format for function calls with "functionName" and "args"
- */
-
-
-
-import Chance from "chance";
-const chance = new Chance();
-import dayjs from "dayjs";
-import utc from "dayjs/plugin/utc.js";
-dayjs.extend(utc);
-import "dotenv/config";
-import { weighNumRange, range, date, initChance, exhaust, integer, decimal, odds } from "../utils/utils.js";
-const { NODE_ENV = "unknown" } = process.env;
-
-
-/** @type {import("../../types.js").Dungeon} */
-//SPLIT HERE
-const DUNGEON = {
-	/**
-	 * ⚠️ IMPORTANT NOTE ABOUT THIS EXAMPLE:
-	 * This is a comprehensive example showing ALL possible features.
-	 * Most dungeons will NOT need all these features, especially:
-	 * - Groups (groupKeys, groupProps) - ONLY for B2B/SaaS scenarios
-	 * - SCDs - ONLY when properties change over time
-	 *
-	 * Focus on the REQUIRED fields: events, funnels, superProps, userProps
-	 */
-
-	/**
-	 * events are the core building blocks of the dungeon
-	 * each event has a name, a weight, and properties
-	 * the weight determines how often the event occurs relative to other events
-	 * properties are the data associated with the event
-	 * they can be simple values or functions that return a value
-	 * we have a few built-in functions to help you generate data
-	 * you MUST create events for every dungeon
-	 *
-	 * IMPORTANT: All function calls use the new JSON format with functionName and args
-	 */
-	events: [
-		{
-			event: "checkout",
-			weight: 2,
-			properties: {
-				amount: { "functionName": "weighNumRange", "args": [5, 500, 0.25] }, // weighted random number in range
-				currency: ["USD", "CAD", "EUR", "BTC", "ETH", "JPY"],
-				coupon: ["none", "none", "none", "none", "10%OFF", "20%OFF", "10%OFF", "20%OFF", "30%OFF", "40%OFF", "50%OFF"],
-				numItems: { "functionName": "weighNumRange", "args": [1, 10] },
-			}
-		},
-		{
-			event: "add to cart",
-			weight: 4,
-			properties: {
-				amount: { "functionName": "weighNumRange", "args": [5, 500, 0.25] },
-				rating: { "functionName": "weighNumRange", "args": [1, 5] },
-				reviews: { "functionName": "weighNumRange", "args": [0, 35] },
-				isFeaturedItem: [true, false, false],
-				itemCategory: ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"],
-				dateItemListed: { "functionName": "date", "args": [30, true, "YYYY-MM-DD"] } // date in the last 30 days
-			}
-		},
-		{
-			event: "page view",
-			weight: 10,
-			properties: {
-				page: ["/", "/", "/help", "/account", "/watch", "/listen", "/product", "/people", "/peace"],
-				utm_source: ["$organic", "$organic", "$organic", "$organic", "google", "google", "google", "facebook", "facebook", "twitter", "linkedin"],
-			}
-		},
-		{
-			event: "watch video",
-			weight: 8,
-			properties: {
-				videoCategory: ["funny", "educational", "inspirational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"],
-				isFeaturedItem: [true, false, false],
-				watchTimeSec: { "functionName": "weighNumRange", "args": [10, 600, 0.25] },
-				quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
-				format: ["mp4", "avi", "mov", "mpg"],
-				uploader_id: { "functionName": "chance.guid", "args": [] } // Using chance.js library with dot notation
-			}
-		},
-		{
-			event: "view item",
-			weight: 8,
-			properties: {
-				isFeaturedItem: [true, false, false],
-				itemCategory: ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"],
-				dateItemListed: { "functionName": "date", "args": [30, true, "YYYY-MM-DD"] }
-			}
-		},
-		{
-			event: "save item",
-			weight: 5,
-			properties: {
-				isFeaturedItem: [true, false, false],
-				itemCategory: ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"],
-				dateItemListed: { "functionName": "date", "args": [30, true, "YYYY-MM-DD"] },
-			}
-		},
-		{
-			event: "sign up",
-			isFirstEvent: true,
-			weight: 1,
-			properties: {
-				wasReferred: [true, false, false, false],
-			}
-		},
-		{
-			event: "search",
-			weight: 6,
-			properties: {
-				query: { "functionName": "chance.word", "args": [] },
-				resultsFound: { "functionName": "weighNumRange", "args": [0, 100, 0.25] },
-			}
-		},
-		{
-			event: "remove from cart",
-			weight: 1,
-			properties: {
-				amount: { "functionName": "weighNumRange", "args": [5, 500, 0.25] },
-				reason: ["changed mind", "too expensive", "found better", "not needed"],
-			}
-		},
-		{
-			event: "share item",
-			weight: 2,
-			properties: {
-				medium: ["email", "facebook", "twitter", "whatsapp", "sms", "slack"],
-				recipient_count: { "functionName": "weighNumRange", "args": [1, 10] },
-			}
-		},
-		{
-			event: "session end",
-			weight: 1
-		}
-	],
-
-	/**
-	 * superProps are properties that are added to EVERY event
-	 * they can be simple values or functions that return a value
-	 * the random values will be chosen for each event
-	 * you MUST create superProps for every dungeon
-	 */
-	superProps: {
-		plan: ["free", "free", "free", "plus", "plus", "pro"], // arrays are randomly sampled from
-		// using arrow functions with the new format
-		region: { "functionName": "arrow", "body": "chance.pickone(['North America', 'Europe', 'Asia', 'South America', 'Africa', 'Oceania'])" }
-	},
-
-	/**
-	 * userProps are properties that are associated with a user
-	 * they can be simple values or functions that return a value
-	 * this is the $set property on the user profile
-	 * you MUST create userProps for every dungeon
-	 */
-	userProps: {
-		favoriteBrand: ["nike", "adidas", "puma", "reebok", "new balance", "asics", "vans", "converse"],
-		favoriteProduct: ["shoes", "clothing", "sports", "equipment"],
-		// Example using chance.integer with options
-		age: { "functionName": "chance.integer", "args": [{"min": 18, "max": 65}] },
-		isSubscribed: [true, false],
-		plan: ["free", "free", "free", "plus", "plus", "pro"],
-		signupDate: { "functionName": "date", "args": [365, true, "YYYY-MM-DD"] },
-		// Example using arrow function for complex expression
-		user_id: { "functionName": "arrow", "body": "`user_${chance.guid()}`" },
-		company: { "functionName": "chance.company", "args": [] },
-		email: { "functionName": "chance.email", "args": [] },
-		name: { "functionName": "chance.name", "args": [] }
-	},
-
-	/**
-	 * funnels are sequences of events that represent a user journey
-	 * each funnel has a sequence of events and a conversion rate
-	 * the conversion rate determines how many users complete the funnel
-	 * you can also add conditions to funnels to filter users based on properties
-	 * you MUST create funnels for every dungeon
-	 */
-	funnels: [
-		{
-			sequence: ["sign up", "page view", "view item", "add to cart", "checkout"],
-			conversionRate: 15,  // Integer 0-100 representing percentage
-		},
-		{
-			sequence: ["page view", "watch video"],
-			conversionRate: 65,
-		},
-		{
-			sequence: ["page view", "view item", "save item"],
-			conversionRate: 25,
-			conditions: { plan: "free" }  // Conditions must be an object, not a string expression
-		},
-		{
-			sequence: ["search", "view item", "add to cart"],
-			conversionRate: 35
-		},
-		{
-			sequence: ["page view", "view item", "share item"],
-			conversionRate: 10
-		}
-	],
-
-	/**
-	 * scdProps are Slowly Changing Dimensions properties
-	 * these are properties that change over time for users or groups
-	 * each SCD property has a type (user or group), frequency, values, timing, and max changes
-	 * OPTIONAL: only include if your use case involves properties that change over time
-	 */
-	scdProps: {
-		role: {
-			type: "user",
-			frequency: "month", // how often the property changes
-			values: ["admin", "user", "viewer", "editor"],
-			timing: "fixed", // fixed or fuzzy
-			max: 5 // maximum number of changes per entity
-		},
-		subscription_tier: {
-			type: "user",
-			frequency: "month", // Changed from "quarter" - valid options are: day, week, month, year
-			values: ["free", "basic", "premium", "enterprise"],
-			timing: "fuzzy", // Changed from "random" - valid options are: fixed, fuzzy
-			max: 3
-		}
-	},
-
-	/**
-	 * groupKeys define the types of groups in your data
-	 * Each group key is a tuple of [groupName, numberOfGroups]
-	 *
-	 * ⚠️ IMPORTANT: Groups are OPTIONAL and should ONLY be included when:
-	 * - The use case explicitly involves B2B relationships
-	 * - You have SaaS with company/team accounts
-	 * - There's a clear one-to-many relationship (one company → many users)
-	 * - Keywords mentioned: "company", "organization", "team", "workspace", "account"
-	 *
-	 * DO NOT include for B2C (business to consumer) scenarios
-	 * DO NOT include unless explicitly needed for group analytics
-	 */
-	groupKeys: [
-		["company_id", 1000], // 1000 companies
-		["team_id", 5000] // 5000 teams
-	],
-
-	/**
-	 * groupProps are properties associated with groups
-	 * Similar to userProps but for group entities
-	 *
-	 * ⚠️ IMPORTANT: ONLY include if you have groupKeys defined
-	 * These represent attributes of the group entities (companies, teams, etc.)
-	 */
-	groupProps: {
-		company_id: {
-			name: { "functionName": "chance.company", "args": [] },
-			plan: ["startup", "growth", "enterprise"],
-			employees: { "functionName": "weighNumRange", "args": [1, 5000, 0.3] },
-			industry: { "functionName": "chance.pickone", "args": [["tech", "finance", "healthcare", "retail", "manufacturing"]] }
-		},
-		team_id: {
-			name: { "functionName": "arrow", "body": "`Team ${chance.word()}`" },
-			size: { "functionName": "weighNumRange", "args": [2, 50, 0.2] },
-			department: ["engineering", "sales", "marketing", "support", "product"]
-		}
-	}
-};
-
-export default DUNGEON;