make-mp-data 3.0.3 → 3.0.5

package/dungeons/capstone/capstone-ic5.js

@@ -0,0 +1,668 @@
+/**
+ * ═══════════════════════════════════════════════════════════════
+ * DATASET OVERVIEW
+ * ═══════════════════════════════════════════════════════════════
+ *
+ * IC5 Capstone — advanced enterprise SaaS for Mixpanel certification.
+ * - Complex events with deeply nested metadata (server, request, response)
+ * - Text generation for support tickets and social posts
+ * - Session computation with duration_ms, written to external JSONL
+ * - Key name remapping (event→action, time→occurred_at, etc.)
+ * - Profile cleanup (removes anonymousIds, sessionIds)
+ *
+ * ═══════════════════════════════════════════════════════════════
+ * ANALYTICS HOOKS (1 pattern — everything hook)
+ * ═══════════════════════════════════════════════════════════════
+ *
+ * 1. DATA TRANSFORMATION + DUPLICATION (everything hook)
+ *    - Renames keys: event→action, time→occurred_at, device_id→client_id, user_id→auth_uuid
+ *    - Cleans profile: removes anonymousIds, sessionIds
+ *    - Computes session aggregates with duration and writes to external file
+ *    - Removes insert_id and session_id from events
+ *    - Duplicates 2-10% of events per user (proportional to event count)
+ *    Students must work with non-standard key names and handle duplicates.
+ */
+
+import Chance from 'chance';
+const chance = new Chance();
+import { weighNumRange, date, integer } from "../brain/utils/utils.js";
+import * as u from 'ak-tools';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import { createTextGenerator, generateBatch } from "../brain/generators/text.js";
+import { PHRASE_BANK } from "../brain/templates/phrases.js";
+import dayjs from 'dayjs';
+
+// create this file if it doesn't exist
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const sessionsFile = path.join(__dirname, '../data/generated_sessions.jsonl');
+
+// Create the sessions file if it doesn't exist (JSON Lines format)
+if (!fs.existsSync(sessionsFile)) {
+	fs.writeFileSync(sessionsFile, '', 'utf8');
+}
+
+
+const enterpriseSupportGen = createTextGenerator({
+	style: "support",
+	tone: "neg",
+	intensity: "high",
+	formality: "technical",
+	keywords: {
+		features: ['Dashboard Analytics', 'Export API', 'SSO Login', 'Admin Console', 'User Management', 'SAML Authentication', 'OAuth Integration', 'Multi-Factor Auth', 'Role-Based Access Control', 'Audit Logs', 'Webhook Configuration', 'Data Pipeline', 'Custom Reports', 'Scheduled Exports', 'Real-Time Sync', 'Bulk Import', 'API Rate Limits', 'Usage Metrics', 'Team Permissions', 'Single Sign-On', 'LDAP Integration', 'Data Retention Policies', 'Custom Domains', 'White-Label Branding', 'Advanced Filtering', 'Query Builder', 'Notification Rules', 'Email Templates', 'Workflow Automation', 'Integration Hub', 'Data Validation', 'Field Mapping', 'Schema Management', 'Version Control', 'Rollback Capability', 'Disaster Recovery', 'High Availability', 'Load Balancing', 'Auto-Scaling', 'Performance Monitoring'],
+		products: ['DataViz Pro', 'Enterprise Suite', 'v3.2.1', 'v2.8.4', 'Analytics Platform', 'Cloud Dashboard', 'v4.1.0', 'v3.9.2', 'Enterprise Edition', 'Professional Tier', 'Business Intelligence Module', 'Data Warehouse Connector', 'Mobile SDK', 'REST API v2', 'GraphQL Endpoint', 'Legacy Platform', 'Next-Gen Analytics', 'Premium Package', 'Ultimate Plan', 'Advanced Analytics Suite', 'Reporting Engine', 'Data Integration Platform', 'ETL Pipeline', 'Real-Time Analytics', 'Batch Processing Module', 'Stream Processing', 'Visualization Library', 'Dashboard Builder', 'Report Designer'],
+		technical: ['CORS error', 'timeout', 'memory leak', 'authentication', 'database', 'connection pooling', 'rate limiting', 'SSL handshake', 'DNS resolution', 'load balancer', 'cache invalidation', 'session timeout', 'deadlock detection', 'query optimization', 'index fragmentation', 'replication lag', 'connection refused', 'network latency', 'packet loss', 'firewall rules', 'proxy configuration', 'certificate expiration', 'API throttling', 'websocket disconnection', 'redis cluster', 'kafka consumer lag', 'database migration', 'schema validation', 'token refresh', 'OAuth flow', 'SAML assertion', 'JWT validation', 'API Gateway', 'reverse proxy', 'CDN distribution', 'geo-replication', 'data consistency', 'eventual consistency', 'distributed transactions', 'circuit breaker', 'retry logic', 'exponential backoff', 'service mesh', 'container orchestration', 'pod scheduling', 'horizontal scaling', 'vertical scaling', 'auto-discovery', 'health checks', 'graceful shutdown'],
+		errors: ['ERR_CONNECTION_REFUSED', '500 Internal Server', 'TIMEOUT_ERROR', 'AUTH_FAILED', '502 Bad Gateway', '503 Service Unavailable', '504 Gateway Timeout', '401 Unauthorized', '403 Forbidden', '429 Too Many Requests', 'ECONNRESET', 'ETIMEDOUT', 'ERR_SSL_PROTOCOL', 'DATABASE_CONNECTION_FAILED', 'INVALID_TOKEN', 'SESSION_EXPIRED', 'QUOTA_EXCEEDED', 'CORS_POLICY_VIOLATION', 'VALIDATION_ERROR', 'PERMISSION_DENIED', 'RESOURCE_NOT_FOUND', 'DUPLICATE_ENTRY', 'CONSTRAINT_VIOLATION', 'OUT_OF_MEMORY', 'DISK_FULL', 'MAX_CONNECTIONS_REACHED', 'DEADLOCK_DETECTED', 'TRANSACTION_ROLLBACK', 'REPLICATION_ERROR', 'CLUSTER_SPLIT_BRAIN', 'FAILOVER_TRIGGERED', 'BACKUP_FAILED', 'RESTORE_ERROR', 'MIGRATION_FAILED', 'SCHEMA_MISMATCH', 'VERSION_CONFLICT'],
+		competitors: ['Tableau', 'PowerBI', 'Looker', 'Qlik', 'Domo', 'Sisense', 'ThoughtSpot', 'Mode Analytics', 'Metabase', 'Redash', 'Chartio', 'Periscope Data', 'Google Data Studio', 'Amazon QuickSight', 'IBM Cognos', 'SAP Analytics', 'Oracle BI', 'MicroStrategy', 'Yellowfin', 'Birst', 'Alteryx', 'Dataiku', 'Databricks', 'Snowflake', 'BigQuery', 'Redshift', 'Azure Synapse', 'Splunk', 'New Relic', 'Datadog']
+	},
+	mixedSentiment: true,
+	authenticityLevel: 0.7,
+	typos: true,
+	typoRate: 0.02,
+	specificityLevel: 0.8,
+	min: 80,
+	max: 254,
+	includeMetadata: false,
+});
+
+// Search queries with realistic typos
+const searchQueryGen = createTextGenerator({
+	style: "search",
+	tone: "neu",
+	formality: "casual",
+	keywords: {
+		features: ['export data', 'user settings', 'help docs', 'pricing', 'billing', 'account settings', 'password reset', 'two factor authentication', 'team management', 'permissions', 'API keys', 'webhooks', 'integrations', 'custom fields', 'bulk upload', 'import CSV', 'export PDF', 'download report', 'sharing options', 'collaboration', 'notifications', 'email preferences', 'mobile app', 'desktop app', 'browser extension', 'keyboard shortcuts', 'templates', 'automation', 'workflows', 'filters', 'search', 'sorting', 'grouping', 'charts', 'dashboards', 'analytics', 'reports', 'insights', 'trends', 'forecasting', 'benchmarks', 'goals', 'KPIs', 'metrics', 'data visualization', 'custom reports', 'scheduled reports', 'alerts', 'reminders', 'tasks', 'projects', 'files', 'attachments', 'comments', 'mentions', 'tags', 'labels', 'categories', 'folders', 'archive', 'trash', 'restore', 'backup', 'export', 'import', 'migrate', 'sync', 'offline access', 'dark theme', 'language settings', 'time zone', 'date format', 'currency', 'units', 'accessibility', 'privacy', 'security', 'compliance', 'audit trail', 'version history', 'changelog'],
+		technical: ['API documentation', 'integration guide', 'troubleshooting', 'SDK reference', 'REST API', 'GraphQL', 'authentication', 'OAuth', 'SSO setup', 'SAML', 'rate limits', 'error codes', 'status page', 'changelog', 'release notes', 'migration guide', 'upgrade path', 'deprecation notice', 'API versioning', 'webhooks setup', 'callback URL', 'payload format', 'request headers', 'response codes', 'pagination', 'filtering', 'sorting API', 'batch endpoints', 'async operations', 'retry logic', 'timeout settings', 'CORS configuration', 'IP whitelist', 'security best practices', 'code examples', 'sample requests', 'postman collection', 'curl commands', 'client libraries', 'npm packages', 'pip packages', 'gem install', 'maven dependency', 'nuget package', 'composer require', 'quickstart tutorial', 'getting started', 'hello world', 'first API call', 'authentication flow', 'token management', 'refresh tokens', 'scopes', 'permissions', 'roles', 'rate limit handling', 'error handling', 'best practices', 'common pitfalls', 'FAQ', 'support forum']
+	},
+	typos: true,
+	typoRate: 0.05,	
+	min: 2,
+	max: 255,
+	includeMetadata: false
+});
+
+// Business feedback with professional tone
+const businessFeedbackGen = createTextGenerator({
+	style: "feedback",
+	tone: "neu",
+	formality: "business",
+	keywords: {
+		metrics: ['ROI', 'efficiency', 'cost savings', 'productivity', 'revenue growth', 'profit margin', 'conversion rate', 'customer acquisition cost', 'lifetime value', 'churn rate', 'retention rate', 'engagement rate', 'adoption rate', 'time to value', 'operational efficiency', 'resource utilization', 'cost per transaction', 'revenue per user', 'gross margin', 'net promoter score', 'customer satisfaction', 'employee satisfaction', 'market share', 'growth rate', 'scalability', 'performance metrics', 'KPI achievement', 'goal attainment', 'benchmark comparison', 'industry standards', 'competitive advantage', 'process improvement', 'quality metrics', 'error rate', 'uptime SLA', 'response time SLA', 'mean time to resolution', 'first contact resolution', 'customer effort score', 'employee Net Promoter Score', 'revenue per employee', 'operating margin', 'EBITDA', 'cash flow', 'burn rate', 'runway', 'payback period', 'IRR', 'NPV', 'break-even analysis', 'variance analysis', 'trend analysis', 'cohort analysis', 'funnel conversion', 'pipeline velocity', 'win rate', 'average deal size', 'sales cycle length', 'quota attainment', 'forecast accuracy'],
+		features: ['reporting', 'analytics', 'integration capabilities', 'dashboard customization', 'automated workflows', 'data visualization', 'forecasting tools', 'predictive analytics', 'real-time monitoring', 'alert system', 'custom reports', 'scheduled reports', 'export functionality', 'API access', 'bulk operations', 'team collaboration', 'role-based access', 'audit trails', 'compliance features', 'data security', 'backup and recovery', 'scalability', 'mobile access', 'offline capabilities', 'third-party integrations', 'CRM integration', 'ERP integration', 'accounting software', 'payment gateways', 'marketing tools', 'customer support', 'training resources', 'documentation', 'professional services', 'dedicated support', 'white-glove onboarding', 'custom development', 'consulting services', 'implementation support', 'change management', 'data migration', 'system integration', 'workflow automation', 'process optimization', 'business intelligence', 'advanced analytics', 'machine learning', 'AI capabilities', 'natural language processing', 'sentiment analysis', 'anomaly detection', 'pattern recognition']
+	},
+	authenticityLevel: 0.3,
+	specificityLevel: 0.7,	
+	min: 40,
+	max: 1000,
+	includeMetadata: false
+});
+
+
+/** @type {import('../types.js').Dungeon} */
+const config = {
+	token: "",
+	seed: "this is going to be hard",
+	name: "ic5-capstone",
+	numDays: 180, //how many days worth of data
+	numEvents: 5_000_000, //how many events
+	numUsers: 25_000, //how many users	
+	format: 'parquet', //csv or json
+	region: "US",
+	gzip: false,
+	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasSessionIds: true, //if true, hasSessionIds are created for each user
+
+	hasLocation: true,
+	hasAndroidDevices: false,
+	hasIOSDevices: false,
+	hasDesktopDevices: false,
+	hasBrowser: false,
+	hasCampaigns: false,
+	isAnonymous: false,
+	hasAdSpend: true,
+
+	hasAvatar: true,
+	makeChart: false,
+
+	batchSize: 250_000,
+	concurrency: 1,
+	writeToDisk: true,
+
+	funnels: [],
+	events: [
+		{
+			"event": "checkout",
+			"weight": 2,
+			"properties": {				
+				currency: ["USD", "USD", "USD", "CAD", "EUR", "EUR", "BTC", "BTC", "ETH", "JPY"],
+				cart: makeProducts(),
+			}
+		},
+		{
+			"event": "add to cart",
+			"weight": 4,
+			"properties": {
+				amount: weighNumRange(5, 500, .25),
+				qty: integer(1, 5),
+				product_id: weighNumRange(1, 250000, 1.4)
+			}
+		},
+		{
+			"event": "page view",
+			"weight": 10,
+			"properties": {
+				page: ["/", "/", "/", "/learn-more", "/pricing", "/contact", "/about", "/careers", "/sign-up", "/login", "/app", "/app", "/app", "/app"],
+				utm_source: ["$organic", "$organic", "$organic", "$organic", "google", "google", "google", "facebook", "facebook", "twitter", "linkedin"],
+			}
+		},
+			{
+			event: "search",
+			weight: 1, // Most frequent event
+			properties: {
+				query_text: () => searchQueryGen.generateOne(),
+				search_category: ["help", "documentation", "features", "pricing", "support"],
+				results_found: weighNumRange(0, 100, 0.3),
+				clicked_result: [true, true, false], // 67% click through
+				result_position_clicked: weighNumRange(1, 10, 0.8), // Most click top results
+				search_duration_seconds: weighNumRange(1, 300, 0.7),
+				refined_search: [true, false, false, false] // 25% refine search
+			}
+		},
+		{
+			event: "write feedback",
+			weight: 1,
+			properties: {
+				feedback_text: () => businessFeedbackGen.generateOne(),
+				satisfaction_score: weighNumRange(1, 10, 0.3),
+				feedback_category: ["feature_request", "improvement", "compliment", "concern"],
+				department: ["engineering", "sales", "marketing", "support", "executive"],
+				follow_up_requested: [true, false, false], // 33% want follow-up
+				urgency: ["low", "medium", "high"],
+				implementation_priority: weighNumRange(1, 5, 0.4)
+			}
+		},
+		{
+			"event": "watch video",
+			"weight": 8,
+			"properties": {
+				category: ["funny", "educational", "inspirational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"],
+				hashTags: makeHashTags,
+				watchTimeSec: weighNumRange(10, 600, .25,),
+				quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
+				format: ["mp4", "avi", "mov", "mpg"],
+				video_id: weighNumRange(1, 500_000, 1.4),
+
+			}
+		},
+		{
+			"event": "comment",
+			"weight": 2,
+			"properties": {
+				length: weighNumRange(1, 500, .25),
+				video_id: weighNumRange(1, 500_000, 1.4),
+				has_replies: [true, false, false, false, false],
+				has_photo: [true, false, false, false, false],
+
+			}
+		},
+		{
+			"event": "save video",
+			"weight": 4,
+			"properties": {
+				video_id: weighNumRange(1, 500_000, 1.4),
+				ui_control: ["toolbar", "menu", "keyboard"]
+
+
+			}
+		},
+		{
+			"event": "view item",
+			"weight": 8,
+			"properties": {
+				product_id: weighNumRange(1, 250000, 3),
+				colors: ["light", "dark", "custom", "dark"]
+			}
+		},
+		{
+			"event": "save item",
+			"weight": 5,
+			"properties": {
+				product_id: weighNumRange(1, 250000, 12),
+				colors: ["light", "dark", "custom", "dark"]
+			}
+		},
+		{
+			"event": "support ticket",
+			"weight": 2,
+			"properties": {				
+				severity: ["low", "medium", "high"],
+				ticket_id: chance.guid.bind(chance),
+				ticket_text: () => enterpriseSupportGen.generateOne()
+			}
+		},
+		{
+			"event": "sign up",
+			"isFirstEvent": true,
+			"weight": 0,
+			"properties": {
+				plan: ["free", "free", "free", "free", "basic", "basic", "basic", "premium", "premium", "enterprise"],
+				dateOfRenewal: date(100, false),
+				codewords: u.makeName,
+			}
+		}
+	],
+	superProps: {
+		linked_device: deviceAttributes(),
+		server_metaData: generateMetaData()
+	},
+	/*
+	user properties work the same as event properties
+	each key should be an array or function reference
+	*/
+	userProps: {
+		title: chance.profession.bind(chance),
+		luckyNumber: weighNumRange(42, 420),
+		experiment: designExperiment(),
+		spiritAnimal: ["unicorn", "dragon", "phoenix", "sasquatch", "yeti", "kraken", "jackalope", "thunderbird", "mothman", "nessie", "chupacabra", "jersey devil", "bigfoot", "weindgo", "bunyip", "mokele-mbembe", "tatzelwurm", "megalodon"],
+		timezone: chance.timezone.bind(chance), // ["America/New_York", "America/Los_Angeles", "America/Chicago", "America/Denver", "America/Phoenix", "America/Anchorage", "Pacific/Honolulu"]
+		ip: chance.ip.bind(chance),
+		// lastCart: makeProducts(5),
+
+	},
+
+	/** each generates it's own table */
+	scdProps: {
+		role: {
+			type: "user",
+			frequency: "week",
+			values: ["admin", "collaborator", "user", "view only", "no access"],
+			timing: 'fuzzy',
+			max: 10
+		},
+		NPS: {
+			type: "user",
+			frequency: "day",
+			values: weighNumRange(1, 10, 2, 150),
+			timing: 'fuzzy',
+			max: 10
+		},
+		MRR: {
+			type: "company_id",
+			frequency: "month",
+			values: weighNumRange(0, 10000, .15),
+			timing: 'fixed',
+			max: 10
+		},
+		AccountHealthScore: {
+			type: "company_id",
+			frequency: "week",
+			values: weighNumRange(1, 10, .15),
+			timing: 'fixed',
+			max: 40
+		},
+		plan: {
+			type: "company_id",
+			frequency: "month",
+			values: ["free", "basic", "premium", "enterprise"],
+			timing: 'fixed',
+			max: 10
+		},
+
+	},
+
+	mirrorProps: {
+	},
+
+	/*
+	for group analytics keys, we need an array of arrays [[],[],[]] 
+	each pair represents a group_key and the number of profiles for that key
+	*/
+	groupKeys: [
+		['company_id', 500, []],
+		['room_id', 10000, ["save video", "comment", "watch video"]],
+
+	],
+	groupProps: {
+		company_id: {
+			name: () => { return chance.company(); },
+			email: () => { return `CSM: ${chance.pickone(["AK", "Jessica", "Michelle", "Dana", "Brian", "Dave"])}`; },
+			"# of employees": weighNumRange(3, 10000),
+			"industry": ["tech", "finance", "healthcare", "education", "government", "non-profit"],
+			"segment": ["enterprise", "SMB", "mid-market"],
+			"products": [["core"], ["core"], ["core", "add-ons"], ["core", "pro-serve"], ["core", "add-ons", "pro-serve"], ["core", "BAA", "enterprise"], ["free"], ["free"], ["free", "addons"]],
+		},
+		room_id: {
+			name: () => { return `#${chance.word({ length: integer(4, 24), capitalize: true })}`; },
+			email: ["public", "private"],
+			"room provider": ["partner", "core", "core", "core"],
+			"room capacity": weighNumRange(3, 1000000),
+			"isPublic": [true, false, false, false, false],
+			"country": chance.country.bind(chance),
+			"isVerified": [true, true, false, false, false],
+		}
+	},
+	groupEvents: [{
+		attribute_to_user: false,
+		event: "card charged",
+		weight: 1,
+		frequency: 30,
+		group_key: "company_id",
+		group_size: 500,
+		properties: {
+			amount: weighNumRange(5, 500, .25),
+			currency: ["USD", "USD", "USD", "CAD", "EUR", "EUR", "BTC", "BTC", "ETH", "JPY"],
+			plan: ["basic", "premium", "enterprise"],
+			"payment method": []
+		}
+	}],
+
+	lookupTables: [
+		{
+			key: "video_id",
+			entries: 500_000,
+			attributes: {
+				isFlagged: [true, false, false, false, false],
+				copyright: ["all rights reserved", "creative commons", "creative commons", "public domain", "fair use"],
+				uploader_id: chance.guid.bind(chance),
+				"uploader influence": ["low", "low", "low", "medium", "medium", "high"],
+				thumbs: weighNumRange(0, 35),
+				rating: ["G", "PG", "PG-13", "R", "NC-17", "PG-13", "R", "NC-17", "R", "PG", "PG"]
+			}
+
+		}
+	],
+
+	hook: function (record, type, meta) {
+
+		if (type === "everything") {
+			const changeKeyNames = [
+				["event", "action"],
+				["time", "occurred_at"],
+				["device_id", "client_id"],
+				["user_id", "auth_uuid"]
+			]
+
+			const keysToRemove = [
+				"insert_id",
+				"session_id"
+			]
+
+			const profileKeysToRemove = [
+				"anonymousIds",
+				"sessionIds"
+			]
+
+			// remove unwanted keys from profile
+			profileKeysToRemove.forEach(key => {
+				if (meta.profile.hasOwnProperty(key)) {
+					delete meta.profile[key];
+				}
+			});
+			
+			record.forEach(event => {
+				// change key names
+				changeKeyNames.forEach(pair => {
+					const [oldKey, newKey] = pair;
+					if (event.hasOwnProperty(oldKey)) {
+						event[newKey] = event[oldKey];
+						delete event[oldKey];
+					}
+				});
+			});
+
+			// get all unique session_id
+			const uniqueSessionIds = new Set();
+			record.forEach(event => {
+				if (event.hasOwnProperty("session_id")) {
+					uniqueSessionIds.add(event["session_id"]);
+				}
+			});
+
+			// compute the length of all sessions
+			const sessions = [];
+			uniqueSessionIds.forEach(sessId => {
+				const sessEvents = record.filter(event => event["session_id"] === sessId);
+				
+				// Parse all timestamps with dayjs and sort
+				const timestamps = sessEvents
+					.map(e => dayjs(e.occurred_at))
+					.sort((a, b) => a.valueOf() - b.valueOf());
+				
+				// First timestamp is earliest, last is latest
+				const startTime = timestamps[0];
+				const endTime = timestamps[timestamps.length - 1];
+				
+				// Calculate duration in milliseconds
+				const durationMs = endTime.diff(startTime, 'second');
+				
+				sessions.push({
+					session_id: sessId,
+					event_count: sessEvents.length,
+					start_time: startTime.toISOString(),
+					duration_ms: durationMs,
+					uuid: meta.profile.distinct_id
+				});
+			});
+
+			// append sessions to the file
+			if (sessions.length > 0) {
+				const sessionLines = sessions.map(s => JSON.stringify(s)).join('\n') + '\n';
+				fs.appendFileSync(sessionsFile, sessionLines, 'utf8');
+			}
+
+			// remove unwanted keys
+			record.forEach(event => {
+				keysToRemove.forEach(key => {
+					if (event.hasOwnProperty(key)) {
+						delete event[key];
+					}
+				});
+			});
+
+			// every users has 2-10% of events duplicated
+			let numDupes = Math.floor(record.length * (integer(2, 10) / 100));			
+			if (record.length < numDupes) numDupes = record.length - 1;
+			// pick random events to duplicate
+			const dupesIndexes = [];
+			while (dupesIndexes.length < numDupes) {
+				const idx = integer(0, record.length - 1);
+				if (!dupesIndexes.includes(idx)) {
+					dupesIndexes.push(idx);
+				}
+			}
+
+			// duplicate those events
+			dupesIndexes.forEach(idx => {
+				const eventToDup = record[idx];
+				const newEvent = { ...eventToDup };
+				record.push(newEvent);
+			});
+		}
+
+		return record;
+	}
+};
+
+
+function generateMetaData() {
+	// should return a deeply nested object
+	return function () {
+		return {
+			server: {
+				id: chance.guid(),
+				location: {
+					region: chance.pickone(["us-east-1", "us-west-2", "eu-central-1", "ap-southeast-1"]),
+					ip: chance.ip()
+				},
+				specs: {
+					cpu: `${chance.integer({ min: 2, max: 64 })} vCPUs`,
+					memory: `${chance.integer({ min: 4, max: 256 })} GB`,
+					storage: `${chance.integer({ min: 100, max: 2000 })} GB`
+				}
+			},
+			application: {
+				version: `v${chance.integer({ min: 1, max: 5 })}.${chance.integer({ min: 0, max: 10 })}.${chance.integer({ min: 0, max: 20 })}`,
+				environment: chance.pickone(["production","production","production","production","production","production","production","production", "staging", "development"])
+			},
+			request: {
+				id: chance.guid(),
+				latency_ms: chance.integer({ min: 20, max: 2000 }),
+				endpoint: chance.pickone(["/api/v1/login", "/api/v1/data", "/api/v1/submit", "/api/v1/logout"]),
+				method: chance.pickone(["GET", "POST", "PUT", "DELETE"])
+			},
+			response: {
+				status_code: chance.pickone([200, 200, 200, 201, 400, 401, 403, 404, 500, 502, 503]),
+				size_bytes: chance.integer({ min: 500, max: 5000 })
+			},
+			services: {
+				database: {
+					type: chance.pickone(["PostgreSQL", "MySQL", "MongoDB", "Redis"]),
+					version: `v${chance.integer({ min: 9, max: 14 })}.${chance.integer({ min: 0, max: 10 })}`,
+					host: chance.ip()
+				},
+				cache: {
+					type: chance.pickone(["Redis", "Memcached"]),
+					version: `v${chance.integer({ min: 4, max: 6 })}.${chance.integer({ min: 0, max: 10 })}`,
+					host: chance.ip()
+				},
+				message_queue: {
+					type: chance.pickone(["RabbitMQ", "Kafka", "AWS SQS"]),
+					version: `v${chance.integer({ min: 1, max: 3 })}.${chance.integer({ min: 0, max: 10 })}`,
+					host: chance.ip()
+				}
+			}
+		};
+	};
+}
+
+
+function makeHashTags() {
+	const possibleHashtags = [];
+	for (let i = 0; i < 20; i++) {
+		possibleHashtags.push('#' + u.makeName(2, ''));
+	}
+
+	const numHashtags = integer(integer(1, 5), integer(5, 10));
+	const hashtags = [];
+	for (let i = 0; i < numHashtags; i++) {
+		hashtags.push(chance.pickone(possibleHashtags));
+	}
+	return [hashtags];
+};
+
+
+
+function designExperiment() {
+	return function () {
+		const variants = ["A", "B", "C", "Control"];
+		const variant = chance.pickone(variants);
+		const experiments = ["no password", "social sign in", "new tutorial", "new search"];
+		const experiment = chance.pickone(experiments);
+		const multi_variates = ["A/B", "A/B/C", "A/B/C/D", "Control"];
+		const multi_variate = chance.pickone(multi_variates);
+		const impression_id = chance.guid();
+
+
+
+		const chosen = {
+			variant,
+			experiment,
+			multi_variate,
+			impression_id
+		};
+
+		return [chosen];
+	};
+}
+
+function deviceAttributes(isMobile = false) {
+	return function () {
+		let devices = ["desktop", "laptop", "desktop", "laptop", "desktop", "laptop", "other"];
+		if (isMobile) devices = [...devices, "mobile", "mobile", "mobile", "tablet"];
+		const device = chance.pickone(devices);
+		let oses = ["Windows", "macOS", "Windows", "macOS", "macOS", "Linux", "Windows", "macOS", "Windows", "macOS", "macOS", "TempleOS"];
+		if (isMobile) oses = [...oses, "iOS", "Android", "iOS", "Android"];
+		const os = chance.pickone(oses);
+		const browser = chance.pickone(["Chrome", "Firefox", "Safari", "Edge", "Opera", "IE", "Brave", "Vivaldi"]);
+		const version = chance.integer({ min: 1, max: 15 });
+		const resolution = chance.pickone(["1920x1080", "1280x720", "1024x768", "800x600", "640x480"]);
+		const language = chance.pickone(["en-US", "en-US", "en-US", "en-GB", "es", "es", "fr", "de", "it", "ja", "zh", "ru"]);
+
+		const chosen = {
+			platform: device,
+			os,
+			browser,
+			version,
+			resolution,
+			language,
+			linking_method: chance.pickone(["cookie", "localStorage", "fingerprint", "login"]),
+			heuristics: {
+				timezone_offset: chance.integer({ min: -720, max: 840 }),
+				plugins_installed: chance.integer({ min: 0, max: 15 }),
+				java_enabled: chance.bool(),
+				cookie_enabled: chance.bool(),
+				screen_color_depth: chance.pickone([24, 30, 32]),
+				screen_width: parseInt(resolution.split('x')[0]),
+				screen_height: parseInt(resolution.split('x')[1]),
+				viewport: {
+					width: parseInt(resolution.split('x')[0]),
+					height: parseInt(resolution.split('x')[1])
+				}
+			}
+		};
+
+		return chosen;
+
+	};
+}
+
+function makeProducts(maxItems = 50) {
+	return function () {
+		const categories = ["electronics", "books", "clothing", "home", "garden", "toys", "sports", "automotive", "beauty", "health", "grocery", "jewelry", "shoes", "tools", "office supplies"];
+		const descriptors = ["brand new", "open box", "refurbished", "used", "like new", "vintage", "antique", "collectible"];
+		const suffix = ["item", "product", "good", "merchandise", "thing", "object", "widget", "gadget", "device", "apparatus", "contraption", "instrument", "tool", "implement", "utensil", "appliance", "machine", "equipment", "gear", "kit", "set", "package"];
+		const assetPreview = ['.png', '.jpg', '.jpeg', '.heic', '.mp4', '.mov', '.avi'];
+		const data = [];
+		const numOfItems = integer(1, maxItems);
+
+		for (var i = 0; i < numOfItems; i++) {
+			const category = chance.pickone(categories);
+			const descriptor = chance.pickone(descriptors);
+			const suffixWord = chance.pickone(suffix);
+			const slug = `${descriptor.replace(/\s+/g, '-').toLowerCase()}-${suffixWord.replace(/\s+/g, '-').toLowerCase()}`;
+			const asset = chance.pickone(assetPreview);
+
+			// const product_id = chance.guid();
+			const price = integer(1, 100);
+			const quantity = integer(1, 5);
+
+			const item = {
+				// product_id: product_id,
+				// sku: integer(11111, 99999),
+				amount: price,
+				quantity: quantity,
+				total_value: price * quantity,
+				featured: chance.pickone([true, false, false]),
+				category: category,
+				descriptor: descriptor,
+				slug: slug,
+				assetPreview: `https://example.com/assets/${slug}${asset}`,
+				assetType: asset
+
+			};
+
+			data.push(item);
+		}
+
+		return () => [data];
+	};
+};
+
+
+
+
+
+export default config;