make-mp-data 2.0.21 → 2.0.23

package/dungeons/student-teacher.js

@@ -1,4 +1,3 @@
-// Generated from prompt: Create a digital classroom data set where we have two types of users students and teachers and students do student actions like reading writing arithmetic homework tests and teachers do teaching actions like teach grade take attendance etc. importantly each event done by each persona should have a 'student:' or teacher: prefix to indicate that this is an event on by students or this is not done by teachers... make many unique events + funnels.
 
 import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
@@ -24,7 +23,7 @@ const config = {
 	hasAnonIds: false,
 	hasSessionIds: false,
 	format: "json",
-	alsoInferFunnels: true,
+	alsoInferFunnels: false,
 	hasLocation: true,
 	hasAndroidDevices: true,
 	hasIOSDevices: true,
@@ -53,6 +52,9 @@ const config = {
 			"isFirstFunnel": true,
 			"conversionRate": 95,
 			"timeToConvert": 0.1,
+			"conditions": {
+				"role": "student"
+			},
 			"order": "sequential",
 			"props": {
 				"onboarding_source": [
@@ -69,6 +71,9 @@ const config = {
 				"teacher: create class",
 				"teacher: post announcement"
 			],
+			"conditions": {
+				"role": "teacher"
+			},
 			"isFirstFunnel": true,
 			"conversionRate": 98,
 			"timeToConvert": 0.2,
@@ -82,6 +87,9 @@ const config = {
 				"student: submit assignment",
 				"student: view grade"
 			],
+			"conditions": {
+				"role": "student"
+			},
 			"isFirstFunnel": false,
 			"conversionRate": 70,
 			"timeToConvert": 48,
@@ -92,6 +100,9 @@ const config = {
 		},
 		{
 			"name": "Student Takes Test",
+			"conditions": {
+				"role": "student"
+			},
 			"sequence": [
 				"student: view assignment",
 				"student: start test",
@@ -109,6 +120,9 @@ const config = {
 		},
 		{
 			"name": "Teacher Grades Assignment",
+			"conditions": {
+				"role": "teacher"
+			},
 			"sequence": [
 				"teacher: create assignment",
 				"teacher: assign homework",
@@ -126,10 +140,21 @@ const config = {
 				"teacher: start lesson",
 				"teacher: end lesson"
 			],
+			"conditions": {
+				"role": "teacher"
+			},
 			"isFirstFunnel": false,
 			"conversionRate": 99,
 			"timeToConvert": 1,
 			"order": "sequential"
+		},
+		{
+			"name": "all peoples doing stuf (no conditions)",
+			"sequence": [
+				"foo",
+				"bar",
+				"baz"
+			]
 		}
 	],
 	"events": [
@@ -291,6 +316,15 @@ const config = {
 					"Event"
 				]
 			}
+		},
+		{
+			"event": "foo"
+		},
+		{
+			"event": "bar"
+		},
+		{
+			"event": "baz"
 		}
 	],
 	"superProps": {
@@ -326,8 +360,6 @@ const config = {
 			"student",
 			"teacher"
 		],
-		"first_name": "chance.first.bind(chance)",
-		"last_name": "chance.last.bind(chance)"
 	},
 	"scdProps": {
 		"gpa": {
@@ -358,88 +390,7 @@ const config = {
 			"max": 12
 		}
 	},
-	"groupKeys": [
-		[
-			"class_id",
-			100
-		]
-	],
-	"groupProps": {
-		"class_id": {
-			"teacher_name": "() => `Mrs. ${chance.last()}`",
-			"school_name": "() => `${chance.city()} ${chance.pickone(['Elementary', 'Middle School', 'High School'])}`",
-			"class_size": "weighNumRange(15, 35, 0.5)",
-			"subject_id": "range(1, 20)"
-		}
-	},
-	"lookupTables": [
-		{
-			"key": "assignment_id",
-			"entries": 500,
-			"attributes": {
-				"type": [
-					"homework",
-					"test",
-					"quiz",
-					"project",
-					"reading"
-				],
-				"difficulty": [
-					"easy",
-					"medium",
-					"hard"
-				],
-				"max_points": [
-					10,
-					20,
-					25,
-					50,
-					100
-				],
-				"subject_id": "range(1, 20)"
-			}
-		},
-		{
-			"key": "subject_id",
-			"entries": 20,
-			"attributes": {
-				"subject_name": [
-					"Mathematics",
-					"English Language Arts",
-					"Science",
-					"Social Studies",
-					"History",
-					"Art",
-					"Music",
-					"Physical Education",
-					"Computer Science",
-					"Spanish",
-					"French",
-					"German",
-					"Biology",
-					"Chemistry",
-					"Physics",
-					"Algebra",
-					"Geometry",
-					"Calculus",
-					"Literature",
-					"World History"
-				],
-				"department": [
-					"STEM",
-					"Humanities",
-					"Arts",
-					"Foreign Language"
-				],
-				"is_core_subject": [
-					true,
-					true,
-					false
-				]
-			}
-		}
-	]
-	,
+
 
 	hook: function (record, type, meta) {
 		const NOW = dayjs();
@@ -460,7 +411,7 @@ const config = {
 
 		}
 
-		if (type === "scd") {
+		if (type === "scd-pre") {
 
 		}
 

package/entry.js

@@ -17,11 +17,17 @@ import getCliParams from './lib/cli/cli.js';
         if (cliConfig.complex) {
             const complexConfig = await import('./dungeons/complex.js');
             finalConfig = { ...complexConfig.default, ...cliConfig };
-        } else if (cliConfig.simple || (!cliConfig.complex && !cliConfig.simple)) {
+        } 
+		else if (cliConfig.sanity) {
+			const sanityConfig = await import('./dungeons/sanity.js');
+			finalConfig = { ...sanityConfig.default, ...cliConfig };
+		}
+		else if (cliConfig.simple || (!cliConfig.complex && !cliConfig.simple)) {
             // Default to simple mode when no flags or when --simple is explicitly set
             const simpleConfig = await import('./dungeons/simple.js');
             finalConfig = { ...simpleConfig.default, ...cliConfig };
         }
+
 		
 		
         const result = await main(finalConfig);        

package/index.js

@@ -89,7 +89,7 @@ function displayConfigurationSummary(config) {
 			if (funnel.sequence) {
 				const arrow = ' → ';
 				const sequence = funnel.sequence.join(arrow);
-				const rate = funnel.conversionRate ? ` (${(funnel.conversionRate * 100).toFixed(0)}% conversion)` : '';
+				const rate = funnel.conversionRate ? ` (${(funnel.conversionRate).toFixed(0)}% conversion)` : '';
 				console.log(`    ${i + 1}. ${sequence}${rate}`);
 			}
 		});
@@ -194,7 +194,12 @@ async function main(config) {
 			await generateCharts(context);
 		}
 
-		// Step 11: Flush storage containers to disk (if writeToDisk enabled)
+		// Step 11a:  flush lookup tables to disk (always as CSVs)
+		if (validatedConfig.writeToDisk) {
+			await flushLookupTablesToDisk(storage, validatedConfig);
+		}
+
+		// Step 11b: Flush other storage containers to disk (if writeToDisk enabled)
 		if (validatedConfig.writeToDisk) {
 			await flushStorageToDisk(storage, validatedConfig);
 		}
@@ -214,7 +219,7 @@ async function main(config) {
 		return {
 			...extractedData,
 			importResults,
-			files: extractFileInfo(storage),
+			files: await extractFileInfo(storage, validatedConfig),
 			time: { start, end, delta, human },
 			operations: context.getOperations(),
 			eventCount: context.getEventCount(),
@@ -423,7 +428,7 @@ async function generateCharts(context) {
 	if (config.makeChart && storage.eventData?.length > 0) {
 		const chartPath = typeof config.makeChart === 'string'
 			? config.makeChart
-			: `./${config.simulationName}-timeline`;
+			: `./${config.name}-timeline`;
 
 		await generateLineChart(storage.eventData, undefined, chartPath);
 
@@ -436,7 +441,33 @@ async function generateCharts(context) {
 }
 
 /**
- * Flush all storage containers to disk
+ * Flush lookup tables to disk (always runs, regardless of writeToDisk setting)
+ * @param {import('./types').Storage} storage - Storage containers
+ * @param {import('./types').Dungeon} config - Configuration object
+ */
+async function flushLookupTablesToDisk(storage, config) {
+	if (!storage.lookupTableData || !Array.isArray(storage.lookupTableData) || storage.lookupTableData.length === 0) {
+		return; // No lookup tables to flush
+	}
+
+	if (config.verbose) {
+		console.log('💾 Writing lookup tables to disk...');
+	}
+
+	const flushPromises = [];
+	storage.lookupTableData.forEach(container => {
+		if (container?.flush) flushPromises.push(container.flush());
+	});
+
+	await Promise.all(flushPromises);
+
+	if (config.verbose) {
+		console.log('🗂️  Lookup tables flushed to disk successfully');
+	}
+}
+
+/**
+ * Flush all storage containers to disk (excluding lookup tables)
  * @param {import('./types').Storage} storage - Storage containers
  * @param {import('./types').Dungeon} config - Configuration object
  */
@@ -454,8 +485,8 @@ async function flushStorageToDisk(storage, config) {
 	if (storage.mirrorEventData?.flush) flushPromises.push(storage.mirrorEventData.flush());
 	if (storage.groupEventData?.flush) flushPromises.push(storage.groupEventData.flush());
 
-	// Flush arrays of HookedArrays
-	[storage.scdTableData, storage.groupProfilesData, storage.lookupTableData].forEach(arrayOfContainers => {
+	// Flush arrays of HookedArrays (excluding lookup tables which are handled separately)
+	[storage.scdTableData, storage.groupProfilesData].forEach(arrayOfContainers => {
 		if (Array.isArray(arrayOfContainers)) {
 			arrayOfContainers.forEach(container => {
 				if (container?.flush) flushPromises.push(container.flush());
@@ -473,11 +504,13 @@ async function flushStorageToDisk(storage, config) {
 /**
  * Extract file information from storage containers
  * @param {import('./types').Storage} storage - Storage object
+ * @param {import('./types').Dungeon} config - Configuration object
  * @returns {string[]} Array of file paths
  */
-function extractFileInfo(storage) {
+async function extractFileInfo(storage, config) {
 	const files = [];
 
+	// Try to get paths from containers first
 	Object.values(storage).forEach(container => {
 		if (Array.isArray(container)) {
 			container.forEach(subContainer => {
@@ -490,6 +523,55 @@ function extractFileInfo(storage) {
 		}
 	});
 
+	// If no files found from containers and writeToDisk is enabled, scan the data directory
+	if (files.length === 0 && config.writeToDisk) {
+		try {
+			const fs = await import('fs');
+			const path = await import('path');
+			
+			let dataDir = path.resolve("./data");
+			if (!fs.existsSync(dataDir)) {
+				dataDir = path.resolve("./");
+			}
+			
+			if (fs.existsSync(dataDir)) {
+				const allFiles = fs.readdirSync(dataDir);
+				const simulationName = config.name;
+				
+				// Filter files that match our patterns and were likely created by this run
+				const relevantFiles = allFiles.filter(file => {
+					// Skip system files
+					if (file.startsWith('.')) return false;
+					
+					// If we have a simulation name, only include files with that prefix
+					if (simulationName && !file.startsWith(simulationName)) {
+						return false;
+					}
+					
+					// Check for common patterns
+					const hasEventPattern = file.includes('-EVENTS.');
+					const hasUserPattern = file.includes('-USERS.');
+					const hasScdPattern = file.includes('-SCD.');
+					const hasGroupPattern = file.includes('-GROUPS.');
+					const hasLookupPattern = file.includes('-LOOKUP.');
+					const hasAdspendPattern = file.includes('-ADSPEND.');
+					const hasMirrorPattern = file.includes('-MIRROR.');
+					
+					return hasEventPattern || hasUserPattern || hasScdPattern || 
+						   hasGroupPattern || hasLookupPattern || hasAdspendPattern || hasMirrorPattern;
+				});
+				
+				// Convert to full paths
+				relevantFiles.forEach(file => {
+					files.push(path.join(dataDir, file));
+				});
+			}
+		} catch (error) {
+			// If scanning fails, just return empty array
+			console.warn('Warning: Could not scan data directory for files:', error.message);
+		}
+	}
+
 	return files;
 }
 

package/lib/cli/cli.js

@@ -121,6 +121,14 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
+		.options("sanity", {
+			demandOption: false,
+			default: false,
+			describe: 'run sanity checks on the generated data',
+			alias: 'san',
+			type: 'boolean',
+			coerce: boolCoerce
+		})
 		.option("writeToDisk", {
 			demandOption: false,
 			default: true,
@@ -215,6 +223,12 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
+		.option("name", {
+			alias: 'n',
+			demandOption: false,
+			describe: 'custom name for generated files (prefix)',
+			type: 'string'
+		})
 
 		.help()
 		.wrap(null)
@@ -228,7 +242,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 }
 
 
-function boolCoerce(value, foo) {
+function boolCoerce(value) {
 	if (typeof value === 'boolean') return value;
 	if (typeof value === 'string') {
 		return value.toLowerCase() === 'true';