make-mp-data 2.0.22 → 2.1.0

package/lib/templates/{instructions.txt → schema-instructions.txt}

@@ -58,7 +58,84 @@ Core Requirements:
 		- Use lookup tables if events reference external entities with their own attributes (e.g., product_id, video_id).
 		- Use funnel conditions when different user segments or cohorts should have different behavioral patterns (e.g., premium vs free users, students vs teachers, rider vs driver, doctor vs patient).
 
-	4. Available Functions: You have access to these built-in functions: date, weighNumRange, range, and the chance library.
+	4. Available Functions: You have access to these built-in functions: date, weighNumRange, range, chance library methods, and createGenerator for structured text generation.
+
+	5. Structured Text Generation: When your use case involves user-generated content (reviews, comments, support tickets, chat messages, social media posts, etc.), use createGenerator() to produce realistic, contextual text.
+
+		Text Generator Usage:
+
+		- createGenerator() creates a text generator object that produces varied, authentic-sounding text
+		- Embed the createGenerator() call directly in the properties object (NOT as a quoted string)
+		- The generator automatically produces unique text for each event
+
+		Example for a customer support platform:
+
+		{
+			event: "support_ticket_submitted",
+			properties: {
+				ticket_text: createGenerator({
+					style: "support",
+					tone: "neg",
+					formality: "business",
+					keywords: {
+						features: ["Dashboard", "Export API", "User Management"],
+						technical: ["timeout", "authentication", "database error"],
+						errors: ["500 Internal Server", "TIMEOUT_ERROR"]
+					},
+					min: 80,
+					max: 300
+				}),
+				priority: ["low", "medium", "high", "critical"]
+			}
+		}
+
+		Example for social media platform:
+
+		{
+			event: "comment_posted",
+			properties: {
+				comment_text: createGenerator({
+					style: "chat",
+					tone: "pos",
+					formality: "casual",
+					keywords: {
+						products: ["the app", "this feature"],
+						emotions: ["love it", "amazing", "great job"]
+					},
+					typos: true,
+					typoRate: 0.03,
+					min: 10,
+					max: 150
+				}),
+				post_type: ["text", "image", "video"]
+			}
+		}
+
+		Text Generator Options:
+
+		- style: "support", "review", "forum", "search", "feedback", "chat", "comments", "tweet", "email"
+		- tone: "pos" (positive), "neg" (negative), "neu" (neutral)
+		- formality: "casual", "business", "technical"
+		- keywords: Object with arrays of domain-specific terms to include (features, products, technical, errors, etc.)
+		- min/max: Character length range
+		- typos: true/false (adds realistic typos)
+		- typoRate: 0.01 to 0.1 (percentage of typos)
+		- mixedSentiment: true/false (varies sentiment within text)
+		- authenticityLevel: 0.0 to 1.0 (how "real" the text sounds)
+		- specificityLevel: 0.0 to 1.0 (level of detail)
+
+		When to Use createGenerator:
+
+		- Customer support tickets and responses
+		- Product reviews and ratings
+		- Social media posts (tweets, LinkedIn, Reddit)
+		- Forum discussions and comments
+		- Chat messages and live support
+		- Search queries
+		- User feedback and feature requests
+		- Email communications
+		- Bug reports
+		- Any scenario involving user-written or generated text content
 
 ❌ Critical Rules to Follow:
 

package/lib/templates/scratch-dungeon-template.js

@@ -8,7 +8,7 @@ dayjs.extend(utc);
 import "dotenv/config";
 import { weighNumRange, range, date, initChance, exhaust, choose, integer, decimal, odds } from "../utils/utils.js";
 const { NODE_ENV = "unknown" } = process.env;
-
+import * as u from '../utils/utils.js'
 
 
 /**

package/lib/templates/textQuickTest.js

@@ -0,0 +1,172 @@
+import { createGenerator, generateBatch } from "../generators/text.js";
+
+function main() {
+	// Enterprise support ticket generator with keywords and high authenticity
+	const enterpriseSupportGen = createGenerator({
+		style: "support",
+		tone: "neg",
+		intensity: "high",
+		formality: "technical",
+		keywords: {
+			features: ['Dashboard Analytics', 'Export API', 'SSO Login', 'Admin Console', 'User Management'],
+			products: ['DataViz Pro', 'Enterprise Suite', 'v3.2.1', 'v2.8.4'],
+			technical: ['CORS error', 'timeout', 'memory leak', 'authentication', 'database'],
+			errors: ['ERR_CONNECTION_REFUSED', '500 Internal Server', 'TIMEOUT_ERROR', 'AUTH_FAILED'],
+			competitors: ['Tableau', 'PowerBI', 'Looker', 'Qlik']
+		},
+		mixedSentiment: true,
+		authenticityLevel: 0.7,
+		typos: true,
+		typoRate: 0.02,
+		specificityLevel: 0.8,
+		min: 80,
+		max: 300,
+		includeMetadata: false,
+		// System is now always optimized for speed + uniqueness
+	});
+
+	const twentyEnterpriseSupport = enterpriseSupportGen.generateBatch({ n: 20 });
+
+	// Casual review generator with typos and mixed sentiment
+	const casualReviewGen = createGenerator({
+		style: "review",
+		tone: "pos",
+		intensity: "medium",
+		formality: "casual",
+		keywords: {
+			features: ['user interface', 'mobile app', 'notifications', 'search function'],
+			products: ['the app', 'this tool', 'the platform'],
+			metrics: ['response time', 'uptime', 'user experience']
+		},
+		mixedSentiment: true,
+		authenticityLevel: 0.4,
+		typos: true,
+		typoRate: 0.03,
+		sentimentDrift: 0.3,
+		min: 30,
+		max: 200,
+		includeMetadata: false
+	});
+
+	const twentyCasualReviews = casualReviewGen.generateBatch({ n: 20 });
+
+	// Technical forum posts with advanced features
+	const technicalForumGen = createGenerator({
+		style: "forum",
+		tone: "neu",
+		formality: "technical",
+		keywords: {
+			technical: ['REST API', 'GraphQL', 'webhooks', 'microservices', 'cloud infrastructure'],
+			versions: ['v1.2.3', 'latest', 'beta', 'stable release'],
+			errors: ['404 Not Found', 'Rate Limiting', 'SSL Certificate']
+		},
+		userPersona: true,
+		timestamps: true,
+		authenticityLevel: 0.6,
+		specificityLevel: 0.9,
+		min: 20,
+		max: 250,
+		includeMetadata: false
+	});
+
+	const twentyTechForms = technicalForumGen.generateBatch({ n: 20 });
+
+	// Search queries with realistic typos
+	const searchQueryGen = createGenerator({
+		style: "search",
+		tone: "neu",
+		formality: "casual",
+		keywords: {
+			features: ['export data', 'user settings', 'help docs', 'pricing'],
+			technical: ['API documentation', 'integration guide', 'troubleshooting']
+		},
+		typos: true,
+		typoRate: 0.05,
+		min: 2,
+		max: 50,
+		includeMetadata: false
+	});
+
+	const twentySearch = searchQueryGen.generateBatch({ n: 20 });
+
+	// Business feedback with professional tone
+	const businessFeedbackGen = createGenerator({
+		style: "feedback",
+		tone: "neu",
+		formality: "business",
+		keywords: {
+			metrics: ['ROI', 'efficiency', 'cost savings', 'productivity'],
+			features: ['reporting', 'analytics', 'integration capabilities']
+		},
+		authenticityLevel: 0.3,
+		specificityLevel: 0.7,
+		min: 40,
+		max: 180,
+		includeMetadata: false
+	});
+
+	const twentyFeedback = businessFeedbackGen.generateBatch({ n: 20 });
+
+	// Chat messages with high authenticity and typos
+	const chatMessageGen = createGenerator({
+		style: "chat",
+		tone: "neu",
+		formality: "casual",
+		keywords: {
+			products: ['the app', 'dashboard', 'mobile version'],
+			features: ['notifications', 'sync', 'offline mode']
+		},
+		mixedSentiment: true,
+		authenticityLevel: 0.8,
+		typos: true,
+		typoRate: 0.04,
+		sentimentDrift: 0.4,
+		min: 5,
+		max: 150,
+		includeMetadata: false
+	});
+
+	const twentyChatMsg = chatMessageGen.generateBatch({ n: 20 });
+
+	// Email communication generator
+	const emailGen = createGenerator({
+		style: "email",
+		tone: "neu",
+		formality: "business",
+		keywords: {
+			features: ['account management', 'billing', 'subscription'],
+			products: ['Enterprise Plan', 'Pro Account']
+		},
+		authenticityLevel: 0.5,
+		userPersona: true,
+		min: 60,
+		max: 300,
+		includeMetadata: false
+	});
+
+	const twentyEmails = emailGen.generateBatch({ n: 20 });
+
+	return {
+		email: twentyEmails,
+		chat: twentyChatMsg,
+		feedback: twentyFeedback,
+		search: twentySearch,
+		tech: twentyTechForms,
+		casual: twentyCasualReviews,
+		enterprise: twentyEnterpriseSupport
+	};
+
+}
+
+
+const result = main();
+
+for (const key in result) {
+    console.log(`${key?.toUpperCase()}`);
+	console.log(`----------`)
+	const toShow = result[key]?.join("\n\n");
+	console.log(toShow)
+	console.log(`----------\n\n`)
+}
+
+// debugger;

package/lib/utils/ai.js

@@ -21,12 +21,28 @@ CURRENT_PROMPT = `build me a dungeon stream with these events and structure
 but use all the different mcdonalds products as a possible values`;
 CURRENT_PROMPT = ``;
 
-
+export async function ask(question) {
+	const prompt = question;
+	const ai = new AITransformer({
+		apiKey: API_KEY,
+		onlyJSON: false,
+		responseSchema: {
+			type: "string"
+		},
+		modelName: "gemini-2.5-flash-lite"
+	});
+	await ai.init();
+	const response = await ai.message(prompt);
+	// if (NODE_ENV === "dev") {
+	// 	debugger;
+	// }
+	return response?.toString()?.trim();
+}
 
 async function main(params) {
 	const { prompt } = params;
 	if (!prompt) throw new Error("Please provide a prompt");
-	let INSTRUCTIONS = await u.load('./lib/templates/instructions.txt', false);
+	let INSTRUCTIONS = await u.load('./lib/templates/schema-instructions.txt', false);
 	const TYPES = await u.load('./lib/templates/abbreviated.d.ts', false);
 	const VERBOSE_SCHEMA_FILE = await u.load('./lib/templates/verbose-schema.js', false);
 	const VERBOSE_SCHEMA = VERBOSE_SCHEMA_FILE.split(`//SPLIT HERE`).pop()?.trim() || ``;
@@ -53,6 +69,39 @@ async function main(params) {
 
 }
 
+/**
+ * Generate AI hooks based on current schema and user's desired trends
+ * @param {object} params - Parameters object
+ * @param {string} params.prompt - User's description of desired trends
+ * @param {object} params.currentSchema - The existing dungeon schema
+ * @returns {Promise<string>} Generated hook function code
+ */
+export async function generateAIHooks(params) {
+	const { prompt, currentSchema } = params;
+	if (!prompt) throw new Error("Please provide a prompt describing the trends you want");
+	if (!currentSchema) throw new Error("Please provide the current schema");
+
+	// Load hooks instructions template
+	let HOOKS_INSTRUCTIONS = await u.load('./lib/templates/hooks-instructions.txt', false);
+
+	// Format the current schema as a readable string
+	const schemaString = JSON.stringify(currentSchema, null, 2);
+
+	// Replace placeholder with actual schema
+	HOOKS_INSTRUCTIONS = HOOKS_INSTRUCTIONS.replace(/<CURRENT_SCHEMA>/g, schemaString);
+
+	const ai = new AITransformer({
+		apiKey: API_KEY,
+		onlyJSON: false,
+		systemInstructions: HOOKS_INSTRUCTIONS?.trim(),
+		modelName: "gemini-2.5-pro",
+	});
+
+	await ai.init();
+	const response = await ai.message(prompt);
+
+	return response?.toString()?.trim();
+}
 
 export default main;
 

package/lib/utils/utils.js

@@ -8,10 +8,11 @@ import utc from 'dayjs/plugin/utc.js';
 import path from 'path';
 import { mkdir, parseGCSUri } from 'ak-tools';
 import { existsSync } from 'fs';
+import zlib from 'zlib';
 dayjs.extend(utc);
 import 'dotenv/config';
 import { domainSuffix, domainPrefix } from '../templates/defaults.js';
-const {NODE_ENV = "unknown"} = process.env;
+const { NODE_ENV = "unknown" } = process.env;
 
 /** @typedef {import('../../types').Dungeon} Config */
 /** @typedef {import('../../types').EventConfig} EventConfig */
@@ -207,6 +208,14 @@ function choose(value) {
 
 	}
 
+	// if the thing has a .next() method, call that
+	try {
+		if (value && typeof value.next === 'function') {
+			return value.next();
+		}
+	} catch (e) {
+		console.error(`Error occurred while calling next(): ${e}`);
+	}
 
 	try {
 		// Keep resolving the value if it's a function (with caching)
@@ -483,15 +492,22 @@ STREAMERS
 ----
 */
 
-function streamJSON(filePath, data) {
+function streamJSON(filePath, data, options = {}) {
 	return new Promise((resolve, reject) => {
 		let writeStream;
+		const { gzip = false } = options;
+
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
 			writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
 		}
 		else {
 			writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
+			if (gzip) {
+				const gzipStream = zlib.createGzip();
+				gzipStream.pipe(writeStream);
+				writeStream = gzipStream;
+			}
 		}
 		data.forEach(item => {
 			writeStream.write(JSON.stringify(item) + '\n');
@@ -504,15 +520,22 @@ function streamJSON(filePath, data) {
 	});
 }
 
-function streamCSV(filePath, data) {
+function streamCSV(filePath, data, options = {}) {
 	return new Promise((resolve, reject) => {
 		let writeStream;
+		const { gzip = false } = options;
+
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
 			writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
 		}
 		else {
 			writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
+			if (gzip) {
+				const gzipStream = zlib.createGzip();
+				gzipStream.pipe(writeStream);
+				writeStream = gzipStream;
+			}
 		}
 
 		// Extract all unique keys from the data array
@@ -539,6 +562,120 @@ function streamCSV(filePath, data) {
 	});
 }
 
+async function streamParquet(filePath, data, options = {}) {
+	const { gzip = false } = options;
+
+	// Dynamically import hyparquet-writer
+	// @ts-ignore
+	const { parquetWriteFile, parquetWriteBuffer } = await import('hyparquet-writer');
+
+	if (data.length === 0) {
+		throw new Error('Cannot write parquet file with empty data');
+	}
+
+	// Extract column names and data from the input array
+	const columns = getUniqueKeys(data);
+	const columnData = columns.map(columnName => {
+		const columnValues = data.map(row => {
+			let value = row[columnName];
+
+			// Handle null/undefined values
+			if (value === null || value === undefined) {
+				return null;
+			}
+
+			// Convert objects to strings
+			if (typeof value === 'object') {
+				value = JSON.stringify(value);
+			}
+
+			return value;
+		});
+
+		// Determine the type based on the first non-null value
+		let type = 'STRING'; // default
+		const firstValue = columnValues.find(v => v !== null && v !== undefined);
+
+		if (firstValue !== undefined) {
+			if (typeof firstValue === 'boolean') {
+				type = 'BOOLEAN';
+			} else if (typeof firstValue === 'number') {
+				// For parquet compatibility, convert numbers to appropriate types
+				if (Number.isInteger(firstValue)) {
+					// Use INT32 for smaller integers, convert to BigInt for INT64 if needed
+					if (firstValue >= -2147483648 && firstValue <= 2147483647) {
+						type = 'INT32';
+					} else {
+						type = 'INT64';
+						// Convert all values to BigInt for INT64
+						for (let i = 0; i < columnValues.length; i++) {
+							if (columnValues[i] !== null && columnValues[i] !== undefined) {
+								columnValues[i] = BigInt(columnValues[i]);
+							}
+						}
+					}
+				} else {
+					type = 'DOUBLE';
+				}
+			} else if (firstValue instanceof Date) {
+				type = 'TIMESTAMP';
+			}
+		}
+
+		return {
+			name: columnName,
+			data: columnValues,
+			type: type
+		};
+	});
+
+	if (filePath?.startsWith('gs://')) {
+		// For GCS, write to buffer first, then upload
+		// @ts-ignore
+		const arrayBuffer = parquetWriteBuffer({ columnData });
+		const { bucket, file } = parseGCSUri(filePath);
+
+		const writeStream = storage.bucket(bucket).file(file).createWriteStream({
+			gzip: gzip || true // Always gzip for GCS
+		});
+
+		return new Promise((resolve, reject) => {
+			writeStream.write(Buffer.from(arrayBuffer));
+			writeStream.end();
+			writeStream.on('finish', () => resolve(filePath));
+			writeStream.on('error', reject);
+		});
+	} else {
+		// For local files
+		let actualFilePath = filePath;
+		if (gzip && !filePath.endsWith('.gz')) {
+			actualFilePath = filePath + '.gz';
+		}
+
+		if (gzip) {
+			// Write to buffer then gzip to disk
+			// @ts-ignore
+			const arrayBuffer = parquetWriteBuffer({ columnData });
+			const buffer = Buffer.from(arrayBuffer);
+			const gzippedBuffer = zlib.gzipSync(buffer);
+
+			return new Promise((resolve, reject) => {
+				fs.writeFile(actualFilePath, gzippedBuffer, (err) => {
+					if (err) reject(err);
+					else resolve(actualFilePath);
+				});
+			});
+		} else {
+			// Direct write to disk
+			parquetWriteFile({
+				filename: filePath,
+				columnData
+			});
+			return Promise.resolve(filePath);
+		}
+	}
+}
+
 
 /*
 ----
@@ -848,7 +985,7 @@ function validEvent(row) {
 	if (!row.time) return false;
 	if (!row.device_id && !row.user_id) return false;
 	if (!row.insert_id) return false;
-	if (!row.source) return false;
+	// if (!row.source) return false;
 	if (typeof row.time !== 'string') return false;
 	return true;
 }
@@ -870,7 +1007,7 @@ function buildFileNames(config) {
 	let extension = "";
 	extension = format === "csv" ? "csv" : "json";
 	// const current = dayjs.utc().format("MM-DD-HH");
-	let simName = config.simulationName;
+	let simName = config.name;
 	let writeDir = typeof config.writeToDisk === 'string' ? config.writeToDisk : "./";
 	if (config.writeToDisk) {
 		const dataFolder = path.resolve("./data");
@@ -1235,7 +1372,7 @@ function wrapFunc(obj, func, recursion = 0, parentKey = null, grandParentKey = n
 // }
 
 const chance = getChance();
-function odds(num) {	
+function odds(num) {
 	return chance.bool({ likelihood: num });
 }
 
@@ -1328,7 +1465,7 @@ export {
 	TimeSoup,
 	companyName,
 	generateEmoji,
-	hasSameKeys as haveSameKeys,
+	hasSameKeys,
 	deepClone,
 	initChance,
 	getChance,
@@ -1362,6 +1499,7 @@ export {
 	buildFileNames,
 	streamJSON,
 	streamCSV,
+	streamParquet,
 	datesBetween,
 	weighChoices,
 	wrapFunc,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "make-mp-data",
-	"version": "2.0.22",
+	"version": "2.1.0",
 	"description": "builds all mixpanel primitives for a given project",
 	"type": "module",
 	"main": "index.js",
@@ -31,7 +31,8 @@
 		"test:quick": "node ./tests/benchmark/quick-test.mjs",
 		"exp:soup": "node ./tests/testSoup.mjs",
 		"func:local": "functions-framework --target=entry",
-		"func:deploy": "./scripts/deploy.sh"
+		"func:deploy": "./scripts/deploy.sh",
+		"textGen": "node ./lib/templates/textQuickTest.js"
 	},
 	"repository": {
 		"type": "git",
@@ -65,13 +66,15 @@
 		"ak-gemini": "^1.0.59",
 		"ak-tools": "^1.1.12",
 		"chance": "^1.1.11",
-		"chart.js": "^3.9.1",
-		"chartjs-node-canvas": "^4.1.6",
 		"dayjs": "^1.11.11",
 		"dotenv": "^16.4.5",
 		"google-auth-library": "^9.15.0",
-		"mixpanel-import": "^2.8.162",
+		"hyparquet-writer": "^0.6.1",
+		"mixpanel-import": "^3.0.0",
 		"p-limit": "^3.1.0",
+		"seedrandom": "^3.0.5",
+		"sentiment": "^5.0.2",
+		"tracery-grammar": "^2.8.4",
 		"yargs": "^17.7.2"
 	},
 	"devDependencies": {