make-mp-data 2.0.21 → 2.0.23

package/lib/utils/utils.js

@@ -8,9 +8,11 @@ import utc from 'dayjs/plugin/utc.js';
 import path from 'path';
 import { mkdir, parseGCSUri } from 'ak-tools';
 import { existsSync } from 'fs';
+import zlib from 'zlib';
 dayjs.extend(utc);
 import 'dotenv/config';
 import { domainSuffix, domainPrefix } from '../templates/defaults.js';
+const {NODE_ENV = "unknown"} = process.env;
 
 /** @typedef {import('../../types').Dungeon} Config */
 /** @typedef {import('../../types').EventConfig} EventConfig */
@@ -482,15 +484,22 @@ STREAMERS
 ----
 */
 
-function streamJSON(filePath, data) {
+function streamJSON(filePath, data, options = {}) {
 	return new Promise((resolve, reject) => {
 		let writeStream;
+		const { gzip = false } = options;
+		
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
 			writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
 		}
 		else {
 			writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
+			if (gzip) {
+				const gzipStream = zlib.createGzip();
+				gzipStream.pipe(writeStream);
+				writeStream = gzipStream;
+			}
 		}
 		data.forEach(item => {
 			writeStream.write(JSON.stringify(item) + '\n');
@@ -503,15 +512,22 @@ function streamJSON(filePath, data) {
 	});
 }
 
-function streamCSV(filePath, data) {
+function streamCSV(filePath, data, options = {}) {
 	return new Promise((resolve, reject) => {
 		let writeStream;
+		const { gzip = false } = options;
+		
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
 			writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
 		}
 		else {
 			writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
+			if (gzip) {
+				const gzipStream = zlib.createGzip();
+				gzipStream.pipe(writeStream);
+				writeStream = gzipStream;
+			}
 		}
 
 		// Extract all unique keys from the data array
@@ -538,6 +554,117 @@ function streamCSV(filePath, data) {
 	});
 }
 
+async function streamParquet(filePath, data, options = {}) {
+	const { gzip = false } = options;
+	
+	// Dynamically import hyparquet-writer
+	const { parquetWriteFile, parquetWriteBuffer } = await import('hyparquet-writer');
+	
+	if (data.length === 0) {
+		throw new Error('Cannot write parquet file with empty data');
+	}
+
+	// Extract column names and data from the input array
+	const columns = getUniqueKeys(data);
+	const columnData = columns.map(columnName => {
+		const columnValues = data.map(row => {
+			let value = row[columnName];
+			
+			// Handle null/undefined values
+			if (value === null || value === undefined) {
+				return null;
+			}
+			
+			// Convert objects to strings
+			if (typeof value === 'object') {
+				value = JSON.stringify(value);
+			}
+			
+			return value;
+		});
+		
+		// Determine the type based on the first non-null value
+		let type = 'STRING'; // default
+		const firstValue = columnValues.find(v => v !== null && v !== undefined);
+		
+		if (firstValue !== undefined) {
+			if (typeof firstValue === 'boolean') {
+				type = 'BOOLEAN';
+			} else if (typeof firstValue === 'number') {
+				// For parquet compatibility, convert numbers to appropriate types
+				if (Number.isInteger(firstValue)) {
+					// Use INT32 for smaller integers, convert to BigInt for INT64 if needed
+					if (firstValue >= -2147483648 && firstValue <= 2147483647) {
+						type = 'INT32';
+					} else {
+						type = 'INT64';
+						// Convert all values to BigInt for INT64
+						for (let i = 0; i < columnValues.length; i++) {
+							if (columnValues[i] !== null && columnValues[i] !== undefined) {
+								columnValues[i] = BigInt(columnValues[i]);
+							}
+						}
+					}
+				} else {
+					type = 'DOUBLE';
+				}
+			} else if (firstValue instanceof Date) {
+				type = 'TIMESTAMP';
+			}
+		}
+		
+		return {
+			name: columnName,
+			data: columnValues,
+			type: type
+		};
+	});
+
+	if (filePath?.startsWith('gs://')) {
+		// For GCS, write to buffer first, then upload
+		const arrayBuffer = parquetWriteBuffer({ columnData });
+		const { bucket, file } = parseGCSUri(filePath);
+		
+		const writeStream = storage.bucket(bucket).file(file).createWriteStream({ 
+			gzip: gzip || true // Always gzip for GCS
+		});
+		
+		return new Promise((resolve, reject) => {
+			writeStream.write(Buffer.from(arrayBuffer));
+			writeStream.end();
+			writeStream.on('finish', () => resolve(filePath));
+			writeStream.on('error', reject);
+		});
+	} else {
+		// For local files
+		let actualFilePath = filePath;
+		if (gzip && !filePath.endsWith('.gz')) {
+			actualFilePath = filePath + '.gz';
+		}
+		
+		if (gzip) {
+			// Write to buffer then gzip to disk
+			const arrayBuffer = parquetWriteBuffer({ columnData });
+			const buffer = Buffer.from(arrayBuffer);
+			const gzippedBuffer = zlib.gzipSync(buffer);
+			
+			return new Promise((resolve, reject) => {
+				fs.writeFile(actualFilePath, gzippedBuffer, (err) => {
+					if (err) reject(err);
+					else resolve(actualFilePath);
+				});
+			});
+		} else {
+			// Direct write to disk
+			parquetWriteFile({
+				filename: filePath,
+				columnData
+			});
+			return Promise.resolve(filePath);
+		}
+	}
+}
+
 
 /*
 ----
@@ -545,13 +672,7 @@ WEIGHERS
 ----
 */
 
-function weighFunnels(acc, funnel) {
-	const weight = funnel?.weight || 1;
-	for (let i = 0; i < weight; i++) {
-		acc.push(funnel);
-	}
-	return acc;
-}
+
 
 /**
  * a utility function to generate a range of numbers within a given skew
@@ -875,7 +996,7 @@ function buildFileNames(config) {
 	let extension = "";
 	extension = format === "csv" ? "csv" : "json";
 	// const current = dayjs.utc().format("MM-DD-HH");
-	let simName = config.simulationName;
+	let simName = config.name;
 	let writeDir = typeof config.writeToDisk === 'string' ? config.writeToDisk : "./";
 	if (config.writeToDisk) {
 		const dataFolder = path.resolve("./data");
@@ -1020,6 +1141,46 @@ let soupHits = 0;
  * @param  {number} [peaks=5]
  */
 function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0) {
+	if (!earliestTime) earliestTime = global.FIXED_BEGIN ? global.FIXED_BEGIN : dayjs().subtract(30, 'd').unix(); // 30 days ago
+	if (!latestTime) latestTime = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
+	const chance = getChance();
+	const totalRange = latestTime - earliestTime;
+	const chunkSize = totalRange / peaks;
+
+	// Select a random chunk based on the number of peaks
+	const peakIndex = integer(0, peaks - 1);
+	const chunkStart = earliestTime + peakIndex * chunkSize;
+	const chunkEnd = chunkStart + chunkSize;
+	const chunkMid = (chunkStart + chunkEnd) / 2;
+
+	// Generate a single timestamp within this chunk using a normal distribution centered at chunkMid
+	let offset;
+	let iterations = 0;
+	let isValidTime = false;
+	do {
+		iterations++;
+		soupHits++;
+		offset = chance.normal({ mean: mean, dev: chunkSize / deviation });
+		isValidTime = validTime(chunkMid + offset, earliestTime, latestTime);
+		if (iterations > 25000) {
+			throw `${iterations} iterations... exceeded`;
+		}
+	} while (chunkMid + offset < chunkStart || chunkMid + offset > chunkEnd);
+
+	try {
+		return dayjs.unix(chunkMid + offset).toISOString();
+	}
+
+	catch (e) {
+		//escape hatch
+		// console.log('BAD TIME', e?.message);
+		if (NODE_ENV === 'dev') debugger;
+		return dayjs.unix(integer(earliestTime, latestTime)).toISOString();
+	}
+}
+
+
+function NewTimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0) {
 	if (!earliestTime) earliestTime = global.FIXED_BEGIN ? global.FIXED_BEGIN : dayjs().subtract(30, 'd').unix(); // 30 days ago
 	if (!latestTime) latestTime = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
 	const chance = getChance();
@@ -1199,7 +1360,10 @@ function wrapFunc(obj, func, recursion = 0, parentKey = null, grandParentKey = n
 
 // }
 
-
+const chance = getChance();
+function odds(num) {	
+	return chance.bool({ likelihood: num });
+}
 
 /**
  * makes a random-sized array of emojis
@@ -1290,12 +1454,12 @@ export {
 	TimeSoup,
 	companyName,
 	generateEmoji,
-	hasSameKeys as haveSameKeys,
+	hasSameKeys,
 	deepClone,
 	initChance,
 	getChance,
 	decimal,
-
+	odds,
 	validTime,
 	validEvent,
 
@@ -1311,7 +1475,6 @@ export {
 	pickAWinner,
 	quickHash,
 	weighArray,
-	weighFunnels,
 	validateEventConfig,
 	shuffleArray,
 	shuffleExceptFirst,
@@ -1325,6 +1488,7 @@ export {
 	buildFileNames,
 	streamJSON,
 	streamCSV,
+	streamParquet,
 	datesBetween,
 	weighChoices,
 	wrapFunc,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "make-mp-data",
-	"version": "2.0.21",
+	"version": "2.0.23",
 	"description": "builds all mixpanel primitives for a given project",
 	"type": "module",
 	"main": "index.js",
@@ -63,14 +63,15 @@
 		"@google-cloud/storage": "^7.14.0",
 		"ak-fetch": "^2.0.12",
 		"ak-gemini": "^1.0.59",
-		"ak-tools": "^1.1.1",
+		"ak-tools": "^1.1.12",
 		"chance": "^1.1.11",
 		"chart.js": "^3.9.1",
 		"chartjs-node-canvas": "^4.1.6",
 		"dayjs": "^1.11.11",
 		"dotenv": "^16.4.5",
 		"google-auth-library": "^9.15.0",
-		"mixpanel-import": "^2.8.15",
+		"hyparquet-writer": "^0.6.1",
+		"mixpanel-import": "^2.8.162",
 		"p-limit": "^3.1.0",
 		"yargs": "^17.7.2"
 	},
@@ -86,4 +87,4 @@
 			"tmp/"
 		]
 	}
-}
+}

package/types.d.ts

@@ -20,7 +20,7 @@ export interface Dungeon {
     epochEnd?: number;
     numEvents?: number;
     numUsers?: number;
-    format?: "csv" | "json" | string;
+    format?: "csv" | "json" | "parquet" | string;
     region?: "US" | "EU";
     concurrency?: number;
     batchSize?: number;
@@ -30,7 +30,6 @@ export interface Dungeon {
     projectId?: string;
 
     // ids
-    simulationName?: string;
     name?: string;
 
     //switches
@@ -44,6 +43,7 @@ export interface Dungeon {
     hasDesktopDevices?: boolean;
     hasBrowser?: boolean;
     writeToDisk?: boolean | string;
+    gzip?: boolean;
     verbose?: boolean;
     hasAnonIds?: boolean;
     hasSessionIds?: boolean;
@@ -97,7 +97,7 @@ export type hookTypes =
     | "user"
     | "group"
     | "lookup"
-    // | "scd"
+    | "scd"
     | "scd-pre"
     | "mirror"
     | "funnel-pre"
@@ -117,7 +117,7 @@ export interface hookArrayOptions<T> {
     hook?: Hook<T>;
     type?: hookTypes;
     filename?: string;
-    format?: "csv" | "json" | string;
+    format?: "csv" | "json" | "parquet" | string;
     concurrency?: number;
     context?: Context;
     [key: string]: any;
@@ -311,6 +311,11 @@ export interface Funnel {
      * funnel properties go onto each event in the funnel and are held constant
      */
     props?: Record<string, ValueValid>;
+	/**
+	 * funnel conditions (user properties) are used to filter users who are eligible for the funnel
+	 * these conditions must match the current user's profile for the user to be eligible for the funnel
+	 */
+	conditions?: Record<string, ValueValid>;
 }
 
 /**