make-mp-data 1.4.2 → 1.4.3

package/{utils.js → core/utils.js}

@@ -7,20 +7,24 @@ const dayjs = require('dayjs');
 const utc = require('dayjs/plugin/utc');
 const path = require('path');
 const { mkdir } = require('ak-tools');
+const { existsSync } = require('fs');
 dayjs.extend(utc);
 require('dotenv').config();
+const { domainSuffix, domainPrefix } = require('./defaults');
 
-/** @typedef {import('./types').Config} Config */
-/** @typedef {import('./types').EventConfig} EventConfig */
-/** @typedef {import('./types').ValueValid} ValueValid */
-/** @typedef {import('./types').EnrichedArray} EnrichArray */
-/** @typedef {import('./types').EnrichArrayOptions} EnrichArrayOptions */
-/** @typedef {import('./types').Person} Person */
-/** @typedef {import('./types').Funnel} Funnel */
+/** @typedef {import('../types').Config} Config */
+/** @typedef {import('../types').EventConfig} EventConfig */
+/** @typedef {import('../types').ValueValid} ValueValid */
+/** @typedef {import('../types').EnrichedArray} EnrichArray */
+/** @typedef {import('../types').EnrichArrayOptions} EnrichArrayOptions */
+/** @typedef {import('../types').Person} Person */
+/** @typedef {import('../types').Funnel} Funnel */
 
 let globalChance;
 let chanceInitialized = false;
 
+
+
 /*
 ----
 RNG
@@ -127,6 +131,21 @@ function dates(inTheLast = 30, numPairs = 5, format = 'YYYY-MM-DD') {
 	return pairs;
 };
 
+function datesBetween(start, end) {
+	const result = [];
+	if (typeof start === 'number') start = dayjs.unix(start).utc();
+	if (typeof start !== 'number') start = dayjs(start).utc();
+	if (typeof end === 'number') end = dayjs.unix(end).utc();
+	if (typeof end !== 'number') end = dayjs(end).utc();
+	const diff = end.diff(start, 'day');
+	for (let i = 0; i < diff; i++) {
+		const day = start.add(i, 'day').startOf('day').add(12, 'hour');
+		result.push(day.toISOString());
+	}
+
+	return result;
+}
+
 /**
  * returns a random date
  * @param  {any} start
@@ -223,77 +242,66 @@ function integer(min = 1, max = 100) {
 };
 
 
+/**
+ * Creates a function that generates a weighted list of items
+ * with a higher likelihood of picking a specified index and clear second and third place indices.
+ * 
+ * @param {Array} items - The list of items to pick from.
+ * @param {number} [mostChosenIndex] - The index of the item to be most favored.
+ * @returns {function} - A function that returns a weighted list of items.
+ */
 function pickAWinner(items, mostChosenIndex) {
 	const chance = getChance();
-	if (!mostChosenIndex) mostChosenIndex = integer(0, items.length);
-	if (mostChosenIndex > items.length) mostChosenIndex = items.length;
+
+	// Ensure mostChosenIndex is within the bounds of the items array
+	if (!items) return () => { return ""; };
+	if (!items.length) return () => { return ""; };
+	if (!mostChosenIndex) mostChosenIndex = chance.integer({ min: 0, max: items.length - 1 });
+	if (mostChosenIndex >= items.length) mostChosenIndex = items.length - 1;
+
+	// Calculate second and third most chosen indices
+	const secondMostChosenIndex = (mostChosenIndex + 1) % items.length;
+	const thirdMostChosenIndex = (mostChosenIndex + 2) % items.length;
+
+	// Return a function that generates a weighted list
 	return function () {
 		const weighted = [];
 		for (let i = 0; i < 10; i++) {
-			if (chance.bool({ likelihood: integer(10, 35) })) {
+			const rand = chance.d10(); // Random number between 1 and 10
+
+			// 35% chance to favor the most chosen index
+			if (chance.bool({ likelihood: 35 })) {
+				// 50% chance to slightly alter the index
 				if (chance.bool({ likelihood: 50 })) {
 					weighted.push(items[mostChosenIndex]);
-				}
-				else {
-					const rand = chance.d10();
+				} else {
 					const addOrSubtract = chance.bool({ likelihood: 50 }) ? -rand : rand;
 					let newIndex = mostChosenIndex + addOrSubtract;
+
+					// Ensure newIndex is within bounds
 					if (newIndex < 0) newIndex = 0;
-					if (newIndex > items.length) newIndex = items.length;
+					if (newIndex >= items.length) newIndex = items.length - 1;
 					weighted.push(items[newIndex]);
 				}
 			}
+			// 25% chance to favor the second most chosen index
+			else if (chance.bool({ likelihood: 25 })) {
+				weighted.push(items[secondMostChosenIndex]);
+			}
+			// 15% chance to favor the third most chosen index
+			else if (chance.bool({ likelihood: 15 })) {
+				weighted.push(items[thirdMostChosenIndex]);
+			}
+			// Otherwise, pick a random item from the list
 			else {
 				weighted.push(chance.pickone(items));
 			}
 		}
 		return weighted;
-
 	};
 }
 
 
-function inferFunnels(events) {
-	const createdFunnels = [];
-	const firstEvents = events.filter((e) => e.isFirstEvent).map((e) => e.event);
-	const usageEvents = events.filter((e) => !e.isFirstEvent).map((e) => e.event);
-	const numFunnelsToCreate = Math.ceil(usageEvents.length);
-	/** @type {Funnel} */
-	const funnelTemplate = {
-		sequence: [],
-		conversionRate: 50,
-		order: 'sequential',
-		requireRepeats: false,
-		props: {},
-		timeToConvert: 1,
-		isFirstFunnel: false,
-		weight: 1
-	};
-	if (firstEvents.length) {
-		for (const event of firstEvents) {
-			createdFunnels.push({ ...clone(funnelTemplate), sequence: [event], isFirstFunnel: true, conversionRate: 100 });
-		}
-	}
-
-	//at least one funnel with all usage events
-	createdFunnels.push({ ...clone(funnelTemplate), sequence: usageEvents });
-
-	//for the rest, make random funnels
-	followUpFunnels: for (let i = 1; i < numFunnelsToCreate; i++) {
-		/** @type {Funnel} */
-		const funnel = { ...clone(funnelTemplate) };
-		funnel.conversionRate = integer(25, 75);
-		funnel.timeToConvert = integer(1, 10);
-		funnel.weight = integer(1, 10);
-		const sequence = shuffleArray(usageEvents).slice(0, integer(2, usageEvents.length));
-		funnel.sequence = sequence;
-		funnel.order = 'random';
-		createdFunnels.push(funnel);
-	}
-
-	return createdFunnels;
-
-}
 
 /*
 ----
@@ -360,6 +368,53 @@ function range(a, b, step = 1) {
 };
 
 
+/**
+ * create funnels out of random events
+ * @param {EventConfig[]} events
+ */
+function inferFunnels(events) {
+	const createdFunnels = [];
+	const firstEvents = events.filter((e) => e.isFirstEvent).map((e) => e.event);
+	const usageEvents = events.filter((e) => !e.isFirstEvent).map((e) => e.event);
+	const numFunnelsToCreate = Math.ceil(usageEvents.length);
+	/** @type {Funnel} */
+	const funnelTemplate = {
+		sequence: [],
+		conversionRate: 50,
+		order: 'sequential',
+		requireRepeats: false,
+		props: {},
+		timeToConvert: 1,
+		isFirstFunnel: false,
+		weight: 1
+	};
+	if (firstEvents.length) {
+		for (const event of firstEvents) {
+			createdFunnels.push({ ...clone(funnelTemplate), sequence: [event], isFirstFunnel: true, conversionRate: 100 });
+		}
+	}
+
+	//at least one funnel with all usage events
+	createdFunnels.push({ ...clone(funnelTemplate), sequence: usageEvents });
+
+	//for the rest, make random funnels
+	followUpFunnels: for (let i = 1; i < numFunnelsToCreate; i++) {
+		/** @type {Funnel} */
+		const funnel = { ...clone(funnelTemplate) };
+		funnel.conversionRate = integer(25, 75);
+		funnel.timeToConvert = integer(1, 10);
+		funnel.weight = integer(1, 10);
+		const sequence = shuffleArray(usageEvents).slice(0, integer(2, usageEvents.length));
+		funnel.sequence = sequence;
+		funnel.order = 'random';
+		createdFunnels.push(funnel);
+	}
+
+	return createdFunnels;
+
+}
+
+
 /*
 ----
 STREAMERS
@@ -425,8 +480,11 @@ function weighFunnels(acc, funnel) {
 /**
  * a utility function to generate a range of numbers within a given skew
  * Skew = 0.5: The values are more concentrated towards the extremes (both ends of the range) with a noticeable dip in the middle. The distribution appears more "U" shaped. Larger sizes result in smoother distributions but maintain the overall shape.
+ * 
  * Skew = 1: This represents the default normal distribution without skew. The values are normally distributed around the mean. Larger sizes create a clearer bell-shaped curve.
+ * 
  * Skew = 2: The values are more concentrated towards the mean, with a steeper drop-off towards the extremes. The distribution appears more peaked, resembling a "sharper" bell curve. Larger sizes enhance the clarity of this peaked distribution.
+ * 
  * Size represents the size of the pool to choose from; Larger sizes result in smoother distributions but maintain the overall shape.
  * @param  {number} min
  * @param  {number} max
@@ -517,6 +575,7 @@ function shuffleOutside(array) {
 }
 
 /**
+ * given a funnel, shuffle the events in the sequence with random events
  * @param  {EventConfig[]} funnel
  * @param  {EventConfig[]} possibles
  */
@@ -662,13 +721,18 @@ function buildFileNames(config) {
 	// const current = dayjs.utc().format("MM-DD-HH");
 	const simName = config.simulationName;
 	let writeDir = "./";
-	if (config.writeToDisk) writeDir = mkdir("./data");
+	if (config.writeToDisk) {
+		const dataFolder = path.resolve("./data");
+		if (existsSync(dataFolder)) writeDir = dataFolder;
+		else writeDir = path.resolve("./");
+	}
 	if (typeof writeDir !== "string") throw new Error("writeDir must be a string");
 	if (typeof simName !== "string") throw new Error("simName must be a string");
 
 	const writePaths = {
 		eventFiles: [path.join(writeDir, `${simName}-EVENTS.${extension}`)],
 		userFiles: [path.join(writeDir, `${simName}-USERS.${extension}`)],
+		adSpendFiles: [path.join(writeDir, `${simName}-AD-SPEND.${extension}`)],
 		scdFiles: [],
 		mirrorFiles: [],
 		groupFiles: [],
@@ -713,11 +777,19 @@ function buildFileNames(config) {
 	return writePaths;
 }
 
-
-function progress(thing, p) {
+/**
+ * @param  {[string, number][]} arrayOfArrays
+ */
+function progress(arrayOfArrays) {
 	// @ts-ignore
 	readline.cursorTo(process.stdout, 0);
-	process.stdout.write(`${thing} processed ... ${comma(p)}`);
+	let message = "";
+	for (const status of arrayOfArrays) {
+		const [thing, p] = status;
+		message += `${thing} processed: ${comma(p)}\t\t`;
+	}
+
+	process.stdout.write(message);
 };
 
 
@@ -823,19 +895,16 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 function person(userId, bornDaysAgo = 30, isAnonymous = false) {
 	const chance = getChance();
 	//names and photos
-	const l = chance.letter;
+	const l = chance.letter.bind(chance);
 	let gender = chance.pickone(['male', 'female']);
 	if (!gender) gender = "female";
 	// @ts-ignore
 	let first = chance.first({ gender });
 	let last = chance.last();
 	let name = `${first} ${last}`;
-	let email = `${first[0]}.${last}@${chance.domain()}.com`;
+	let email = `${first[0]}.${last}@${choose(domainPrefix)}.${choose(domainSuffix)}`;
 	let avatarPrefix = `https://randomuser.me/api/portraits`;
-	let randomAvatarNumber = chance.integer({
-		min: 1,
-		max: 99
-	});
+	let randomAvatarNumber = integer(1, 99);
 	let avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
 	let avatar = avatarPrefix + avPath;
 	let created = dayjs.unix(global.NOW).subtract(bornDaysAgo, 'day').format('YYYY-MM-DD');
@@ -855,8 +924,8 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false) {
 
 	if (isAnonymous) {
 		user.name = "Anonymous User";
-		user.email = `${l()}${l()}****${l()}${l()}@${l()}**${l()}*.com`;		
-		delete user.avatar;		
+		user.email = l() + l() + `*`.repeat(integer(3, 6)) + l() + `@` + l() + `*`.repeat(integer(3, 6)) + l() + `.` + choose(domainSuffix);
+		delete user.avatar;
 
 	}
 
@@ -980,5 +1049,6 @@ module.exports = {
 	buildFileNames,
 	streamJSON,
 	streamCSV,
-	inferFunnels
+	inferFunnels,
+	datesBetween
 };

package/package.json

@@ -1,26 +1,25 @@
 {
 	"name": "make-mp-data",
-	"version": "1.4.02",
+	"version": "1.4.03",
 	"description": "builds all mixpanel primitives for a given project",
 	"main": "index.js",
 	"types": "types.d.ts",
 	"scripts": {
-		"start": "node index.js",
+		"start": "node ./core/index.js",
 		"dev": "nodemon scratch.mjs --ignore /data",
-		"complex": "nodemon index.js --complex --e 10000 --u 100",
-		"simple": "nodemon index.js  --simple --e 10000 --u 100",
+		"complex": "nodemon ./core/index.js --complex --e 10000 --u 100",
+		"simple": "nodemon ./core/index.js  --simple --e 10000 --u 100",
 		"prune": "rm -f ./data/* && rm -f ./tmp/*",
 		"post": "npm publish",
 		"test": "NODE_ENV=test jest --runInBand",
-		"deps": "sh ./scripts/deps.sh",
-		"udacity": "nodemon index.js ./customers/udacity.js"
+		"deps": "sh ./scripts/deps.sh"
 	},
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/ak--47/make-mp-data.git"
 	},
 	"bin": {
-		"make-mp-data": "./index.js"
+		"make-mp-data": "./core/index.js"
 	},
 	"keywords": [
 		"mixpanel",
@@ -47,7 +46,7 @@
 		"chartjs-node-canvas": "^4.1.6",
 		"dayjs": "^1.11.11",
 		"dotenv": "^16.4.5",
-		"mixpanel-import": "^2.5.55",
+		"mixpanel-import": "^2.5.552",
 		"yargs": "^17.7.2"
 	},
 	"devDependencies": {

package/schemas/anon.js

@@ -0,0 +1,104 @@
+/**
+ * This is the default configuration file for the data generator in SIMPLE mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+
+
+const Chance = require('chance');
+const chance = new Chance();
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+const { uid, comma } = require('ak-tools');
+const { pickAWinner, weightedRange, date, integer } = require('../core/utils');
+
+
+
+/** @type {import('../types').Config} */
+const config = {
+	token: "",
+	seed: "foo bar",
+	numDays: 365, //how many days worth of data
+	numEvents: 100000, //how many events
+	numUsers: 10000, //how many users	
+	format: 'json', //csv or json
+	region: "US",
+	anonIds: true, //if true, anonymousIds are created for each user
+	sessionIds: false, //if true, sessionIds are created for each user
+	isAnonymous: true,
+	hasLocation: true, 
+	events: [
+		{
+			event: "foo",
+			weight: 10,
+			properties: {}
+		},
+		{
+			event: "bar",
+			weight: 9,
+			properties: {}
+		},
+		{
+			event: "baz",
+			weight: 8,
+			properties: {}
+		},
+		{
+			event: "qux",
+			weight: 7,
+			properties: {}
+		},
+		{
+			event: "garply",
+			weight: 6,
+			properties: {}
+		},
+		{
+			event: "durtle",
+			weight: 5,
+			properties: {}
+		},
+		{
+			event: "linny",
+			weight: 4,
+			properties: {}
+		},
+		{
+			event: "fonk",
+			weight: 3,
+			properties: {}
+		},
+		{
+			event: "crumn",
+			weight: 2,
+			properties: {}
+		},
+		{
+			event: "yak",
+			weight: 1,
+			properties: {}
+		}
+	],
+	superProps: {},
+	userProps: {
+		title: chance.profession.bind(chance),
+		luckyNumber: weightedRange(42, 420),
+		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
+	},
+	scdProps: {},
+	mirrorProps: {},
+	groupKeys: [],
+	groupProps: {},
+	lookupTables: [],
+	hook: function (record, type, meta) {
+		return record;
+	}
+};
+
+
+
+module.exports = config;

package/schemas/complex.js

@@ -8,7 +8,7 @@
 
 const Chance = require('chance');
 const chance = new Chance();
-const { weightedRange, date, integer } = require('../utils.js');
+const { weightedRange, date, integer } = require('../core/utils.js');
 const u = require('ak-tools');
 
 /** @type {import('../types.js').Config} */
@@ -23,6 +23,15 @@ const config = {
 	anonIds: true, //if true, anonymousIds are created for each user
 	sessionIds: true, //if true, sessionIds are created for each user
 
+	hasLocation: true,
+	hasAndroidDevices: true,
+	hasIOSDevices: true,
+	hasDesktopDevices: true,
+	hasBrowser: true,
+	hasCampaigns: true,
+	isAnonymous: false,
+	hasAdSpend: true,
+
 	events: [
 		{
 			"event": "checkout",
@@ -122,7 +131,7 @@ const config = {
 		}
 	],
 	superProps: {
-		device: deviceAttributes()
+		linked_device: deviceAttributes()
 		// emotions: generateEmoji(),
 
 	},

package/schemas/deepNest.js

@@ -1,5 +1,9 @@
 const Chance = require('chance');
 const chance = new Chance();
+const u = require('../core/utils');
+
+const plans = ['free', 'premium', 'casual', 'influencer'];
+const marketingChannels = ["Organic", "Organic", "Organic", "Organic", "Instagram Ads", "Facebook Ads", "Google Ads", "Youtube Ads", "Instagram Post", "Instagram Post", "Facebook Post"];
 
 const config = {
 	token: "",
@@ -35,7 +39,7 @@ function generateVideoMeta() {
 			availableFormats: chance.pickset(formats, int(1, formats.length)),
 			availableAspectRatios: chance.pickset(ratios, int(1, ratios.length)),
 			availableContainerFormats: chance.pickset(containers, int(1, containers.length)),
-			observedLatencyTimestamps: chance.pickset([].range(1, 300000), int(1, 40))
+			observedLatencyTimestamps: chance.pickset(u.range(1, 300000), int(1, 40))
 		},
 		videoStats: {
 			numberOfPlays: int(10, 10000000),

package/schemas/foobar.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { pickAWinner, weightedRange, date, integer } = require('../utils');
+const { pickAWinner, weightedRange, date, integer } = require('../core/utils');
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 
@@ -31,7 +31,7 @@ const config = {
 	region: "US",
 	anonIds: true, //if true, anonymousIds are created for each user
 	sessionIds: false, //if true, sessionIds are created for each user
-
+	
 	events: [
 		{
 			event: "foo",

package/schemas/funnels.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { pickAWinner, weightedRange, date, integer } = require('../utils');
+const { pickAWinner, weightedRange, date, integer } = require('../core/utils');
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 

package/schemas/simple.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { pickAWinner, weightedRange, date, integer } = require('../utils');
+const { pickAWinner, weightedRange, date, integer } = require('../core/utils');
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 

package/scratch.mjs

@@ -1,23 +1,36 @@
-import main from "./index.js";
+import main from "./core/index.js";
 import amir from './customers/amir.js';
 import simple from './schemas/simple.js';
 import funnels from './schemas/funnels.js';
 import foobar from './schemas/foobar.js';
 import complex from './schemas/complex.js';
 import deepNest from './schemas/deepNest.js';
+import anon from './schemas/anon.js';
 import execSync from 'child_process';
 
 
 /** @type {main.Config} */
-const spec = {	
-	...simple,	
+const spec = {
+	...complex,
 	writeToDisk: false,
 	verbose: true,
 	makeChart: false,
-	token: "e98e6af94f6ddfb5e967fa265484539a"
+	numUsers: 100,
+	numEvents: 10000,
+	numDays: 90,
+	token: ""
 };
 
 
 execSync.execSync('npm run prune');
-const { eventData, groupProfilesData, lookupTableData, mirrorEventData, scdTableData, userProfilesData, importResults } = await main(spec);
+const { eventData,
+	groupProfilesData,
+	lookupTableData,
+	mirrorEventData,
+	scdTableData,
+	userProfilesData,
+	importResults,
+	files,
+	adSpendData
+} = await main(spec);
 debugger;

package/scripts/jsdoctest.js

@@ -1,5 +1,5 @@
 /* eslint-disable no-unused-vars */
 // @ts-check
-const module = require('../index.js');
+const module = require('../core/index.js');
 
 module({})