make-mp-data 1.4.4 → 1.5.0

package/schemas/mirror.js

@@ -0,0 +1,129 @@
+/**
+ * This is the default configuration file for the data generator in SIMPLE mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+
+
+const Chance = require('chance');
+const chance = new Chance();
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+const { uid, comma } = require('ak-tools');
+const { pickAWinner, weighNumRange, date, integer } = require('../src/utils');
+
+
+/** @type {import('../types').Config} */
+const config = {
+	token: "",
+	seed: "mirror me",
+	numDays: 30, //how many days worth of data
+	numEvents: 10000, //how many events
+	numUsers: 1000, //how many users	
+	format: 'json', //csv or json
+	region: "US",
+	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasSessionIds: false, //if true, hasSessionIds are created for each user
+
+	events: [
+		{
+			event: "foo",
+			weight: 10,
+			properties: {}
+		},
+		{
+			event: "bar",
+			weight: 9,
+			properties: {}
+		},
+		{
+			event: "baz",
+			weight: 8,
+			properties: {}
+		},
+		{
+			event: "qux",
+			weight: 7,
+			properties: {}
+		},
+		{
+			event: "garply",
+			weight: 6,
+			properties: {}
+		},
+		{
+			event: "durtle",
+			weight: 5,
+			properties: {}
+		},
+		{
+			event: "linny",
+			weight: 4,
+			properties: {}
+		},
+		{
+			event: "fonk",
+			weight: 3,
+			properties: {}
+		},
+		{
+			event: "crumn",
+			weight: 2,
+			properties: {}
+		},
+		{
+			event: "yak",
+			weight: 1,
+			properties: {}
+		}
+	],
+	superProps: {
+		color: ["red", "orange", "yellow", "green", "blue", "indigo", "violet"],
+		deleteMe: ["hello", "world", "i", "should", "be", "deleted"],		
+		updateMe: weighNumRange(1, 10)
+	},
+	userProps: {
+		title: chance.profession.bind(chance),
+		luckyNumber: weighNumRange(42, 420),
+		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
+	},
+
+	scdProps: {},
+	mirrorProps: {
+		"newlyCreated": {
+			events: "*",
+			strategy: "create",
+			values: ["fickle", "buckle", "tickle", "mackle"]
+
+		},
+		"deleteMe": {
+			events: "*",
+			strategy: "delete",
+			values: ["ignored"]
+		},
+		"fillMe": {
+			events: "*",
+			strategy: "fill",
+			values: ["deal", "with", "it"]
+		},
+		"updateMe": {
+			events: "*",
+			strategy: "update",
+			values: weighNumRange(11, 20)
+		},
+	},
+	groupKeys: [],
+	groupProps: {},
+	lookupTables: [],
+	hook: function (record, type, meta) {
+		return record;
+	}
+};
+
+
+
+module.exports = config;

package/schemas/simple.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { pickAWinner, weightedRange, date, integer } = require('../core/utils');
+const { pickAWinner, weighNumRange, date, integer, weighChoices } = require('../src/utils');
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 
@@ -29,8 +29,8 @@ const config = {
 	numUsers: 500, //how many users	
 	format: 'csv', //csv or json
 	region: "US",
-	anonIds: false, //if true, anonymousIds are created for each user
-	sessionIds: false, //if true, sessionIds are created for each user
+	hasAnonIds: false, //if true, anonymousIds are created for each user
+	hasSessionIds: false, //if true, hasSessionIds are created for each user
 	hasAdSpend: false,
 
 	hasLocation: true,
@@ -40,17 +40,17 @@ const config = {
 	hasBrowser: true,
 	hasCampaigns: true,
 	isAnonymous: false,
-	
+
 
 	events: [
 		{
 			event: "checkout",
 			weight: 2,
 			properties: {
-				amount: weightedRange(5, 500, .25),
-				currency: ["USD", "CAD", "EUR", "BTC", "ETH", "JPY"],
-				coupon: ["none", "none", "none", "none", "10%OFF", "20%OFF", "10%OFF", "20%OFF", "30%OFF", "40%OFF", "50%OFF"],
-				numItems: weightedRange(1, 10),
+				amount: weighNumRange(5, 500, .25),
+				currency: pickAWinner(["USD", "CAD", "EUR", "BTC", "ETH", "JPY"], 0),
+				coupon: weighChoices(["none", "none", "none", "none", "10%OFF", "20%OFF", "10%OFF", "20%OFF", "30%OFF", "40%OFF", "50%OFF"]),
+				numItems: weighNumRange(1, 10),
 
 			}
 		},
@@ -58,21 +58,20 @@ const config = {
 			event: "add to cart",
 			weight: 4,
 			properties: {
-				amount: weightedRange(5, 500, .25),
-				rating: weightedRange(1, 5),
-				reviews: weightedRange(0, 35),
+				amount: weighNumRange(5, 500, .25),
+				rating: weighNumRange(1, 5),
+				reviews: weighNumRange(0, 35),
 				isFeaturedItem: [true, false, false],
 				itemCategory: pickAWinner(itemCategories, integer(0, 27)),
 				dateItemListed: date(30, true, 'YYYY-MM-DD'),
-				itemId: integer(1000, 9999),
 			}
 		},
 		{
 			event: "page view",
 			weight: 10,
 			properties: {
-				page: ["/", "/", "/help", "/account", "/watch", "/listen", "/product", "/people", "/peace"],
-				utm_source: ["$organic", "$organic", "$organic", "$organic", "google", "google", "google", "facebook", "facebook", "twitter", "linkedin"],
+				page: pickAWinner(["/", "/", "/help", "/account", "/watch", "/listen", "/product", "/people", "/peace"]),
+				utm_source: pickAWinner(["$organic", "$organic", "$organic", "$organic", "google", "google", "google", "facebook", "facebook", "twitter", "linkedin"]),
 			}
 		},
 		{
@@ -81,7 +80,7 @@ const config = {
 			properties: {
 				videoCategory: pickAWinner(videoCategories, integer(0, 9)),
 				isFeaturedItem: [true, false, false],
-				watchTimeSec: weightedRange(10, 600, .25),
+				watchTimeSec: weighNumRange(10, 600, .25),
 				quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
 				format: ["mp4", "avi", "mov", "mpg"],
 				uploader_id: chance.guid.bind(chance)
@@ -95,7 +94,6 @@ const config = {
 				isFeaturedItem: [true, false, false],
 				itemCategory: pickAWinner(itemCategories, integer(0, 27)),
 				dateItemListed: date(30, true, 'YYYY-MM-DD'),
-				itemId: integer(1000, 9999),
 			}
 		},
 		{
@@ -105,7 +103,6 @@ const config = {
 				isFeaturedItem: [true, false, false],
 				itemCategory: pickAWinner(itemCategories, integer(0, 27)),
 				dateItemListed: date(30, true, 'YYYY-MM-DD'),
-				itemId: integer(1000, 9999),
 			}
 		},
 		{
@@ -123,9 +120,7 @@ const config = {
 	],
 	superProps: {
 		platform: ["web", "mobile", "web", "mobile", "web", "web", "kiosk", "smartTV"],
-		currentTheme: ["light", "dark", "custom", "light", "dark"],
-		// emotions: generateEmoji(),
-
+		currentTheme: weighChoices(["light", "dark", "custom", "light", "dark"]),
 	},
 	/*
 	user properties work the same as event properties
@@ -133,20 +128,11 @@ const config = {
 	*/
 	userProps: {
 		title: chance.profession.bind(chance),
-		luckyNumber: weightedRange(42, 420),
-		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
+		luckyNumber: weighNumRange(42, 420, .3),
+		spiritAnimal: pickAWinner(["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"])
 	},
-
 	scdProps: {},
-	mirrorProps: {
-		isBot: { events: "*", values: [false, false, false, false, true] },
-		profit: { events: ["checkout"], values: [4, 2, 42, 420] },
-		watchTimeSec: {
-			events: ["watch video"],
-			values: weightedRange(50, 1200, 6)
-		}
-
-	},
+	mirrorProps: {},
 
 	/*
 	for group analytics keys, we need an array of arrays [[],[],[]] 

package/scratch.mjs

@@ -1,29 +1,49 @@
-import main from "./core/index.js";
-import amir from './customers/amir.js';
+/*
+----
+TO DOs
+----
+*/
+
+//!feature: fixedTimeFunnel? if set this funnel will occur for all users at the same time ['cards charged', 'charge complete']
+//!feature: churn ... is churnFunnel, possible to return, etc
+//!feature: send SCD data to mixpanel (blocked on dev)
+//!feature: send and map lookup tables to mixpanel (also blocked on dev)
+//!bug: using --mc flag reverts to --complex for some reason
+
+
+import main from "./index.js";
+
 import simple from './schemas/simple.js';
 import funnels from './schemas/funnels.js';
 import foobar from './schemas/foobar.js';
 import complex from './schemas/complex.js';
-import deepNest from './schemas/deepNest.js';
+import adspend from './schemas/adspend.js'
+
 import anon from './schemas/anon.js';
 import execSync from 'child_process';
+import mirror from './schemas/mirror.js'
+// import mds from './dungeons/modern-data-stack.js'
+import big from './schemas/big.js'
 
+const numEvents = 1000;
 
 /** @type {main.Config} */
 const spec = {
-	...complex,
-	writeToDisk: false,
+	...big,
+	writeToDisk: true,
 	verbose: true,
 	makeChart: false,
-	numUsers: 100,
-	numEvents: 10000,
-	numDays: 90,
-	token: ""
+	// format: "csv",
+	// numEvents,
+	// numUsers: numEvents / 100,
+	
 };
 
 
 execSync.execSync('npm run prune');
-const { eventData,
+const RESULT = await main(spec);
+const { 
+	eventData,
 	groupProfilesData,
 	lookupTableData,
 	mirrorEventData,
@@ -32,5 +52,7 @@ const { eventData,
 	importResults,
 	files,
 	adSpendData
-} = await main(spec);
+} = RESULT;
+
+
 debugger;

package/scripts/jsdoctest.js

@@ -1,5 +1,5 @@
 /* eslint-disable no-unused-vars */
 // @ts-check
-const module = require('../core/index.js');
+const module = require('../index.js/index.js');
 
 module({})

package/scripts/new.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# Ensure the ./path directory exists
+mkdir -p ./dungeons
+
+# Generate a random file name with .js extension
+random_file_name=$(mktemp ./dungeons/my-file-XXXXXXXX.js)
+
+# Initial text to write to the file
+initial_text='
+const SEED = "my-seed";
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+require("dotenv").config();
+const u = require("../src/utils");
+const v = require("ak-tools");
+const chance = u.initChance(SEED);
+const num_users = 25_000
+const days = 100
+
+/** @type {import("../types").Config} */
+const config = {
+    token: "",
+    seed: SEED,
+    numDays: days, 
+    numEvents: num_users * 100, 
+    numUsers: num_users, 
+	hasAnonIds: true, 
+	hasSessionIds: true, 
+
+	hasLocation: true,
+	hasAndroidDevices: true,
+	hasIOSDevices: true,
+	hasDesktopDevices: true,
+	hasBrowser: true,
+	hasCampaigns: true,
+	isAnonymous: false,
+	hasAdSpend: true,
+	
+	hasAvatar: true,
+	makeChart: false,
+
+	batchSize: 500_000,
+	concurrency: 500,
+	writeToDisk: false,
+	
+	funnels: [],
+    events: [],
+    superProps: {},
+    userProps: {},
+    scdProps: {},
+    mirrorProps: {},
+    groupKeys: [],
+    groupProps: {},
+    lookupTables: [],
+    hook: function(record, type, meta) {
+        return record;
+    }
+};
+
+module.exports = config;'
+
+# Write the initial text to the new file
+echo "$initial_text" > "$random_file_name"
+
+# Output the name of the created file
+echo "File created: $random_file_name"

package/{core → src}/cli.js

@@ -15,7 +15,6 @@ by ak@mixpanel.com
 
 function cliParams() {
 	console.log(hero);
-	// @ts-ignore
 	const args = yargs(process.argv.splice(2))
 		.scriptName("make-mp-data")
 		.usage(`\nusage:\nnpx $0 [dataModel.js] [options]
@@ -85,6 +84,13 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			describe: 'either US or EU',
 			type: 'string'
 		})
+		.option('concurrency', {
+			alias: 'conn',
+			default: 500,
+			demandOption: false,
+			describe: 'concurrency level for data generation',
+			type: 'number'
+		})
 		.options("complex", {
 			demandOption: false,
 			default: false,
@@ -101,7 +107,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
-		.option("sessionIds", {
+		.option("hasSessionIds", {
 			demandOption: false,
 			default: false,
 			describe: 'create session ids in the data',
@@ -109,7 +115,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
-		.option("anonIds", {
+		.option("hasAnonIds", {
 			demandOption: false,
 			default: false,
 			describe: 'create anonymous ids in the data',
@@ -187,6 +193,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
+		
 		.help()
 		.wrap(null)
 		.argv;