make-mp-data 1.4.5 → 1.5.1

package/schemas/complex.js

@@ -8,7 +8,7 @@
 
 const Chance = require('chance');
 const chance = new Chance();
-const { weighNumRange, date, integer } = require('../core/utils.js');
+const { weighNumRange, date, integer } = require('../components/utils.js');
 const u = require('ak-tools');
 
 /** @type {import('../types.js').Config} */
@@ -20,8 +20,8 @@ const config = {
 	numUsers: 1000, //how many users	
 	format: 'csv', //csv or json
 	region: "US",
-	anonIds: true, //if true, anonymousIds are created for each user
-	sessionIds: true, //if true, sessionIds are created for each user
+	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasSessionIds: true, //if true, hasSessionIds are created for each user
 
 	hasLocation: true,
 	hasAndroidDevices: true,
@@ -31,7 +31,14 @@ const config = {
 	hasCampaigns: true,
 	isAnonymous: false,
 	hasAdSpend: true,
-
+	
+	hasAvatar: true,
+	makeChart: false,
+
+	batchSize: 500_000,
+	concurrency: 500,
+	
+	funnels: [],
 	events: [
 		{
 			"event": "checkout",
@@ -164,7 +171,7 @@ const config = {
 		profit: { events: ["checkout"], values: [4, 2, 42, 420] },
 		watchTimeSec: {
 			events: ["watch video"],
-			values: weighNumRange(50, 1200, 6)
+			values: weighNumRange(50, 1200, 2)
 		}
 	},
 
@@ -208,7 +215,7 @@ const config = {
 				"demand": ["high", "medium", "medium", "low"],
 				"supply": ["high", "medium", "medium", "low"],
 				"manufacturer": chance.company.bind(chance),
-				"price": weighNumRange(5, 500, 1000, .25),
+				"price": weighNumRange(5, 500, .25),
 				"rating": weighNumRange(1, 5),
 				"reviews": weighNumRange(0, 35)
 			}

package/schemas/foobar.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { pickAWinner, weighNumRange, date, integer } = require('../core/utils');
+const { pickAWinner, weighNumRange, date, integer } = require('../components/utils');
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 
@@ -29,9 +29,9 @@ const config = {
 	numUsers: 25000, //how many users	
 	format: 'json', //csv or json
 	region: "US",
-	anonIds: true, //if true, anonymousIds are created for each user
-	sessionIds: false, //if true, sessionIds are created for each user
-	
+	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasSessionIds: false, //if true, hasSessionIds are created for each user
+
 	events: [
 		{
 			event: "foo",
@@ -97,9 +97,24 @@ const config = {
 
 	scdProps: {},
 	mirrorProps: {},
-	groupKeys: [],
-	groupProps: {},
 	lookupTables: [],
+	groupKeys: [
+		["company_id", 1000],
+		["product_id", 10000]
+	],
+	groupProps: {
+		company_id: {
+			name: chance.company.bind(chance),
+			industry: chance.pickone.bind(chance, ["tech", "retail", "finance", "healthcare", "manufacturing", "media", "entertainment", "education", "government", "nonprofit", "other"])
+		},
+		product_id: {
+			name: chance.sentence.bind(chance, { words: 3 }),
+			category: chance.pickone.bind(chance, itemCategories),
+			price: chance.floating.bind(chance, { min: 0, max: 1000, fixed: 2 }),
+			quantity: integer
+		}
+	},
+
 	hook: function (record, type, meta) {
 		return record;
 	}

package/schemas/funnels.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { pickAWinner, weighNumRange, date, integer } = require('../core/utils');
+const { pickAWinner, weighNumRange, date, integer } = require('../components/utils');
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 
@@ -29,8 +29,8 @@ const config = {
 	numUsers: 500, //how many users	
 	format: 'csv', //csv or json
 	region: "US",
-	anonIds: false, //if true, anonymousIds are created for each user
-	sessionIds: false, //if true, sessionIds are created for each user
+	hasAnonIds: false, //if true, anonymousIds are created for each user
+	hasSessionIds: false, //if true, hasSessionIds are created for each user
 
 	events: [
 		{

package/schemas/mirror.js

@@ -0,0 +1,129 @@
+/**
+ * This is the default configuration file for the data generator in SIMPLE mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+
+
+const Chance = require('chance');
+const chance = new Chance();
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+const { uid, comma } = require('ak-tools');
+const { pickAWinner, weighNumRange, date, integer } = require('../components/utils');
+
+
+/** @type {import('../types').Config} */
+const config = {
+	token: "",
+	seed: "mirror me",
+	numDays: 30, //how many days worth of data
+	numEvents: 10000, //how many events
+	numUsers: 1000, //how many users	
+	format: 'json', //csv or json
+	region: "US",
+	hasAnonIds: true, //if true, anonymousIds are created for each user
+	hasSessionIds: false, //if true, hasSessionIds are created for each user
+
+	events: [
+		{
+			event: "foo",
+			weight: 10,
+			properties: {}
+		},
+		{
+			event: "bar",
+			weight: 9,
+			properties: {}
+		},
+		{
+			event: "baz",
+			weight: 8,
+			properties: {}
+		},
+		{
+			event: "qux",
+			weight: 7,
+			properties: {}
+		},
+		{
+			event: "garply",
+			weight: 6,
+			properties: {}
+		},
+		{
+			event: "durtle",
+			weight: 5,
+			properties: {}
+		},
+		{
+			event: "linny",
+			weight: 4,
+			properties: {}
+		},
+		{
+			event: "fonk",
+			weight: 3,
+			properties: {}
+		},
+		{
+			event: "crumn",
+			weight: 2,
+			properties: {}
+		},
+		{
+			event: "yak",
+			weight: 1,
+			properties: {}
+		}
+	],
+	superProps: {
+		color: ["red", "orange", "yellow", "green", "blue", "indigo", "violet"],
+		deleteMe: ["hello", "world", "i", "should", "be", "deleted"],		
+		updateMe: weighNumRange(1, 10)
+	},
+	userProps: {
+		title: chance.profession.bind(chance),
+		luckyNumber: weighNumRange(42, 420),
+		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
+	},
+
+	scdProps: {},
+	mirrorProps: {
+		"newlyCreated": {
+			events: "*",
+			strategy: "create",
+			values: ["fickle", "buckle", "tickle", "mackle"]
+
+		},
+		"deleteMe": {
+			events: "*",
+			strategy: "delete",
+			values: ["ignored"]
+		},
+		"fillMe": {
+			events: "*",
+			strategy: "fill",
+			values: ["deal", "with", "it"]
+		},
+		"updateMe": {
+			events: "*",
+			strategy: "update",
+			values: weighNumRange(11, 20)
+		},
+	},
+	groupKeys: [],
+	groupProps: {},
+	lookupTables: [],
+	hook: function (record, type, meta) {
+		return record;
+	}
+};
+
+
+
+module.exports = config;

package/schemas/sanity.js

@@ -0,0 +1,118 @@
+/**
+ * This is the default configuration file for the data generator in SIMPLE mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+
+
+const Chance = require('chance');
+const chance = new Chance();
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+const { weighNumRange, integer } = require('../components/utils');
+
+
+/** @type {import('../types').Config} */
+const config = {
+	token: "",
+	seed: "foo bar",
+	numDays: 90, //how many days worth of data
+	numEvents: 50_000, //how many events
+	numUsers: 500, //how many users	
+	format: 'json', //csv or json
+	region: "US",
+	hasAnonIds: false, //if true, anonymousIds are created for each user
+	hasSessionIds: false, //if true, hasSessionIds are created for each user
+	alsoInferFunnels: true, //if true, infer funnels from events
+	makeChart: true,
+	funnels: [
+		{
+			sequence: ["qux", "garply", "durtle", "linny", "fonk", "crumn", "yak"],			
+		},
+		{
+			sequence: ["foo", "bar", "baz"],
+			isFirstFunnel: true,
+		}
+	],
+	events: [
+		{
+			event: "foo",
+			weight: 10,
+			properties: {}
+		},
+		{
+			event: "bar",
+			weight: 9,
+			isFirstEvent: true,
+			properties: {}
+		},
+		{
+			event: "baz",
+			weight: 8,
+			properties: {}
+		},
+		{
+			event: "qux",
+			weight: 7,
+			properties: {}
+		},
+		{
+			event: "garply",
+			weight: 6,
+			properties: {}
+		},
+		{
+			event: "durtle",
+			weight: 5,
+			properties: {}
+		},
+		{
+			event: "linny",
+			weight: 4,
+			properties: {}
+		},
+		{
+			event: "fonk",
+			weight: 3,
+			properties: {}
+		},
+		{
+			event: "crumn",
+			weight: 2,
+			properties: {}
+		},
+		{
+			event: "yak",
+			weight: 1,
+			properties: {}
+		}
+	],
+	superProps: {
+		color: ["red", "orange", "yellow", "green", "blue", "indigo", "violet"],
+		number: integer,
+
+	},
+	userProps: {
+		title: chance.profession.bind(chance),
+		luckyNumber: weighNumRange(42, 420),
+		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
+	},
+
+	scdProps: {},
+	mirrorProps: {},
+	lookupTables: [],
+	groupKeys: [],
+	groupProps: {},
+
+	hook: function (record, type, meta) {
+		return record;
+	}
+};
+
+
+
+module.exports = config;

package/schemas/session-replay.js

@@ -0,0 +1,136 @@
+
+const SEED = "my-seed";
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+require("dotenv").config();
+const u = require("../components/utils");
+const v = require("ak-tools");
+const chance = u.initChance(SEED);
+const num_users = 1000;
+const days = 90;
+
+/** @type {import("../types").Config} */
+const config = {
+	token: "0135b13525b21294e9e599a483eb35b2",
+	seed: SEED,
+	numDays: days,
+	numEvents: num_users * 100,
+	numUsers: num_users,
+	hasAnonIds: false,
+	hasSessionIds: false,
+	format: "json",
+	alsoInferFunnels: false,
+	hasLocation: true,
+	hasAndroidDevices: false,
+	hasIOSDevices: false,
+	hasDesktopDevices: true,
+	hasBrowser: true,
+	hasCampaigns: true,
+	isAnonymous: false,
+	hasAdSpend: true,
+
+	hasAvatar: true,
+	makeChart: true,
+
+	batchSize: 500_000,
+	concurrency: 500,
+	writeToDisk: false,
+
+	funnels: [
+		{
+			sequence: ["view page", "watch video", "like video"],
+			conversionRate: 25,
+			timeToConvert: 1
+		},
+		{
+			sequence: ["view page", "watch video", "dislike video"],
+			conversionRate: 20,
+			timeToConvert: 1
+		},
+		{
+			sequence: ["view page", "watch video", "comment on video"],
+			conversionRate: 15,
+			timeToConvert: 1
+		},
+		{
+			sequence: ["view page", "sign up"],
+			conversionRate: 70,
+			isFirstFunnel: true,
+
+		}
+	],
+	events: [
+		{
+			"event": "view page",
+			weight: 100,
+
+		},
+		{
+			"event": "watch video",
+			weight: 150,
+			"properties": {
+				"watch time (mins)": u.weighNumRange(1, 60, .34)
+			}
+		},
+		{
+			"event": "like video",
+			weight: 50
+		},
+		{
+			"event": "dislike video",
+			weight: 40
+		},
+		{
+			"event": "comment on video",
+			weight: 30
+		},
+		{
+			"event": "new feature: AI Filters",
+			weight: 10
+		},
+		{
+			"event": "sign up",
+			weight: 10,
+			"isFirstEvent": true,
+		}
+	],
+	superProps: {},
+	userProps: {},
+	scdProps: {},
+	mirrorProps: {},
+	groupKeys: [],
+	groupProps: {},
+	lookupTables: [],
+	hook: function (record, type, meta) {
+		const NOW = dayjs();	
+		const bugStartDate = NOW.subtract(14, 'days');
+		const bugFixedDate = NOW.subtract(3, 'days');
+		const featureReleaseData = NOW.subtract(7, 'days');
+
+		if (type === "event") {
+			const EVENT_TIME = dayjs(record.time);
+			if (record.event === "new feature: AI Filters" && EVENT_TIME.isBefore(featureReleaseData)) {
+				return null;
+			}
+		}
+
+		if (type === "funnel-post") {
+			// there's a bug in Chrome where users can't complete the funnel
+			if (dayjs(record[0].time).isAfter(bugStartDate) && dayjs(record[0].time).isBefore(bugFixedDate)) {
+				if (record[0].browser === "Chrome") {
+					if (record.length > 2) {						
+						record.pop(); // remove the last event						
+					}
+				}
+			}
+			
+		}
+
+		return record;
+	}
+
+
+};
+
+module.exports = config;

package/schemas/simple.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { pickAWinner, weighNumRange, date, integer, weighChoices } = require('../core/utils');
+const { pickAWinner, weighNumRange, date, integer, weighChoices } = require('../components/utils');
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 
@@ -29,10 +29,10 @@ const config = {
 	numUsers: 500, //how many users	
 	format: 'csv', //csv or json
 	region: "US",
-	anonIds: false, //if true, anonymousIds are created for each user
-	sessionIds: false, //if true, sessionIds are created for each user
+	hasAnonIds: false, //if true, anonymousIds are created for each user
+	hasSessionIds: false, //if true, hasSessionIds are created for each user
 	hasAdSpend: false,
-
+	makeChart: true,
 	hasLocation: true,
 	hasAndroidDevices: true,
 	hasIOSDevices: true,

package/scratch.mjs

@@ -1,26 +1,49 @@
-import main from "./core/index.js";
-import amir from './dungeons/amir.js';
+/*
+----
+TO DOs
+----
+*/
+
+//!feature: fixedTimeFunnel? if set this funnel will occur for all users at the same time ['cards charged', 'charge complete']
+//!feature: churn ... is churnFunnel, possible to return, etc
+//!feature: send SCD data to mixpanel (blocked on dev)
+//!feature: send and map lookup tables to mixpanel (also blocked on dev)
+//!bug: using --mc flag reverts to --complex for some reason
+
+
+import main from "./index.js";
+
 import simple from './schemas/simple.js';
 import funnels from './schemas/funnels.js';
 import foobar from './schemas/foobar.js';
 import complex from './schemas/complex.js';
-import deepNest from './schemas/deepNest.js';
+import adspend from './schemas/adspend.js'
+
 import anon from './schemas/anon.js';
 import execSync from 'child_process';
-import gio from './dungeons/gio-apt-list.js';
+import mirror from './schemas/mirror.js'
+// import mds from './dungeons/modern-data-stack.js'
+import big from './schemas/big.js'
 
+const numEvents = 1000;
 
 /** @type {main.Config} */
 const spec = {
-	...gio,
-	writeToDisk: false,
+	...big,
+	writeToDisk: true,
 	verbose: true,
-	makeChart: false,	
+	makeChart: false,
+	// format: "csv",
+	// numEvents,
+	// numUsers: numEvents / 100,
+	
 };
 
 
 execSync.execSync('npm run prune');
-const { eventData,
+const RESULT = await main(spec);
+const { 
+	eventData,
 	groupProfilesData,
 	lookupTableData,
 	mirrorEventData,
@@ -29,5 +52,7 @@ const { eventData,
 	importResults,
 	files,
 	adSpendData
-} = await main(spec);
+} = RESULT;
+
+
 debugger;

package/scripts/jsdoctest.js

@@ -1,5 +1,5 @@
 /* eslint-disable no-unused-vars */
 // @ts-check
-const module = require('../core/index.js');
+const module = require('../index.js');
 
 module({})

package/scripts/{new.sh → new-dungeon.sh}

@@ -13,23 +13,40 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 require("dotenv").config();
-const u = require("../core/utils");
+const u = require("../components/utils");
 const v = require("ak-tools");
 const chance = u.initChance(SEED);
+const num_users = 25_000
+const days = 100
 
 /** @type {import("../types").Config} */
 const config = {
     token: "",
     seed: SEED,
-    numDays: 100, //how many days worth of data
-    numEvents: 100000, //how many events
-    numUsers: 1000, //how many users    
-    format: "csv", //csv or json
-    region: "US",
-    makeChart: false,
-    anonIds: false,
-    sessionIds: false,
-    writeToDisk: false,
+    numDays: days, 
+    numEvents: num_users * 100, 
+    numUsers: num_users, 
+	hasAnonIds: false, 
+	hasSessionIds: false, 
+	format: "json",
+	alsoInferFunnels: true,
+	hasLocation: true,
+	hasAndroidDevices: true,
+	hasIOSDevices: true,
+	hasDesktopDevices: true,
+	hasBrowser: true,
+	hasCampaigns: true,
+	isAnonymous: false,
+	hasAdSpend: true,
+	
+	hasAvatar: true,
+	makeChart: false,
+
+	batchSize: 500_000,
+	concurrency: 500,
+	writeToDisk: false,
+	
+	funnels: [],
     events: [],
     superProps: {},
     userProps: {},
@@ -38,9 +55,31 @@ const config = {
     groupKeys: [],
     groupProps: {},
     lookupTables: [],
-    hook: function(record, type, meta) {
-        return record;
-    }
+	hook: function (record, type, meta) {
+		const NOW = dayjs.unix(global.NOW);
+
+		if (type === "event") {
+			const EVENT_TIME = dayjs(record.time);
+		}
+
+		if (type === "user") {
+
+		}
+
+		if (type === "funnel-post") {
+			
+		}
+
+		if (type === "funnel-pre") {
+
+		}
+
+		if (type === "scd") {
+
+		}
+
+		return record;
+	}
 };
 
 module.exports = config;'