make-mp-data 1.4.2 → 1.4.4

package/package.json

@@ -1,26 +1,25 @@
 {
 	"name": "make-mp-data",
-	"version": "1.4.02",
+	"version": "1.4.04",
 	"description": "builds all mixpanel primitives for a given project",
 	"main": "index.js",
 	"types": "types.d.ts",
 	"scripts": {
-		"start": "node index.js",
+		"start": "node ./core/index.js",
 		"dev": "nodemon scratch.mjs --ignore /data",
-		"complex": "nodemon index.js --complex --e 10000 --u 100",
-		"simple": "nodemon index.js  --simple --e 10000 --u 100",
+		"complex": "nodemon ./core/index.js --complex --e 10000 --u 100",
+		"simple": "nodemon ./core/index.js  --simple --e 10000 --u 100",
 		"prune": "rm -f ./data/* && rm -f ./tmp/*",
 		"post": "npm publish",
 		"test": "NODE_ENV=test jest --runInBand",
-		"deps": "sh ./scripts/deps.sh",
-		"udacity": "nodemon index.js ./customers/udacity.js"
+		"deps": "sh ./scripts/deps.sh"
 	},
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/ak--47/make-mp-data.git"
 	},
 	"bin": {
-		"make-mp-data": "./index.js"
+		"make-mp-data": "./core/index.js"
 	},
 	"keywords": [
 		"mixpanel",
@@ -47,7 +46,7 @@
 		"chartjs-node-canvas": "^4.1.6",
 		"dayjs": "^1.11.11",
 		"dotenv": "^16.4.5",
-		"mixpanel-import": "^2.5.55",
+		"mixpanel-import": "^2.5.552",
 		"yargs": "^17.7.2"
 	},
 	"devDependencies": {

package/schemas/anon.js

@@ -0,0 +1,104 @@
+/**
+ * This is the default configuration file for the data generator in SIMPLE mode
+ * notice how the config object is structured, and see it's type definition in ./types.d.ts
+ * feel free to modify this file to customize the data you generate
+ * see helper functions in utils.js for more ways to generate data
+ */
+
+
+
+
+const Chance = require('chance');
+const chance = new Chance();
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+const { uid, comma } = require('ak-tools');
+const { pickAWinner, weightedRange, date, integer } = require('../core/utils');
+
+
+
+/** @type {import('../types').Config} */
+const config = {
+	token: "",
+	seed: "foo bar",
+	numDays: 365, //how many days worth of data
+	numEvents: 100000, //how many events
+	numUsers: 10000, //how many users	
+	format: 'json', //csv or json
+	region: "US",
+	anonIds: true, //if true, anonymousIds are created for each user
+	sessionIds: false, //if true, sessionIds are created for each user
+	isAnonymous: true,
+	hasLocation: true, 
+	events: [
+		{
+			event: "foo",
+			weight: 10,
+			properties: {}
+		},
+		{
+			event: "bar",
+			weight: 9,
+			properties: {}
+		},
+		{
+			event: "baz",
+			weight: 8,
+			properties: {}
+		},
+		{
+			event: "qux",
+			weight: 7,
+			properties: {}
+		},
+		{
+			event: "garply",
+			weight: 6,
+			properties: {}
+		},
+		{
+			event: "durtle",
+			weight: 5,
+			properties: {}
+		},
+		{
+			event: "linny",
+			weight: 4,
+			properties: {}
+		},
+		{
+			event: "fonk",
+			weight: 3,
+			properties: {}
+		},
+		{
+			event: "crumn",
+			weight: 2,
+			properties: {}
+		},
+		{
+			event: "yak",
+			weight: 1,
+			properties: {}
+		}
+	],
+	superProps: {},
+	userProps: {
+		title: chance.profession.bind(chance),
+		luckyNumber: weightedRange(42, 420),
+		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
+	},
+	scdProps: {},
+	mirrorProps: {},
+	groupKeys: [],
+	groupProps: {},
+	lookupTables: [],
+	hook: function (record, type, meta) {
+		return record;
+	}
+};
+
+
+
+module.exports = config;

package/schemas/complex.js

@@ -8,7 +8,7 @@
 
 const Chance = require('chance');
 const chance = new Chance();
-const { weightedRange, date, integer } = require('../utils.js');
+const { weightedRange, date, integer } = require('../core/utils.js');
 const u = require('ak-tools');
 
 /** @type {import('../types.js').Config} */
@@ -23,6 +23,15 @@ const config = {
 	anonIds: true, //if true, anonymousIds are created for each user
 	sessionIds: true, //if true, sessionIds are created for each user
 
+	hasLocation: true,
+	hasAndroidDevices: true,
+	hasIOSDevices: true,
+	hasDesktopDevices: true,
+	hasBrowser: true,
+	hasCampaigns: true,
+	isAnonymous: false,
+	hasAdSpend: true,
+
 	events: [
 		{
 			"event": "checkout",
@@ -122,7 +131,7 @@ const config = {
 		}
 	],
 	superProps: {
-		device: deviceAttributes()
+		linked_device: deviceAttributes()
 		// emotions: generateEmoji(),
 
 	},

package/schemas/deepNest.js

@@ -1,5 +1,9 @@
 const Chance = require('chance');
 const chance = new Chance();
+const u = require('../core/utils');
+
+const plans = ['free', 'premium', 'casual', 'influencer'];
+const marketingChannels = ["Organic", "Organic", "Organic", "Organic", "Instagram Ads", "Facebook Ads", "Google Ads", "Youtube Ads", "Instagram Post", "Instagram Post", "Facebook Post"];
 
 const config = {
 	token: "",
@@ -35,7 +39,7 @@ function generateVideoMeta() {
 			availableFormats: chance.pickset(formats, int(1, formats.length)),
 			availableAspectRatios: chance.pickset(ratios, int(1, ratios.length)),
 			availableContainerFormats: chance.pickset(containers, int(1, containers.length)),
-			observedLatencyTimestamps: chance.pickset([].range(1, 300000), int(1, 40))
+			observedLatencyTimestamps: chance.pickset(u.range(1, 300000), int(1, 40))
 		},
 		videoStats: {
 			numberOfPlays: int(10, 10000000),

package/schemas/foobar.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { pickAWinner, weightedRange, date, integer } = require('../utils');
+const { pickAWinner, weightedRange, date, integer } = require('../core/utils');
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 
@@ -31,7 +31,7 @@ const config = {
 	region: "US",
 	anonIds: true, //if true, anonymousIds are created for each user
 	sessionIds: false, //if true, sessionIds are created for each user
-
+	
 	events: [
 		{
 			event: "foo",

package/schemas/funnels.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { pickAWinner, weightedRange, date, integer } = require('../utils');
+const { pickAWinner, weightedRange, date, integer } = require('../core/utils');
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 

package/schemas/simple.js

@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const { uid, comma } = require('ak-tools');
-const { pickAWinner, weightedRange, date, integer } = require('../utils');
+const { pickAWinner, weightedRange, date, integer } = require('../core/utils');
 
 const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
 

package/scratch.mjs

@@ -1,23 +1,36 @@
-import main from "./index.js";
+import main from "./core/index.js";
 import amir from './customers/amir.js';
 import simple from './schemas/simple.js';
 import funnels from './schemas/funnels.js';
 import foobar from './schemas/foobar.js';
 import complex from './schemas/complex.js';
 import deepNest from './schemas/deepNest.js';
+import anon from './schemas/anon.js';
 import execSync from 'child_process';
 
 
 /** @type {main.Config} */
-const spec = {	
-	...simple,	
+const spec = {
+	...complex,
 	writeToDisk: false,
 	verbose: true,
 	makeChart: false,
-	token: "e98e6af94f6ddfb5e967fa265484539a"
+	numUsers: 100,
+	numEvents: 10000,
+	numDays: 90,
+	token: ""
 };
 
 
 execSync.execSync('npm run prune');
-const { eventData, groupProfilesData, lookupTableData, mirrorEventData, scdTableData, userProfilesData, importResults } = await main(spec);
+const { eventData,
+	groupProfilesData,
+	lookupTableData,
+	mirrorEventData,
+	scdTableData,
+	userProfilesData,
+	importResults,
+	files,
+	adSpendData
+} = await main(spec);
 debugger;

package/scripts/jsdoctest.js

@@ -1,5 +1,5 @@
 /* eslint-disable no-unused-vars */
 // @ts-check
-const module = require('../index.js');
+const module = require('../core/index.js');
 
 module({})

package/tests/e2e.test.js

@@ -3,7 +3,7 @@
 /* eslint-disable no-undef */
 /* eslint-disable no-debugger */
 /* eslint-disable no-unused-vars */
-const generate = require('../index.js');
+const generate = require('../core/index.js');
 require('dotenv').config();
 const { execSync } = require("child_process");
 const u = require('ak-tools');
@@ -11,6 +11,7 @@ const u = require('ak-tools');
 const simple = require('../schemas/simple.js');
 const complex = require('../schemas/complex.js');
 const deep = require('../schemas/deepNest.js');
+const anon = require('../schemas/anon.js');
 
 const timeout = 60000;
 const testToken = process.env.TEST_TOKEN;
@@ -86,30 +87,39 @@ describe('module', () => {
 });
 
 describe('cli', () => {
+	test('works as CLI (no args)', async () => {
+		console.log('COMPLEX CLI TEST');
+		const run = execSync(`node ./core/index.js --numEvents 1000 --numUsers 100`, { stdio: 'ignore' });
+		// expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
+		expect(csvs.length).toBe(4);
+		clearData();
+	}, timeout);
+
 	test('works as CLI (complex)', async () => {
 		console.log('COMPLEX CLI TEST');
-		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --complex`, { stdio: 'ignore' });
+		const run = execSync(`node ./core/index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --complex`, { stdio: 'ignore' });
 		// expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(12);
+		expect(csvs.length).toBe(13);
 		clearData();
 	}, timeout);
 
 	test('works as CLI (simple)', async () => {
 		console.log('simple CLI TEST');
-		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it"`);
+		const run = execSync(`node ./core/index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --simple`);
 		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(3);
+		expect(csvs.length).toBe(4);
 		clearData();
 	}, timeout);
 
 	test('works as CLI (custom)', async () => {
 		console.log('custom CLI TEST');
-		const run = execSync(`node ./index.js ./schemas/deepNest.js`);
+		const run = execSync(`node ./core/index.js ./schemas/deepNest.js`);
 		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(2);
+		expect(csvs.length).toBe(3);
 		clearData();
 	}, timeout);
 
@@ -179,6 +189,14 @@ describe('options + tweaks', () => {
 
 	}, timeout);
 
+	test('anonymous users', async () => {
+		console.log('ANON TEST');
+		const results = await generate({ ...anon, writeToDisk: false, verbose: true });
+		const { userProfilesData } = results;
+		expect(userProfilesData.every(u => u.name === 'Anonymous User')).toBe(true);
+
+	}, timeout);
+
 });
 
 afterAll(() => {

package/{testSoup.mjs → tests/testSoup.mjs}

@@ -1,5 +1,5 @@
-import { generateLineChart } from './chart.js';
-import { TimeSoup } from './utils.js';
+import { generateLineChart } from '../core/chart.js';
+import { TimeSoup } from '../core/utils.js';
 import dayjs from 'dayjs';
 import { progress } from 'ak-tools';
 import TEST_CASES from './testCases.mjs';

package/tests/unit.test.js

@@ -1,4 +1,4 @@
-const generate = require('../index.js');
+const generate = require('../core/index.js');
 const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 const fs = require('fs');
@@ -44,8 +44,9 @@ const { applySkew,
 	validateTime,
 	interruptArray,
 	optimizedBoxMuller,
-	inferFunnels
-} = require('../utils');
+	inferFunnels,
+	datesBetween
+} = require('../core/utils.js');
 
 
 describe('timesoup', () => {
@@ -231,6 +232,152 @@ describe('generation', () => {
 		expect(createdDate.isValid()).toBeTruthy();
 		expect(createdDate.isBefore(dayjs.unix(global.NOW))).toBeTruthy();
 	});
+
+	test('winner: return func', () => {
+		const items = ['a', 'b', 'c'];
+		const result = pickAWinner(items, 0);
+		expect(typeof result).toBe('function');
+	});
+
+	test('winner: first most', () => {
+		const items = ['a', 'b', 'c'];
+		const mostChosenIndex = 0;
+		const pickFunction = pickAWinner(items, mostChosenIndex);
+		const weightedList = pickFunction();
+
+		// Expect the most chosen item to appear at least once
+		expect(weightedList.includes(items[mostChosenIndex])).toBeTruthy();
+	});
+
+	test('winner: second most', () => {
+		const items = ['a', 'b', 'c'];
+		const mostChosenIndex = 0;
+		const pickFunction = pickAWinner(items, mostChosenIndex);
+		const weightedList = pickFunction();
+
+		const secondMostChosenIndex = (mostChosenIndex + 1) % items.length;
+
+		// Expect the second most chosen item to appear at least once
+		expect(weightedList.includes(items[secondMostChosenIndex])).toBeTruthy();
+	});
+
+	test('winner: third most', () => {
+		const items = ['a', 'b', 'c'];
+		const mostChosenIndex = 0;
+		const pickFunction = pickAWinner(items, mostChosenIndex);
+		const weightedList = pickFunction();
+
+		const thirdMostChosenIndex = (mostChosenIndex + 2) % items.length;
+
+		// Expect the third most chosen item to appear at least once
+		expect(weightedList.includes(items[thirdMostChosenIndex])).toBeTruthy();
+	});
+
+	test('winner: exceed array bounds', () => {
+		const items = ['a', 'b', 'c'];
+		const mostChosenIndex = 0;
+		const pickFunction = pickAWinner(items, mostChosenIndex);
+		const weightedList = pickFunction();
+
+		// Ensure all indices are within the bounds of the array
+		weightedList.forEach(item => {
+			expect(items.includes(item)).toBeTruthy();
+		});
+	});
+
+	test('winner: single item array', () => {
+		const items = ['a'];
+		const mostChosenIndex = 0;
+		const pickFunction = pickAWinner(items, mostChosenIndex);
+		const weightedList = pickFunction();
+
+		// Since there's only one item, all winner: he same
+		weightedList.forEach(item => {
+			expect(item).toBe('a');
+		});
+	});
+
+	test('winner: empty array', () => {
+		const items = [];
+		const pickFunction = pickAWinner(items, 0);
+		const weightedList = pickFunction();
+
+		// Expect the result to be an empty array
+		expect(weightedList.length).toBe(0);
+	});
+
+	test('dates: same start end', () => {
+		const start = '2023-06-10';
+		const end = '2023-06-10';
+		const result = datesBetween(start, end);
+		expect(result).toEqual([]);
+	});
+
+	test('dates: start after end', () => {
+		const start = '2023-06-12';
+		const end = '2023-06-10';
+		const result = datesBetween(start, end);
+		expect(result).toEqual([]);
+	});
+
+	test('dates: correct', () => {
+		const start = '2023-06-10';
+		const end = '2023-06-13';
+		const result = datesBetween(start, end);
+		expect(result).toEqual([
+			'2023-06-10T12:00:00.000Z',
+			'2023-06-11T12:00:00.000Z',
+			'2023-06-12T12:00:00.000Z'
+		]);
+	});
+
+	test('dates: unix times', () => {
+		const start = dayjs('2023-06-10').unix();
+		const end = dayjs('2023-06-13').unix();
+		const result = datesBetween(start, end);
+		expect(result).toEqual([
+			'2023-06-10T12:00:00.000Z',
+			'2023-06-11T12:00:00.000Z',
+			'2023-06-12T12:00:00.000Z'
+		]);
+	});
+
+	test('dates: mixed formats', () => {
+		const start = '2023-06-10';
+		const end = dayjs('2023-06-13').unix();
+		const result = datesBetween(start, end);
+		expect(result).toEqual([
+			'2023-06-10T12:00:00.000Z',
+			'2023-06-11T12:00:00.000Z',
+			'2023-06-12T12:00:00.000Z'
+		]);
+	});
+
+	test('dates: invalid dates', () => {
+		const start = 'invalid-date';
+		const end = '2023-06-13';
+		const result = datesBetween(start, end);
+		expect(result).toEqual([]);
+	});
+
+	test('dates: same day', () => {
+		const start = '2023-06-10T08:00:00.000Z';
+		const end = '2023-06-10T20:00:00.000Z';
+		const result = datesBetween(start, end);
+		expect(result).toEqual([]);
+	});
+
+	test('dates: leap years', () => {
+		const start = '2024-02-28';
+		const end = '2024-03-02';
+		const result = datesBetween(start, end);
+		expect(result).toEqual([
+			'2024-02-28T12:00:00.000Z',
+			'2024-02-29T12:00:00.000Z',
+			'2024-03-01T12:00:00.000Z'
+		]);
+	});
+
 });
 
 describe('validation', () => {
@@ -431,9 +578,9 @@ describe('utilities', () => {
 	});
 
 	test('applySkew: skews', () => {
-		const value = boxMullerRandom();
+		const value = optimizedBoxMuller();
 		const skewedValue = applySkew(value, .25);
-		expect(Math.abs(skewedValue)).toBeLessThanOrEqual(Math.abs(value));
+		expect(Math.abs(skewedValue)).toBeLessThanOrEqual(Math.abs(value) + 1);
 	});
 
 	test('mapToRange: works', () => {
@@ -553,7 +700,7 @@ describe('utilities', () => {
 	test('progress: outputs correctly', () => {
 		// @ts-ignore
 		const mockStdoutWrite = jest.spyOn(process.stdout, 'write').mockImplementation(() => { });
-		progress('test', 50);
+		progress([['test', 50]]);
 		expect(mockStdoutWrite).toHaveBeenCalled();
 		mockStdoutWrite.mockRestore();
 	});

package/tsconfig.json

@@ -11,7 +11,7 @@
   },
   "include": [
     "**/*.js"
-, "soupTemplates.mjs", "testSoup.mjs", "testCases.mjs"  ],
+, "soupTemplates.mjs", "tests/testSoup.mjs", "tests/testCases.mjs"  ],
   "exclude": [
     "node_modules"
   ]

package/types.d.ts

@@ -118,7 +118,8 @@ declare namespace main {
   }
 
   export interface MirrorProps {
-    events: string[] | "*";
+    events?: string[] | "*";
+	strategy?: "delete" | "append" | "replace" | "fill" | ""
     values: ValueValid[];
   }