make-mp-data 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/defaults.js +974 -0
- package/index.js +303 -152
- package/package.json +3 -3
- package/schemas/complex.js +14 -14
- package/schemas/funnels.js +66 -67
- package/schemas/simple.js +14 -4
- package/scratch.mjs +7 -4
- package/tests/unit.test.js +110 -14
- package/types.d.ts +24 -4
- package/utils.js +176 -41
package/index.js
CHANGED
|
@@ -6,26 +6,43 @@ by AK
|
|
|
6
6
|
ak@mixpanel.com
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
+
//todo: ads-data
|
|
10
|
+
//todo: cart analysis
|
|
11
|
+
//todo: churn ... is churnFunnel, possible to return, etc
|
|
12
|
+
//todo: fixedTimeFunnel? if set this funnel will occur for all users at the same time ['cart charged', 'charge complete']
|
|
13
|
+
|
|
14
|
+
/** @typedef {import('./types.d.ts').Config} Config */
|
|
15
|
+
/** @typedef {import('./types.d.ts').EventConfig} EventConfig */
|
|
16
|
+
/** @typedef {import('./types.d.ts').Funnel} Funnel */
|
|
17
|
+
/** @typedef {import('./types.d.ts').Person} Person */
|
|
18
|
+
/** @typedef {import('./types.d.ts').SCDTableRow} SCDTableRow */
|
|
19
|
+
/** @typedef {import('./types.d.ts').UserProfile} UserProfile */
|
|
20
|
+
/** @typedef {import('./types.d.ts').EventSpec} EventSpec */
|
|
21
|
+
|
|
9
22
|
const dayjs = require("dayjs");
|
|
10
23
|
const utc = require("dayjs/plugin/utc");
|
|
11
24
|
dayjs.extend(utc);
|
|
12
25
|
const NOW = dayjs('2024-02-02').unix(); //this is a FIXED POINT and we will shift it later
|
|
13
26
|
global.NOW = NOW;
|
|
14
|
-
|
|
27
|
+
|
|
28
|
+
const os = require("os");
|
|
15
29
|
const path = require("path");
|
|
16
30
|
const { comma, bytesHuman, makeName, md5, clone, tracker, uid } = require("ak-tools");
|
|
17
31
|
const { generateLineChart } = require('./chart.js');
|
|
18
32
|
const { version } = require('./package.json');
|
|
19
|
-
const
|
|
33
|
+
const mp = require("mixpanel-import");
|
|
20
34
|
const metrics = tracker("make-mp-data", "db99eb8f67ae50949a13c27cacf57d41", os.userInfo().username);
|
|
21
35
|
|
|
22
|
-
|
|
23
36
|
const u = require("./utils.js");
|
|
24
|
-
const
|
|
37
|
+
const getCliParams = require("./cli.js");
|
|
38
|
+
const { campaigns, devices, locations } = require('./defaults.js');
|
|
25
39
|
|
|
26
40
|
let VERBOSE = false;
|
|
27
41
|
let isCLI = false;
|
|
42
|
+
/** @type {Config} */
|
|
28
43
|
let CONFIG;
|
|
44
|
+
|
|
45
|
+
let DEFAULTS;
|
|
29
46
|
require('dotenv').config();
|
|
30
47
|
|
|
31
48
|
|
|
@@ -34,25 +51,20 @@ function track(name, props, ...rest) {
|
|
|
34
51
|
metrics(name, props, ...rest);
|
|
35
52
|
}
|
|
36
53
|
|
|
37
|
-
|
|
38
|
-
/** @typedef {import('./types.d.ts').EventConfig} EventConfig */
|
|
39
|
-
/** @typedef {import('./types.d.ts').Funnel} Funnel */
|
|
40
|
-
/** @typedef {import('./types.d.ts').Person} Person */
|
|
41
|
-
/** @typedef {import('./types.d.ts').SCDTableRow} SCDTableRow */
|
|
42
|
-
/** @typedef {import('./types.d.ts').UserProfile} UserProfile */
|
|
43
|
-
/** @typedef {import('./types.d.ts').EventSpec} EventSpec */
|
|
54
|
+
|
|
44
55
|
|
|
45
56
|
/**
|
|
46
57
|
* generates fake mixpanel data
|
|
47
58
|
* @param {Config} config
|
|
48
59
|
*/
|
|
49
60
|
async function main(config) {
|
|
50
|
-
|
|
61
|
+
|
|
62
|
+
//seed the random number generator
|
|
63
|
+
// ^ this is critical; same seed = same data; seed can be passed in as an env var or in the config
|
|
51
64
|
const seedWord = process.env.SEED || config.seed || "hello friend!";
|
|
52
65
|
config.seed = seedWord;
|
|
53
66
|
u.initChance(seedWord);
|
|
54
|
-
const chance = u.getChance();
|
|
55
|
-
config.chance = chance;
|
|
67
|
+
const chance = u.getChance(); // ! this is the only safe way to get the chance instance
|
|
56
68
|
let {
|
|
57
69
|
seed,
|
|
58
70
|
numEvents = 100000,
|
|
@@ -61,10 +73,9 @@ async function main(config) {
|
|
|
61
73
|
epochStart = 0,
|
|
62
74
|
epochEnd = dayjs().unix(),
|
|
63
75
|
events = [{ event: "foo" }, { event: "bar" }, { event: "baz" }],
|
|
64
|
-
superProps = {
|
|
76
|
+
superProps = { luckyNumber: [2, 2, 4, 4, 42, 42, 42, 2, 2, 4, 4, 42, 42, 42, 420] },
|
|
65
77
|
funnels = [],
|
|
66
78
|
userProps = {
|
|
67
|
-
favoriteColor: ["red", "green", "blue", "yellow"],
|
|
68
79
|
spiritAnimal: chance.animal.bind(chance),
|
|
69
80
|
},
|
|
70
81
|
scdProps = {},
|
|
@@ -82,10 +93,20 @@ async function main(config) {
|
|
|
82
93
|
makeChart = false,
|
|
83
94
|
soup = {},
|
|
84
95
|
hook = (record) => record,
|
|
96
|
+
hasAdSpend = false,
|
|
97
|
+
hasCampaigns = false,
|
|
98
|
+
hasLocation = false,
|
|
99
|
+
isAnonymous = false,
|
|
100
|
+
hasBrowser = false,
|
|
101
|
+
hasAndroidDevices = false,
|
|
102
|
+
hasDesktopDevices = false,
|
|
103
|
+
hasIOSDevices = false
|
|
85
104
|
} = config;
|
|
105
|
+
|
|
86
106
|
if (!config.superProps) config.superProps = superProps;
|
|
87
107
|
if (!config.userProps || Object.keys(config?.userProps)) config.userProps = userProps;
|
|
88
|
-
|
|
108
|
+
|
|
109
|
+
|
|
89
110
|
config.simulationName = makeName();
|
|
90
111
|
const { simulationName } = config;
|
|
91
112
|
if (epochStart && !numDays) numDays = dayjs.unix(epochEnd).diff(dayjs.unix(epochStart), "day");
|
|
@@ -117,13 +138,33 @@ async function main(config) {
|
|
|
117
138
|
config.makeChart = makeChart;
|
|
118
139
|
config.soup = soup;
|
|
119
140
|
config.hook = hook;
|
|
120
|
-
|
|
141
|
+
config.hasAdSpend = hasAdSpend;
|
|
142
|
+
config.hasCampaigns = hasCampaigns;
|
|
143
|
+
config.hasLocation = hasLocation;
|
|
144
|
+
config.isAnonymous = isAnonymous;
|
|
145
|
+
config.hasBrowser = hasBrowser;
|
|
146
|
+
config.hasAndroidDevices = hasAndroidDevices;
|
|
147
|
+
config.hasDesktopDevices = hasDesktopDevices;
|
|
148
|
+
config.hasIOSDevices = hasIOSDevices;
|
|
149
|
+
|
|
121
150
|
//event validation
|
|
122
|
-
const validatedEvents =
|
|
151
|
+
const validatedEvents = u.validateEventConfig(events);
|
|
123
152
|
events = validatedEvents;
|
|
124
153
|
config.events = validatedEvents;
|
|
154
|
+
|
|
155
|
+
//globals
|
|
125
156
|
global.MP_SIMULATION_CONFIG = config;
|
|
126
157
|
CONFIG = config;
|
|
158
|
+
VERBOSE = verbose;
|
|
159
|
+
DEFAULTS = {
|
|
160
|
+
locations: u.pickAWinner(locations, 0),
|
|
161
|
+
iOSDevices: u.pickAWinner(devices.iosDevices, 0),
|
|
162
|
+
androidDevices: u.pickAWinner(devices.androidDevices, 0),
|
|
163
|
+
desktopDevices: u.pickAWinner(devices.desktopDevices, 0),
|
|
164
|
+
browsers: u.pickAWinner(devices.browsers, 0),
|
|
165
|
+
campaigns: u.pickAWinner(campaigns, 0),
|
|
166
|
+
};
|
|
167
|
+
|
|
127
168
|
const runId = uid(42);
|
|
128
169
|
let trackingParams = { runId, seed, numEvents, numUsers, numDays, anonIds, sessionIds, format, targetToken: token, region, writeToDisk, isCLI, version };
|
|
129
170
|
track('start simulation', trackingParams);
|
|
@@ -147,46 +188,9 @@ async function main(config) {
|
|
|
147
188
|
|
|
148
189
|
// if no funnels, make some out of events...
|
|
149
190
|
if (!funnels || !funnels.length) {
|
|
150
|
-
|
|
151
|
-
const firstEvents = events.filter((e) => e.isFirstEvent).map((e) => e.event);
|
|
152
|
-
const usageEvents = events.filter((e) => !e.isFirstEvent).map((e) => e.event);
|
|
153
|
-
const numFunnelsToCreate = Math.ceil(usageEvents.length);
|
|
154
|
-
/** @type {Funnel} */
|
|
155
|
-
const funnelTemplate = {
|
|
156
|
-
sequence: [],
|
|
157
|
-
conversionRate: 50,
|
|
158
|
-
order: 'sequential',
|
|
159
|
-
props: {},
|
|
160
|
-
timeToConvert: 1,
|
|
161
|
-
isFirstFunnel: false,
|
|
162
|
-
weight: 1
|
|
163
|
-
};
|
|
164
|
-
if (firstEvents.length) {
|
|
165
|
-
for (const event of firstEvents) {
|
|
166
|
-
createdFunnels.push({ ...clone(funnelTemplate), sequence: [event], isFirstFunnel: true, conversionRate: 100 });
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
//at least one funnel with all usage events
|
|
171
|
-
createdFunnels.push({ ...clone(funnelTemplate), sequence: usageEvents });
|
|
172
|
-
|
|
173
|
-
//for the rest, make random funnels
|
|
174
|
-
followUpFunnels: for (let i = 1; i < numFunnelsToCreate; i++) {
|
|
175
|
-
/** @type {Funnel} */
|
|
176
|
-
const funnel = { ...clone(funnelTemplate) };
|
|
177
|
-
funnel.conversionRate = u.integer(25, 75);
|
|
178
|
-
funnel.timeToConvert = u.integer(1, 10);
|
|
179
|
-
funnel.weight = u.integer(1, 10);
|
|
180
|
-
const sequence = u.shuffleArray(usageEvents).slice(0, u.integer(2, usageEvents.length));
|
|
181
|
-
funnel.sequence = sequence;
|
|
182
|
-
funnel.order = 'random';
|
|
183
|
-
createdFunnels.push(funnel);
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
funnels = createdFunnels;
|
|
191
|
+
funnels = u.inferFunnels(events);
|
|
187
192
|
config.funnels = funnels;
|
|
188
193
|
CONFIG = config;
|
|
189
|
-
|
|
190
194
|
}
|
|
191
195
|
|
|
192
196
|
//user loop
|
|
@@ -194,11 +198,19 @@ async function main(config) {
|
|
|
194
198
|
loopUsers: for (let i = 1; i < numUsers + 1; i++) {
|
|
195
199
|
u.progress("users", i);
|
|
196
200
|
const userId = chance.guid();
|
|
197
|
-
|
|
198
|
-
const user = u.generateUser(userId, numDays);
|
|
201
|
+
const user = u.person(userId, numDays, isAnonymous);
|
|
199
202
|
const { distinct_id, created, anonymousIds, sessionIds } = user;
|
|
200
203
|
let numEventsPreformed = 0;
|
|
201
204
|
|
|
205
|
+
if (hasLocation) {
|
|
206
|
+
const location = u.choose(DEFAULTS.locations().map(l => { delete l.country; return l; }));
|
|
207
|
+
for (const key in location) {
|
|
208
|
+
user[key] = location[key];
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
|
|
202
214
|
// profile creation
|
|
203
215
|
const profile = makeProfile(userProps, user);
|
|
204
216
|
userProfilesData.hookPush(profile);
|
|
@@ -271,8 +283,7 @@ async function main(config) {
|
|
|
271
283
|
u.progress("groups", i);
|
|
272
284
|
const group = {
|
|
273
285
|
[groupKey]: i,
|
|
274
|
-
...makeProfile(groupProps[groupKey])
|
|
275
|
-
// $distinct_id: i,
|
|
286
|
+
...makeProfile(groupProps[groupKey])
|
|
276
287
|
};
|
|
277
288
|
group["distinct_id"] = i;
|
|
278
289
|
groupProfiles.push(group);
|
|
@@ -300,18 +311,24 @@ async function main(config) {
|
|
|
300
311
|
const actualNow = dayjs();
|
|
301
312
|
const fixedNow = dayjs.unix(global.NOW);
|
|
302
313
|
const timeShift = actualNow.diff(fixedNow, "second");
|
|
303
|
-
|
|
314
|
+
|
|
304
315
|
eventData.forEach((event) => {
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
316
|
+
try {
|
|
317
|
+
const newTime = dayjs(event.time).add(timeShift, "second");
|
|
318
|
+
event.time = newTime.toISOString();
|
|
319
|
+
if (epochStart && newTime.unix() < epochStart) event = {};
|
|
320
|
+
if (epochEnd && newTime.unix() > epochEnd) event = {};
|
|
321
|
+
}
|
|
322
|
+
catch (e) {
|
|
323
|
+
//noop
|
|
324
|
+
}
|
|
309
325
|
});
|
|
310
326
|
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
327
|
+
// const dayShift = actualNow.diff(global.NOW, "day");
|
|
328
|
+
// userProfilesData.forEach((profile) => {
|
|
329
|
+
// const newTime = dayjs(profile.created).add(dayShift, "day");
|
|
330
|
+
// profile.created = newTime.toISOString();
|
|
331
|
+
// });
|
|
315
332
|
|
|
316
333
|
|
|
317
334
|
// draw charts
|
|
@@ -405,7 +422,7 @@ async function main(config) {
|
|
|
405
422
|
fixData: true,
|
|
406
423
|
verbose: false,
|
|
407
424
|
forceStream: true,
|
|
408
|
-
strict:
|
|
425
|
+
strict: false, //! sometimes we get events in the future... it happens
|
|
409
426
|
dryRun: false,
|
|
410
427
|
abridged: false,
|
|
411
428
|
fixJson: true,
|
|
@@ -461,6 +478,11 @@ async function main(config) {
|
|
|
461
478
|
};
|
|
462
479
|
}
|
|
463
480
|
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
|
|
464
486
|
/**
|
|
465
487
|
* creates a random event
|
|
466
488
|
* @param {string} distinct_id
|
|
@@ -473,27 +495,42 @@ async function main(config) {
|
|
|
473
495
|
* @param {Boolean} isFirstEvent=false
|
|
474
496
|
*/
|
|
475
497
|
function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, chosenEvent, superProps, groupKeys, isFirstEvent = false) {
|
|
476
|
-
const
|
|
498
|
+
const chance = u.getChance();
|
|
499
|
+
const { mean = 0, deviation = 2, peaks = 5 } = CONFIG.soup;
|
|
500
|
+
const { hasAndroidDevices, hasBrowser, hasCampaigns, hasDesktopDevices, hasIOSDevices, hasLocation } = CONFIG;
|
|
477
501
|
//event model
|
|
478
502
|
const eventTemplate = {
|
|
479
503
|
event: chosenEvent.event,
|
|
480
504
|
source: "dm4",
|
|
481
505
|
};
|
|
482
506
|
|
|
507
|
+
let defaultProps = {};
|
|
508
|
+
let devicePool = [];
|
|
509
|
+
if (hasLocation) defaultProps.location = DEFAULTS.locations().map(l => { delete l.country_code; return l; });
|
|
510
|
+
if (hasBrowser) defaultProps.browser = DEFAULTS.browsers();
|
|
511
|
+
if (hasAndroidDevices) devicePool.push(DEFAULTS.androidDevices());
|
|
512
|
+
if (hasIOSDevices) devicePool.push(DEFAULTS.iOSDevices());
|
|
513
|
+
if (hasDesktopDevices) devicePool.push(DEFAULTS.desktopDevices());
|
|
514
|
+
// we don't always have campaigns, because of attribution
|
|
515
|
+
if (hasCampaigns && chance.bool({ likelihood: 25 })) defaultProps.campaigns = DEFAULTS.campaigns();
|
|
516
|
+
const devices = devicePool.flat();
|
|
517
|
+
if (devices.length) defaultProps.device = devices;
|
|
518
|
+
|
|
519
|
+
|
|
483
520
|
//event time
|
|
484
521
|
if (earliestTime > NOW) {
|
|
485
522
|
earliestTime = dayjs.unix(NOW).subtract(2, 'd').unix();
|
|
486
523
|
};
|
|
487
524
|
|
|
488
525
|
if (isFirstEvent) eventTemplate.time = dayjs.unix(earliestTime).toISOString();
|
|
489
|
-
if (!isFirstEvent) eventTemplate.time = u.TimeSoup(earliestTime, NOW, peaks,
|
|
526
|
+
if (!isFirstEvent) eventTemplate.time = u.TimeSoup(earliestTime, NOW, peaks, deviation, mean);
|
|
490
527
|
|
|
491
528
|
// anonymous and session ids
|
|
492
|
-
if (CONFIG?.anonIds) eventTemplate.device_id =
|
|
493
|
-
if (CONFIG?.sessionIds) eventTemplate.session_id =
|
|
529
|
+
if (CONFIG?.anonIds) eventTemplate.device_id = chance.pickone(anonymousIds);
|
|
530
|
+
if (CONFIG?.sessionIds) eventTemplate.session_id = chance.pickone(sessionIds);
|
|
494
531
|
|
|
495
532
|
//sometimes have a user_id
|
|
496
|
-
if (!isFirstEvent &&
|
|
533
|
+
if (!isFirstEvent && chance.bool({ likelihood: 42 })) eventTemplate.user_id = distinct_id;
|
|
497
534
|
|
|
498
535
|
// ensure that there is a user_id or device_id
|
|
499
536
|
if (!eventTemplate.user_id && !eventTemplate.device_id) eventTemplate.user_id = distinct_id;
|
|
@@ -510,6 +547,34 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, chosenEv
|
|
|
510
547
|
}
|
|
511
548
|
}
|
|
512
549
|
|
|
550
|
+
//iterate through default properties
|
|
551
|
+
for (const key in defaultProps) {
|
|
552
|
+
if (Array.isArray(defaultProps[key])) {
|
|
553
|
+
const choice = u.choose(defaultProps[key]);
|
|
554
|
+
if (typeof choice === "string") {
|
|
555
|
+
if (!eventTemplate[key]) eventTemplate[key] = choice;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
else if (Array.isArray(choice)) {
|
|
559
|
+
for (const subChoice of choice) {
|
|
560
|
+
if (!eventTemplate[key]) eventTemplate[key] = subChoice;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
else if (typeof choice === "object") {
|
|
565
|
+
for (const subKey in choice) {
|
|
566
|
+
if (Array.isArray(choice[subKey])) {
|
|
567
|
+
const subChoice = u.choose(choice[subKey]);
|
|
568
|
+
if (!eventTemplate[subKey]) eventTemplate[subKey] = subChoice;
|
|
569
|
+
}
|
|
570
|
+
if (!eventTemplate[subKey]) eventTemplate[subKey] = choice[subKey];
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
|
|
513
578
|
//iterate through groups
|
|
514
579
|
for (const groupPair of groupKeys) {
|
|
515
580
|
const groupKey = groupPair[0];
|
|
@@ -528,8 +593,8 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, chosenEv
|
|
|
528
593
|
}
|
|
529
594
|
|
|
530
595
|
/**
|
|
531
|
-
*
|
|
532
|
-
* this is called
|
|
596
|
+
* from a funnel spec to a funnel that a user completes/doesn't complete
|
|
597
|
+
* this is called MANY times per user
|
|
533
598
|
* @param {Funnel} funnel
|
|
534
599
|
* @param {Person} user
|
|
535
600
|
* @param {UserProfile} profile
|
|
@@ -539,13 +604,22 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, chosenEv
|
|
|
539
604
|
* @return {[EventSpec[], Boolean]}
|
|
540
605
|
*/
|
|
541
606
|
function makeFunnel(funnel, user, profile, scd, firstEventTime, config) {
|
|
607
|
+
const chance = u.getChance();
|
|
542
608
|
const { hook } = config;
|
|
543
609
|
hook(funnel, "funnel-pre", { user, profile, scd, funnel, config });
|
|
544
|
-
|
|
610
|
+
let {
|
|
611
|
+
sequence,
|
|
612
|
+
conversionRate = 50,
|
|
613
|
+
order = 'sequential',
|
|
614
|
+
timeToConvert = 1,
|
|
615
|
+
props,
|
|
616
|
+
requireRepeats = false,
|
|
617
|
+
} = funnel;
|
|
545
618
|
const { distinct_id, created, anonymousIds, sessionIds } = user;
|
|
546
619
|
const { superProps, groupKeys } = config;
|
|
547
620
|
const { name, email } = profile;
|
|
548
621
|
|
|
622
|
+
//choose the properties for this funnel
|
|
549
623
|
const chosenFunnelProps = { ...props, ...superProps };
|
|
550
624
|
for (const key in props) {
|
|
551
625
|
try {
|
|
@@ -556,32 +630,90 @@ function makeFunnel(funnel, user, profile, scd, firstEventTime, config) {
|
|
|
556
630
|
}
|
|
557
631
|
}
|
|
558
632
|
|
|
559
|
-
const funnelPossibleEvents = sequence
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
633
|
+
const funnelPossibleEvents = sequence
|
|
634
|
+
.map((eventName) => {
|
|
635
|
+
const foundEvent = config.events.find((e) => e.event === eventName);
|
|
636
|
+
/** @type {EventConfig} */
|
|
637
|
+
const eventSpec = foundEvent || { event: eventName, properties: {} };
|
|
638
|
+
for (const key in eventSpec.properties) {
|
|
639
|
+
try {
|
|
640
|
+
eventSpec.properties[key] = u.choose(eventSpec.properties[key]);
|
|
641
|
+
} catch (e) {
|
|
642
|
+
console.error(`error with ${key} in ${eventSpec.event} event`, e);
|
|
643
|
+
debugger;
|
|
644
|
+
}
|
|
569
645
|
}
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
646
|
+
delete eventSpec.isFirstEvent;
|
|
647
|
+
delete eventSpec.weight;
|
|
648
|
+
eventSpec.properties = { ...eventSpec.properties, ...chosenFunnelProps };
|
|
649
|
+
return eventSpec;
|
|
650
|
+
})
|
|
651
|
+
.reduce((acc, step) => {
|
|
652
|
+
if (!requireRepeats) {
|
|
653
|
+
if (acc.find(e => e.event === step.event)) {
|
|
654
|
+
if (chance.bool({ likelihood: 50 })) {
|
|
655
|
+
conversionRate = Math.floor(conversionRate * 1.25); //increase conversion rate
|
|
656
|
+
acc.push(step);
|
|
657
|
+
}
|
|
658
|
+
//A SKIPPED STEP!
|
|
659
|
+
else {
|
|
660
|
+
conversionRate = Math.floor(conversionRate * .75); //reduce conversion rate
|
|
661
|
+
return acc; //early return to skip the step
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
else {
|
|
665
|
+
acc.push(step);
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
else {
|
|
669
|
+
acc.push(step);
|
|
670
|
+
}
|
|
671
|
+
return acc;
|
|
672
|
+
}, []);
|
|
576
673
|
|
|
577
|
-
|
|
674
|
+
let doesUserConvert = chance.bool({ likelihood: conversionRate });
|
|
578
675
|
let numStepsUserWillTake = sequence.length;
|
|
579
676
|
if (!doesUserConvert) numStepsUserWillTake = u.integer(1, sequence.length - 1);
|
|
580
677
|
const funnelTotalRelativeTimeInHours = timeToConvert / numStepsUserWillTake;
|
|
581
678
|
const msInHour = 60000 * 60;
|
|
679
|
+
const funnelStepsUserWillTake = funnelPossibleEvents.slice(0, numStepsUserWillTake);
|
|
680
|
+
|
|
681
|
+
let funnelActualOrder = [];
|
|
682
|
+
|
|
683
|
+
switch (order) {
|
|
684
|
+
case "sequential":
|
|
685
|
+
funnelActualOrder = funnelStepsUserWillTake;
|
|
686
|
+
break;
|
|
687
|
+
case "random":
|
|
688
|
+
funnelActualOrder = u.shuffleArray(funnelStepsUserWillTake);
|
|
689
|
+
break;
|
|
690
|
+
case "first-fixed":
|
|
691
|
+
funnelActualOrder = u.shuffleExceptFirst(funnelStepsUserWillTake);
|
|
692
|
+
break;
|
|
693
|
+
case "last-fixed":
|
|
694
|
+
funnelActualOrder = u.shuffleExceptLast(funnelStepsUserWillTake);
|
|
695
|
+
break;
|
|
696
|
+
case "first-and-last-fixed":
|
|
697
|
+
funnelActualOrder = u.fixFirstAndLast(funnelStepsUserWillTake);
|
|
698
|
+
break;
|
|
699
|
+
case "middle-fixed":
|
|
700
|
+
funnelActualOrder = u.shuffleOutside(funnelStepsUserWillTake);
|
|
701
|
+
break;
|
|
702
|
+
case "interrupted":
|
|
703
|
+
const potentialSubstitutes = config?.events
|
|
704
|
+
?.filter(e => !e.isFirstEvent)
|
|
705
|
+
?.filter(e => !sequence.includes(e.event)) || [];
|
|
706
|
+
funnelActualOrder = u.interruptArray(funnelStepsUserWillTake, potentialSubstitutes);
|
|
707
|
+
break;
|
|
708
|
+
default:
|
|
709
|
+
funnelActualOrder = funnelStepsUserWillTake;
|
|
710
|
+
break;
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
|
|
582
714
|
|
|
583
715
|
let lastTimeJump = 0;
|
|
584
|
-
const
|
|
716
|
+
const funnelActualEventsWithOffset = funnelActualOrder
|
|
585
717
|
.map((event, index) => {
|
|
586
718
|
if (index === 0) {
|
|
587
719
|
event.relativeTimeMs = 0;
|
|
@@ -606,36 +738,9 @@ function makeFunnel(funnel, user, profile, scd, firstEventTime, config) {
|
|
|
606
738
|
});
|
|
607
739
|
|
|
608
740
|
|
|
609
|
-
let funnelActualOrder = [];
|
|
610
|
-
|
|
611
|
-
//todo
|
|
612
|
-
switch (order) {
|
|
613
|
-
case "sequential":
|
|
614
|
-
funnelActualOrder = funnelActualEvents;
|
|
615
|
-
break;
|
|
616
|
-
case "random":
|
|
617
|
-
funnelActualOrder = u.shuffleArray(funnelActualEvents);
|
|
618
|
-
break;
|
|
619
|
-
case "first-fixed":
|
|
620
|
-
funnelActualOrder = u.shuffleExceptFirst(funnelActualEvents);
|
|
621
|
-
break;
|
|
622
|
-
case "last-fixed":
|
|
623
|
-
funnelActualOrder = u.shuffleExceptLast(funnelActualEvents);
|
|
624
|
-
break;
|
|
625
|
-
case "first-and-last-fixed":
|
|
626
|
-
funnelActualOrder = u.fixFirstAndLast(funnelActualEvents);
|
|
627
|
-
break;
|
|
628
|
-
case "middle-fixed":
|
|
629
|
-
funnelActualOrder = u.shuffleOutside(funnelActualEvents);
|
|
630
|
-
break;
|
|
631
|
-
default:
|
|
632
|
-
funnelActualOrder = funnelActualEvents;
|
|
633
|
-
break;
|
|
634
|
-
}
|
|
635
|
-
|
|
636
741
|
const earliestTime = firstEventTime || dayjs(created).unix();
|
|
637
742
|
let funnelStartTime;
|
|
638
|
-
let finalEvents =
|
|
743
|
+
let finalEvents = funnelActualEventsWithOffset
|
|
639
744
|
.map((event, index) => {
|
|
640
745
|
const newEvent = makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, event, {}, groupKeys);
|
|
641
746
|
if (index === 0) {
|
|
@@ -643,9 +748,15 @@ function makeFunnel(funnel, user, profile, scd, firstEventTime, config) {
|
|
|
643
748
|
delete newEvent.relativeTimeMs;
|
|
644
749
|
return newEvent;
|
|
645
750
|
}
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
751
|
+
try {
|
|
752
|
+
newEvent.time = dayjs(funnelStartTime).add(event.relativeTimeMs, "milliseconds").toISOString();
|
|
753
|
+
delete newEvent.relativeTimeMs;
|
|
754
|
+
return newEvent;
|
|
755
|
+
}
|
|
756
|
+
catch (e) {
|
|
757
|
+
|
|
758
|
+
debugger;
|
|
759
|
+
}
|
|
649
760
|
});
|
|
650
761
|
|
|
651
762
|
|
|
@@ -704,29 +815,69 @@ function makeSCD(prop, scdKey, distinct_id, mutations, created) {
|
|
|
704
815
|
return scdEntries;
|
|
705
816
|
}
|
|
706
817
|
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
818
|
+
//todo
|
|
819
|
+
// function makeAdSpend(spec, usersCreated, specialDays = []) {
|
|
820
|
+
// const networks = [
|
|
821
|
+
// ...new Set(
|
|
822
|
+
// attribution
|
|
823
|
+
// .filter(camp => !["$organic", "$referral"].includes(camp.utm_source.join()))
|
|
824
|
+
// .map(network => network.utm_source)
|
|
825
|
+
// .flat()
|
|
826
|
+
// )
|
|
827
|
+
// ];
|
|
828
|
+
|
|
829
|
+
// const campaigns = attribution.slice().pop().utm_campaign;
|
|
830
|
+
// const campaignParams = attribution.slice().pop();
|
|
831
|
+
// const { startDate, endDate } = spec;
|
|
832
|
+
// const days = getDeltaDays(startDate, endDate);
|
|
833
|
+
// const numDays = days.length;
|
|
834
|
+
// const data = [];
|
|
835
|
+
// for (const network of networks) {
|
|
836
|
+
// for (const day of days) {
|
|
837
|
+
// //cost per acquisition ~ $10-50
|
|
838
|
+
// let CAC = chance.integer({ min: 10, max: 50 });
|
|
839
|
+
// // daily spend is total users / total days * CAC / num networks
|
|
840
|
+
// let cost = Math.floor(((usersCreated / numDays) * CAC) / networks.length);
|
|
841
|
+
// // CTR ~ 0.5-10%
|
|
842
|
+
// let clicks = Math.floor(cost / chance.floating({ min: 0.5, max: 10 }));
|
|
843
|
+
// //boost CTR on "special days" ~ 15-50%
|
|
844
|
+
// if (day in specialDays) clicks *= Math.floor(clicks * chance.floating({ min: 1.15, max: 1.5 }));
|
|
845
|
+
// //impressions ~100-500 * cost
|
|
846
|
+
// let impressions = Math.floor(cost * chance.integer({ min: 100, max: 500 }));
|
|
847
|
+
// // views ~ 25-80% of impressions
|
|
848
|
+
// let views = Math.floor(impressions * chance.floating({ min: 0.25, max: 0.8 }));
|
|
849
|
+
|
|
850
|
+
// let campaign_name = chance.pickone(campaigns);
|
|
851
|
+
// let campaign_id = md5(campaign_name);
|
|
852
|
+
// data.push({
|
|
853
|
+
// event: "ad data",
|
|
854
|
+
// properties: {
|
|
855
|
+
// cost,
|
|
856
|
+
// clicks,
|
|
857
|
+
// impressions,
|
|
858
|
+
// campaign_name,
|
|
859
|
+
// campaign_id,
|
|
860
|
+
// network,
|
|
861
|
+
// views,
|
|
862
|
+
|
|
863
|
+
// //addendum
|
|
864
|
+
// utm_campaign: campaign_name,
|
|
865
|
+
// utm_source: network,
|
|
866
|
+
// utm_medium: campaignParams.utm_medium,
|
|
867
|
+
// utm_content: campaignParams.utm_content,
|
|
868
|
+
// utm_term: campaignParams.utm_term,
|
|
869
|
+
|
|
870
|
+
// source: network,
|
|
871
|
+
// $insert_id: campaign_id,
|
|
872
|
+
// time: dayjs.utc(day).hour(12).unix(),
|
|
873
|
+
// $source: "DM3",
|
|
874
|
+
// $mp_lib: "DM3"
|
|
875
|
+
// }
|
|
876
|
+
// });
|
|
877
|
+
// }
|
|
878
|
+
// }
|
|
879
|
+
// return data;
|
|
880
|
+
// }
|
|
730
881
|
|
|
731
882
|
|
|
732
883
|
|
|
@@ -737,7 +888,7 @@ function validateEvents(events) {
|
|
|
737
888
|
// this is for CLI
|
|
738
889
|
if (require.main === module) {
|
|
739
890
|
isCLI = true;
|
|
740
|
-
const args =
|
|
891
|
+
const args = getCliParams();
|
|
741
892
|
// @ts-ignore
|
|
742
893
|
let { token, seed, format, numDays, numUsers, numEvents, region, writeToDisk, complex = false, sessionIds, anonIds } = args;
|
|
743
894
|
// @ts-ignore
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "make-mp-data",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.02",
|
|
4
4
|
"description": "builds all mixpanel primitives for a given project",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"types": "types.d.ts",
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
"chartjs-node-canvas": "^4.1.6",
|
|
48
48
|
"dayjs": "^1.11.11",
|
|
49
49
|
"dotenv": "^16.4.5",
|
|
50
|
-
"mixpanel-import": "^2.5.
|
|
50
|
+
"mixpanel-import": "^2.5.55",
|
|
51
51
|
"yargs": "^17.7.2"
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
@@ -57,4 +57,4 @@
|
|
|
57
57
|
"jest": {
|
|
58
58
|
"preset": "./tests/jest.config.js"
|
|
59
59
|
}
|
|
60
|
-
}
|
|
60
|
+
}
|