make-mp-data 1.0.16 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli.js +6 -3
- package/default.js +26 -4
- package/e2e.test.js +13 -2
- package/index.js +115 -81
- package/package.json +11 -10
- package/scripts/deps.sh +3 -0
- package/scripts/go.sh +2 -0
- package/types.d.ts +14 -22
- package/utils.js +103 -3
package/cli.js
CHANGED
|
@@ -18,10 +18,13 @@ function cliParams() {
|
|
|
18
18
|
.scriptName("make-mp-data")
|
|
19
19
|
.usage(`\nusage:\nnpx $0 [dataModel.js] [options]
|
|
20
20
|
ex:
|
|
21
|
-
npx $0
|
|
22
|
-
npx $0
|
|
21
|
+
npx $0
|
|
22
|
+
npx $0 --token 1234 --u 100 --e 1000 --d 7 --w false
|
|
23
|
+
npx $0 myDataConfig.js
|
|
23
24
|
|
|
24
|
-
DOCS: https://github.com/ak--47/make-mp-data
|
|
25
|
+
DOCS: https://github.com/ak--47/make-mp-data
|
|
26
|
+
DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
|
|
27
|
+
`)
|
|
25
28
|
.command('$0', 'model mixpanel data', () => { })
|
|
26
29
|
.option("token", {
|
|
27
30
|
demandOption: false,
|
package/default.js
CHANGED
|
@@ -1,7 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This is the default configuration file for the data generator
|
|
3
|
+
* notice how the config object is structured, and see it's type definition in ./types.d.ts
|
|
4
|
+
* feel free to modify this file to customize the data you generate
|
|
5
|
+
* see helper functions in utils.js for more ways to generate data
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
|
|
1
9
|
const Chance = require('chance');
|
|
2
10
|
const chance = new Chance();
|
|
3
|
-
const { weightedRange, makeProducts, date, generateEmoji } = require('./utils.js');
|
|
11
|
+
const { weightedRange, makeProducts, date, generateEmoji, makeHashTags } = require('./utils.js');
|
|
4
12
|
|
|
13
|
+
/** @type {import('./types.d.ts').Config} */
|
|
5
14
|
const config = {
|
|
6
15
|
token: "",
|
|
7
16
|
seed: "foo bar baz",
|
|
@@ -40,6 +49,19 @@ const config = {
|
|
|
40
49
|
utm_source: ["$organic", "$organic", "$organic", "$organic", "google", "google", "google", "facebook", "facebook", "twitter", "linkedin"],
|
|
41
50
|
}
|
|
42
51
|
},
|
|
52
|
+
{
|
|
53
|
+
"event": "watch video",
|
|
54
|
+
"weight": 8,
|
|
55
|
+
"properties": {
|
|
56
|
+
category: ["funny", "educational", "inspirational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"],
|
|
57
|
+
hashTags: makeHashTags,
|
|
58
|
+
watchTimeSec: weightedRange(10, 600, 1000, .25),
|
|
59
|
+
quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
|
|
60
|
+
format: ["mp4", "avi", "mov", "mpg"],
|
|
61
|
+
uploader_id: chance.guid.bind(chance)
|
|
62
|
+
|
|
63
|
+
}
|
|
64
|
+
},
|
|
43
65
|
{
|
|
44
66
|
"event": "view item",
|
|
45
67
|
"weight": 8,
|
|
@@ -67,7 +89,7 @@ const config = {
|
|
|
67
89
|
}
|
|
68
90
|
],
|
|
69
91
|
superProps: {
|
|
70
|
-
platform: ["web", "mobile", "web", "mobile", "web", "kiosk"],
|
|
92
|
+
platform: ["web", "mobile", "web", "mobile", "web", "kiosk", "smartTV"],
|
|
71
93
|
emotions: generateEmoji(),
|
|
72
94
|
|
|
73
95
|
},
|
|
@@ -78,7 +100,7 @@ const config = {
|
|
|
78
100
|
userProps: {
|
|
79
101
|
title: chance.profession.bind(chance),
|
|
80
102
|
luckyNumber: weightedRange(42, 420),
|
|
81
|
-
vibe: generateEmoji(),
|
|
103
|
+
vibe: generateEmoji(),
|
|
82
104
|
spiritAnimal: chance.animal.bind(chance)
|
|
83
105
|
},
|
|
84
106
|
|
|
@@ -101,7 +123,7 @@ const config = {
|
|
|
101
123
|
groupProps: {
|
|
102
124
|
company_id: {
|
|
103
125
|
$name: () => { return chance.company(); },
|
|
104
|
-
$email: () => { return `CSM ${chance.pickone(["AK", "Jessica", "Michelle", "Dana", "Brian", "Dave"])}`; },
|
|
126
|
+
$email: () => { return `CSM: ${chance.pickone(["AK", "Jessica", "Michelle", "Dana", "Brian", "Dave"])}`; },
|
|
105
127
|
"# of employees": weightedRange(3, 10000),
|
|
106
128
|
"sector": ["tech", "finance", "healthcare", "education", "government", "non-profit"],
|
|
107
129
|
"segment": ["enterprise", "SMB", "mid-market"],
|
package/e2e.test.js
CHANGED
|
@@ -9,15 +9,16 @@ const { execSync } = require("child_process");
|
|
|
9
9
|
const u = require('ak-tools');
|
|
10
10
|
|
|
11
11
|
const timeout = 60000;
|
|
12
|
+
const testToken = process.env.TEST_TOKEN;
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
describe('e2e', () => {
|
|
15
16
|
|
|
16
17
|
test('works as module', async () => {
|
|
17
18
|
console.log('MODULE TEST');
|
|
18
|
-
const results = await generate({ writeToDisk: false, numEvents:
|
|
19
|
+
const results = await generate({ writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
|
|
19
20
|
const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
|
|
20
|
-
expect(eventData.length).toBeGreaterThan(
|
|
21
|
+
expect(eventData.length).toBeGreaterThan(980);
|
|
21
22
|
expect(groupProfilesData.length).toBe(0);
|
|
22
23
|
expect(lookupTableData.length).toBe(0);
|
|
23
24
|
expect(scdTableData.length).toBeGreaterThan(200);
|
|
@@ -33,6 +34,16 @@ describe('e2e', () => {
|
|
|
33
34
|
expect(csvs.length).toBe(5);
|
|
34
35
|
}, timeout);
|
|
35
36
|
|
|
37
|
+
test('sends data to mixpanel', async () => {
|
|
38
|
+
console.log('NETWORK TEST');
|
|
39
|
+
const results = await generate({ writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it", token: testToken});
|
|
40
|
+
const {events, users, groups } = results.import;
|
|
41
|
+
expect(events.success).toBeGreaterThan(980);
|
|
42
|
+
expect(users.success).toBe(100);
|
|
43
|
+
expect(groups.length).toBe(0);
|
|
44
|
+
|
|
45
|
+
}, timeout);
|
|
46
|
+
|
|
36
47
|
|
|
37
48
|
|
|
38
49
|
});
|
package/index.js
CHANGED
|
@@ -35,11 +35,15 @@ const dayjs = require("dayjs");
|
|
|
35
35
|
const utc = require("dayjs/plugin/utc");
|
|
36
36
|
dayjs.extend(utc);
|
|
37
37
|
const cliParams = require("./cli.js");
|
|
38
|
-
|
|
38
|
+
const { makeName } = require('ak-tools');
|
|
39
|
+
|
|
39
40
|
Array.prototype.pickOne = pick;
|
|
40
41
|
const NOW = dayjs().unix();
|
|
42
|
+
let VERBOSE = false;
|
|
41
43
|
|
|
42
44
|
/** @typedef {import('./types.d.ts').Config} Config */
|
|
45
|
+
/** @typedef {import('./types.d.ts').EventConfig} EventConfig */
|
|
46
|
+
|
|
43
47
|
|
|
44
48
|
const PEAK_DAYS = [
|
|
45
49
|
dayjs().subtract(2, "day").unix(),
|
|
@@ -78,18 +82,10 @@ async function main(config) {
|
|
|
78
82
|
token = null,
|
|
79
83
|
region = "US",
|
|
80
84
|
writeToDisk = false,
|
|
85
|
+
verbose = false,
|
|
81
86
|
} = config;
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
if (require.main === module) {
|
|
85
|
-
if (!token) {
|
|
86
|
-
if (!writeToDisk) {
|
|
87
|
-
writeToDisk = true;
|
|
88
|
-
config.writeToDisk = true;
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
|
|
87
|
+
VERBOSE = verbose;
|
|
88
|
+
config.simulationName = makeName();
|
|
93
89
|
const uuidChance = new Chance(seed);
|
|
94
90
|
|
|
95
91
|
//the function which generates $distinct_id + $created, skewing towards the present
|
|
@@ -115,10 +111,12 @@ async function main(config) {
|
|
|
115
111
|
.reduce((acc, event) => {
|
|
116
112
|
const weight = event.weight || 1;
|
|
117
113
|
for (let i = 0; i < weight; i++) {
|
|
114
|
+
|
|
118
115
|
acc.push(event);
|
|
119
116
|
}
|
|
120
117
|
return acc;
|
|
121
118
|
}, [])
|
|
119
|
+
|
|
122
120
|
.filter((e) => !e.isFirstEvent);
|
|
123
121
|
|
|
124
122
|
const firstEvents = events.filter((e) => e.isFirstEvent);
|
|
@@ -133,18 +131,19 @@ async function main(config) {
|
|
|
133
131
|
for (let i = 1; i < numUsers + 1; i++) {
|
|
134
132
|
progress("users", i);
|
|
135
133
|
const user = uuid();
|
|
136
|
-
const { distinct_id, $created } = user;
|
|
134
|
+
const { distinct_id, $created, anonymousIds } = user;
|
|
137
135
|
userProfilesData.push(makeProfile(userProps, user));
|
|
138
|
-
const mutations = chance.integer({ min: 1, max:
|
|
136
|
+
const mutations = chance.integer({ min: 1, max: 10 });
|
|
139
137
|
scdTableData.push(makeSCD(scdProps, distinct_id, mutations, $created));
|
|
140
138
|
const numEventsThisUser = Math.round(
|
|
141
|
-
chance.normal({ mean: avgEvPerUser, dev: avgEvPerUser /
|
|
139
|
+
chance.normal({ mean: avgEvPerUser, dev: avgEvPerUser / integer(3, 7) })
|
|
142
140
|
);
|
|
143
141
|
|
|
144
142
|
if (firstEvents.length) {
|
|
145
143
|
eventData.push(
|
|
146
144
|
makeEvent(
|
|
147
145
|
distinct_id,
|
|
146
|
+
anonymousIds,
|
|
148
147
|
dayjs($created).unix(),
|
|
149
148
|
firstEvents,
|
|
150
149
|
superProps,
|
|
@@ -159,6 +158,7 @@ async function main(config) {
|
|
|
159
158
|
eventData.push(
|
|
160
159
|
makeEvent(
|
|
161
160
|
distinct_id,
|
|
161
|
+
anonymousIds,
|
|
162
162
|
dayjs($created).unix(),
|
|
163
163
|
weightedEvents,
|
|
164
164
|
superProps,
|
|
@@ -170,7 +170,7 @@ async function main(config) {
|
|
|
170
170
|
//flatten SCD
|
|
171
171
|
scdTableData = scdTableData.flat();
|
|
172
172
|
|
|
173
|
-
|
|
173
|
+
log("\n");
|
|
174
174
|
|
|
175
175
|
// make group profiles
|
|
176
176
|
for (const groupPair of groupKeys) {
|
|
@@ -188,7 +188,7 @@ async function main(config) {
|
|
|
188
188
|
}
|
|
189
189
|
groupProfilesData.push({ key: groupKey, data: groupProfiles });
|
|
190
190
|
}
|
|
191
|
-
|
|
191
|
+
log("\n");
|
|
192
192
|
|
|
193
193
|
// make lookup tables
|
|
194
194
|
for (const lookupTable of lookupTables) {
|
|
@@ -213,7 +213,7 @@ async function main(config) {
|
|
|
213
213
|
[groupFiles, groupProfilesData],
|
|
214
214
|
[lookupFiles, lookupTableData],
|
|
215
215
|
];
|
|
216
|
-
|
|
216
|
+
log("\n");
|
|
217
217
|
|
|
218
218
|
if (!writeToDisk && !token)
|
|
219
219
|
return {
|
|
@@ -224,79 +224,89 @@ async function main(config) {
|
|
|
224
224
|
lookupTableData,
|
|
225
225
|
};
|
|
226
226
|
//write the files
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
for (const
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
227
|
+
if (writeToDisk) {
|
|
228
|
+
if (verbose) log(`writing files... for ${config.simulationName}`);
|
|
229
|
+
for (const pair of pairs) {
|
|
230
|
+
const [paths, data] = pair;
|
|
231
|
+
for (const path of paths) {
|
|
232
|
+
let datasetsToWrite;
|
|
233
|
+
if (data?.[0]?.["key"]) datasetsToWrite = data.map((d) => d.data);
|
|
234
|
+
else datasetsToWrite = [data];
|
|
235
|
+
for (const writeData of datasetsToWrite) {
|
|
236
|
+
if (format === "csv") {
|
|
237
|
+
log(`writing ${path}`);
|
|
238
|
+
const columns = getUniqueKeys(writeData);
|
|
239
|
+
//papa parse needs nested JSON stringified
|
|
240
|
+
writeData.forEach((e) => {
|
|
241
|
+
for (const key in e) {
|
|
242
|
+
if (typeof e[key] === "object") e[key] = JSON.stringify(e[key]);
|
|
243
|
+
}
|
|
244
|
+
});
|
|
245
|
+
const csv = Papa.unparse(writeData, { columns });
|
|
246
|
+
await touch(path, csv);
|
|
247
|
+
log(`\tdone\n`);
|
|
248
|
+
} else {
|
|
249
|
+
await touch(path, data, true);
|
|
250
|
+
}
|
|
248
251
|
}
|
|
249
252
|
}
|
|
250
253
|
}
|
|
251
254
|
}
|
|
252
255
|
|
|
253
256
|
const importResults = { events: {}, users: {}, groups: [] };
|
|
254
|
-
|
|
255
|
-
const creds = { token };
|
|
256
|
-
/** @type {import('mixpanel-import').Options} */
|
|
257
|
-
const importOpts = {
|
|
258
|
-
region,
|
|
259
|
-
fixData: true,
|
|
260
|
-
verbose: false,
|
|
261
|
-
forceStream: true,
|
|
262
|
-
strict: false,
|
|
263
|
-
dryRun: false,
|
|
264
|
-
abridged: false,
|
|
265
|
-
};
|
|
257
|
+
|
|
266
258
|
//send to mixpanel
|
|
267
259
|
if (token) {
|
|
260
|
+
/** @type {import('mixpanel-import').Creds} */
|
|
261
|
+
const creds = { token };
|
|
262
|
+
/** @type {import('mixpanel-import').Options} */
|
|
263
|
+
const commonOpts = {
|
|
264
|
+
|
|
265
|
+
region,
|
|
266
|
+
fixData: true,
|
|
267
|
+
verbose: false,
|
|
268
|
+
forceStream: true,
|
|
269
|
+
strict: false,
|
|
270
|
+
dryRun: false,
|
|
271
|
+
abridged: false,
|
|
272
|
+
};
|
|
273
|
+
|
|
268
274
|
if (eventData) {
|
|
269
|
-
|
|
275
|
+
log(`importing events to mixpanel...`);
|
|
270
276
|
const imported = await mp(creds, eventData, {
|
|
271
277
|
recordType: "event",
|
|
272
|
-
|
|
278
|
+
fixData: true,
|
|
279
|
+
fixJson: true,
|
|
280
|
+
strict: false,
|
|
281
|
+
...commonOpts,
|
|
273
282
|
});
|
|
274
|
-
|
|
283
|
+
log(`\tsent ${comma(imported.success)} events\n`);
|
|
275
284
|
importResults.events = imported;
|
|
276
285
|
}
|
|
277
286
|
if (userProfilesData) {
|
|
278
|
-
|
|
287
|
+
log(`importing user profiles to mixpanel...`);
|
|
279
288
|
const imported = await mp(creds, userProfilesData, {
|
|
280
289
|
recordType: "user",
|
|
281
|
-
...
|
|
290
|
+
...commonOpts,
|
|
282
291
|
});
|
|
283
|
-
|
|
292
|
+
log(`\tsent ${comma(imported.success)} user profiles\n`);
|
|
284
293
|
importResults.users = imported;
|
|
285
294
|
}
|
|
286
295
|
if (groupProfilesData) {
|
|
287
296
|
for (const groupProfiles of groupProfilesData) {
|
|
288
297
|
const groupKey = groupProfiles.key;
|
|
289
298
|
const data = groupProfiles.data;
|
|
290
|
-
|
|
299
|
+
log(`importing ${groupKey} profiles to mixpanel...`);
|
|
291
300
|
const imported = await mp({ token, groupKey }, data, {
|
|
292
301
|
recordType: "group",
|
|
293
|
-
...
|
|
302
|
+
...commonOpts,
|
|
294
303
|
});
|
|
295
|
-
|
|
304
|
+
log(`\tsent ${comma(imported.success)} ${groupKey} profiles\n`);
|
|
305
|
+
|
|
296
306
|
importResults.groups.push(imported);
|
|
297
307
|
}
|
|
298
308
|
}
|
|
299
|
-
|
|
309
|
+
log(`\n\n`);
|
|
300
310
|
}
|
|
301
311
|
return {
|
|
302
312
|
import: importResults,
|
|
@@ -341,19 +351,33 @@ function makeSCD(props, distinct_id, mutations, $created) {
|
|
|
341
351
|
return scdEntries;
|
|
342
352
|
}
|
|
343
353
|
|
|
344
|
-
|
|
354
|
+
/**
|
|
355
|
+
* creates a random event
|
|
356
|
+
* @param {string} distinct_id
|
|
357
|
+
* @param {string[]} anonymousIds
|
|
358
|
+
* @param {number} earliestTime
|
|
359
|
+
* @param {Object[]} events
|
|
360
|
+
* @param {Object} superProps
|
|
361
|
+
* @param {Object} groupKeys
|
|
362
|
+
* @param {Boolean} isFirstEvent=false
|
|
363
|
+
*/
|
|
364
|
+
function makeEvent(distinct_id, anonymousIds, earliestTime, events, superProps, groupKeys, isFirstEvent = false) {
|
|
365
|
+
|
|
345
366
|
let chosenEvent = events.pickOne();
|
|
346
367
|
if (typeof chosenEvent === "string")
|
|
347
368
|
chosenEvent = { event: chosenEvent, properties: {} };
|
|
348
369
|
const event = {
|
|
349
370
|
event: chosenEvent.event,
|
|
350
|
-
|
|
371
|
+
$device_id: chance.pickone(anonymousIds), // always have a $device_id
|
|
351
372
|
$source: "AKsTimeSoup",
|
|
352
373
|
};
|
|
353
374
|
|
|
354
|
-
if (isFirstEvent) event.time = earliestTime;
|
|
375
|
+
if (isFirstEvent) event.time = dayjs.unix(earliestTime).toISOString();
|
|
355
376
|
if (!isFirstEvent) event.time = AKsTimeSoup(earliestTime, NOW, PEAK_DAYS);
|
|
356
377
|
|
|
378
|
+
//sometimes have a $user_id
|
|
379
|
+
if (!isFirstEvent && chance.bool({ likelihood: 42 })) event.$user_id = distinct_id;
|
|
380
|
+
|
|
357
381
|
const props = { ...chosenEvent.properties, ...superProps };
|
|
358
382
|
|
|
359
383
|
//iterate through custom properties
|
|
@@ -361,6 +385,7 @@ function makeEvent(distinct_id, earliestTime, events, superProps, groupKeys, isF
|
|
|
361
385
|
try {
|
|
362
386
|
event[key] = choose(props[key]);
|
|
363
387
|
} catch (e) {
|
|
388
|
+
console.error(`error with ${key} in ${chosenEvent.event} event`, e);
|
|
364
389
|
debugger;
|
|
365
390
|
}
|
|
366
391
|
}
|
|
@@ -369,6 +394,7 @@ function makeEvent(distinct_id, earliestTime, events, superProps, groupKeys, isF
|
|
|
369
394
|
for (const groupPair of groupKeys) {
|
|
370
395
|
const groupKey = groupPair[0];
|
|
371
396
|
const groupCardinality = groupPair[1];
|
|
397
|
+
|
|
372
398
|
event[groupKey] = weightedRange(1, groupCardinality).pickOne();
|
|
373
399
|
}
|
|
374
400
|
|
|
@@ -378,14 +404,15 @@ function makeEvent(distinct_id, earliestTime, events, superProps, groupKeys, isF
|
|
|
378
404
|
function buildFileNames(config) {
|
|
379
405
|
const { format = "csv", groupKeys = [], lookupTables = [] } = config;
|
|
380
406
|
const extension = format === "csv" ? "csv" : "json";
|
|
381
|
-
const current = dayjs.utc().format("MM-DD-HH");
|
|
407
|
+
// const current = dayjs.utc().format("MM-DD-HH");
|
|
408
|
+
const simName = config.simulationName;
|
|
382
409
|
let writeDir = "./";
|
|
383
410
|
if (config.writeToDisk) writeDir = mkdir("./data");
|
|
384
411
|
|
|
385
412
|
const writePaths = {
|
|
386
|
-
eventFiles: [path.join(writeDir, `events-${
|
|
387
|
-
userFiles: [path.join(writeDir, `users-${
|
|
388
|
-
scdFiles: [path.join(writeDir, `scd-${
|
|
413
|
+
eventFiles: [path.join(writeDir, `events-${simName}.${extension}`)],
|
|
414
|
+
userFiles: [path.join(writeDir, `users-${simName}.${extension}`)],
|
|
415
|
+
scdFiles: [path.join(writeDir, `scd-${simName}.${extension}`)],
|
|
389
416
|
groupFiles: [],
|
|
390
417
|
lookupFiles: [],
|
|
391
418
|
folder: writeDir,
|
|
@@ -394,14 +421,14 @@ function buildFileNames(config) {
|
|
|
394
421
|
for (const groupPair of groupKeys) {
|
|
395
422
|
const groupKey = groupPair[0];
|
|
396
423
|
writePaths.groupFiles.push(
|
|
397
|
-
path.join(writeDir, `group-${groupKey}-${
|
|
424
|
+
path.join(writeDir, `group-${groupKey}-${simName}.${extension}`)
|
|
398
425
|
);
|
|
399
426
|
}
|
|
400
427
|
|
|
401
428
|
for (const lookupTable of lookupTables) {
|
|
402
429
|
const { key } = lookupTable;
|
|
403
430
|
writePaths.lookupFiles.push(
|
|
404
|
-
path.join(writeDir, `lookup-${key}-${
|
|
431
|
+
path.join(writeDir, `lookup-${key}-${simName}.${extension}`)
|
|
405
432
|
);
|
|
406
433
|
}
|
|
407
434
|
|
|
@@ -450,7 +477,7 @@ function AKsTimeSoup(earliestTime, latestTime = NOW, peakDays = PEAK_DAYS) {
|
|
|
450
477
|
// usually, ensure the event time is within business hours
|
|
451
478
|
if (chance.bool({ likelihood: 42 })) eventTime = Math.min(Math.max(eventTime, businessStart), businessEnd);
|
|
452
479
|
|
|
453
|
-
return eventTime;
|
|
480
|
+
return dayjs.unix(eventTime).toISOString();
|
|
454
481
|
}
|
|
455
482
|
|
|
456
483
|
|
|
@@ -464,10 +491,10 @@ if (require.main === module) {
|
|
|
464
491
|
//if the user specifics an separate config file
|
|
465
492
|
let config = null;
|
|
466
493
|
if (suppliedConfig) {
|
|
467
|
-
|
|
494
|
+
log(`using ${suppliedConfig} for data\n`);
|
|
468
495
|
config = require(path.resolve(suppliedConfig));
|
|
469
496
|
} else {
|
|
470
|
-
|
|
497
|
+
log(`... using default configuration ...\n`);
|
|
471
498
|
config = require("./default.js");
|
|
472
499
|
}
|
|
473
500
|
|
|
@@ -480,13 +507,15 @@ if (require.main === module) {
|
|
|
480
507
|
if (numEvents) config.numEvents = numEvents;
|
|
481
508
|
if (region) config.region = region;
|
|
482
509
|
if (writeToDisk) config.writeToDisk = writeToDisk;
|
|
510
|
+
if (writeToDisk === 'false') config.writeToDisk = false;
|
|
511
|
+
config.verbose = true;
|
|
483
512
|
|
|
484
513
|
main(config)
|
|
485
514
|
.then((data) => {
|
|
486
|
-
|
|
515
|
+
log(`------------------SUMMARY------------------`);
|
|
487
516
|
const { events, groups, users } = data.import;
|
|
488
517
|
const files = data.files;
|
|
489
|
-
const folder = files
|
|
518
|
+
const folder = files?.pop();
|
|
490
519
|
const groupBytes = groups.reduce((acc, group) => {
|
|
491
520
|
return acc + group.bytes;
|
|
492
521
|
}, 0);
|
|
@@ -501,18 +530,18 @@ if (require.main === module) {
|
|
|
501
530
|
bytes: bytesHuman(bytes || 0),
|
|
502
531
|
};
|
|
503
532
|
if (bytes > 0) console.table(stats);
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
533
|
+
log(`\nfiles written to ${folder}...`);
|
|
534
|
+
log("\t" + files?.flat().join("\n\t"));
|
|
535
|
+
log(`\n------------------SUMMARY------------------\n\n\n`);
|
|
507
536
|
})
|
|
508
537
|
.catch((e) => {
|
|
509
|
-
|
|
538
|
+
log(`------------------ERROR------------------`);
|
|
510
539
|
console.error(e);
|
|
511
|
-
|
|
540
|
+
log(`------------------ERROR------------------`);
|
|
512
541
|
debugger;
|
|
513
542
|
})
|
|
514
543
|
.finally(() => {
|
|
515
|
-
|
|
544
|
+
log("have a wonderful day :)");
|
|
516
545
|
openFinder(path.resolve("./data"));
|
|
517
546
|
});
|
|
518
547
|
} else {
|
|
@@ -532,3 +561,8 @@ if (require.main === module) {
|
|
|
532
561
|
openFinder,
|
|
533
562
|
};
|
|
534
563
|
}
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
function log(...args) {
|
|
567
|
+
if (VERBOSE) console.log(...args);
|
|
568
|
+
}
|
package/package.json
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "make-mp-data",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "builds all mixpanel primitives for a given project",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"types": "types.d.ts",
|
|
7
7
|
"scripts": {
|
|
8
8
|
"start": "node index.js",
|
|
9
9
|
"prune": "rm ./data/*",
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
"go": "sh ./scripts/go.sh",
|
|
11
|
+
"post": "npm publish",
|
|
12
|
+
"test": "jest",
|
|
13
|
+
"deps": "sh ./scripts/deps.sh"
|
|
13
14
|
},
|
|
14
15
|
"repository": {
|
|
15
16
|
"type": "git",
|
|
@@ -25,10 +26,10 @@
|
|
|
25
26
|
"tracking",
|
|
26
27
|
"server",
|
|
27
28
|
"CLI",
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
"datamart",
|
|
30
|
+
"scd 2",
|
|
31
|
+
"dummy data",
|
|
32
|
+
"fake data"
|
|
32
33
|
],
|
|
33
34
|
"author": "ak@mixpanel.com",
|
|
34
35
|
"license": "ISC",
|
|
@@ -37,10 +38,10 @@
|
|
|
37
38
|
},
|
|
38
39
|
"homepage": "https://github.com/ak--47/make-mp-data#readme",
|
|
39
40
|
"dependencies": {
|
|
40
|
-
"ak-tools": "^1.0.
|
|
41
|
+
"ak-tools": "^1.0.52",
|
|
41
42
|
"chance": "^1.1.7",
|
|
42
43
|
"dayjs": "^1.11.10",
|
|
43
|
-
"mixpanel-import": "^2.5.
|
|
44
|
+
"mixpanel-import": "^2.5.5",
|
|
44
45
|
"yargs": "^17.7.2"
|
|
45
46
|
}
|
|
46
47
|
}
|
package/scripts/deps.sh
ADDED
package/scripts/go.sh
ADDED
package/types.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// types.d.ts
|
|
2
|
-
|
|
2
|
+
|
|
3
|
+
type primitives = string | number | boolean | Date | Object;
|
|
4
|
+
type valueValid = primitives | primitives[] | (() => primitives | primitives[]);
|
|
3
5
|
|
|
4
6
|
export interface Config {
|
|
5
7
|
token?: string;
|
|
@@ -10,44 +12,34 @@ export interface Config {
|
|
|
10
12
|
format?: "csv" | "json";
|
|
11
13
|
region?: string;
|
|
12
14
|
events?: EventConfig[];
|
|
13
|
-
superProps?: Record<string,
|
|
14
|
-
userProps?: Record<string,
|
|
15
|
-
scdProps?:
|
|
16
|
-
plan?: string[];
|
|
17
|
-
MRR?: number;
|
|
18
|
-
NPS?: number;
|
|
19
|
-
marketingOptIn?: boolean[];
|
|
20
|
-
dateOfRenewal?: Date;
|
|
21
|
-
};
|
|
15
|
+
superProps?: Record<string, valueValid>;
|
|
16
|
+
userProps?: Record<string, valueValid>;
|
|
17
|
+
scdProps?: Record<string, valueValid>;
|
|
22
18
|
groupKeys?: [string, number][];
|
|
23
19
|
groupProps?: Record<string, GroupProperty>; // Adjust according to usage
|
|
24
20
|
lookupTables?: LookupTable[];
|
|
25
21
|
writeToDisk?: boolean;
|
|
22
|
+
simulationName?: string;
|
|
23
|
+
verbose?: boolean;
|
|
26
24
|
}
|
|
27
25
|
|
|
28
26
|
interface EventConfig {
|
|
29
27
|
event?: string;
|
|
30
28
|
weight?: number;
|
|
31
29
|
properties?: {
|
|
32
|
-
[key: string]:
|
|
30
|
+
[key: string]: valueValid; // Consider refining based on actual properties used
|
|
33
31
|
};
|
|
34
32
|
isFirstEvent?: boolean;
|
|
35
33
|
}
|
|
36
34
|
|
|
37
35
|
interface GroupProperty {
|
|
38
|
-
[key?: string]:
|
|
36
|
+
[key?: string]: valueValid;
|
|
39
37
|
}
|
|
40
38
|
|
|
41
39
|
interface LookupTable {
|
|
42
|
-
key
|
|
43
|
-
entries
|
|
44
|
-
attributes
|
|
45
|
-
|
|
46
|
-
demand?: string[];
|
|
47
|
-
supply?: string[];
|
|
48
|
-
manufacturer?: () => string;
|
|
49
|
-
price?: number;
|
|
50
|
-
rating?: number;
|
|
51
|
-
reviews?: number;
|
|
40
|
+
key: string;
|
|
41
|
+
entries: number;
|
|
42
|
+
attributes: {
|
|
43
|
+
[key?: string]: valueValid;
|
|
52
44
|
};
|
|
53
45
|
}
|
package/utils.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
const Chance = require('chance');
|
|
2
2
|
const chance = new Chance();
|
|
3
3
|
const readline = require('readline');
|
|
4
|
-
const { comma } = require('ak-tools');
|
|
4
|
+
const { comma, uid } = require('ak-tools');
|
|
5
5
|
const { spawn } = require('child_process');
|
|
6
6
|
const dayjs = require('dayjs');
|
|
7
7
|
const utc = require('dayjs/plugin/utc');
|
|
@@ -111,6 +111,30 @@ function integer(min, max) {
|
|
|
111
111
|
return 0;
|
|
112
112
|
}
|
|
113
113
|
|
|
114
|
+
function makeHashTags() {
|
|
115
|
+
const popularHashtags = [
|
|
116
|
+
'#GalacticAdventures',
|
|
117
|
+
'#EnchantedExplorations',
|
|
118
|
+
'#MagicalMoments',
|
|
119
|
+
'#EpicQuests',
|
|
120
|
+
'#WonderfulWorlds',
|
|
121
|
+
'#FantasyFrenzy',
|
|
122
|
+
'#MysticalMayhem',
|
|
123
|
+
'#MythicalMarvels',
|
|
124
|
+
'#LegendaryLegends',
|
|
125
|
+
'#DreamlandDiaries',
|
|
126
|
+
'#WhimsicalWonders',
|
|
127
|
+
'#FabledFables'
|
|
128
|
+
];
|
|
129
|
+
|
|
130
|
+
const numHashtags = integer(integer(1, 5), integer(5, 10));
|
|
131
|
+
const hashtags = [];
|
|
132
|
+
for (let i = 0; i < numHashtags; i++) {
|
|
133
|
+
hashtags.push(chance.pickone(popularHashtags));
|
|
134
|
+
}
|
|
135
|
+
return hashtags;
|
|
136
|
+
}
|
|
137
|
+
|
|
114
138
|
function makeProducts() {
|
|
115
139
|
let categories = ["Device Accessories", "eBooks", "Automotive", "Baby Products", "Beauty", "Books", "Camera & Photo", "Cell Phones & Accessories", "Collectible Coins", "Consumer Electronics", "Entertainment Collectibles", "Fine Art", "Grocery & Gourmet Food", "Health & Personal Care", "Home & Garden", "Independent Design", "Industrial & Scientific", "Accessories", "Major Appliances", "Music", "Musical Instruments", "Office Products", "Outdoors", "Personal Computers", "Pet Supplies", "Software", "Sports", "Sports Collectibles", "Tools & Home Improvement", "Toys & Games", "Video, DVD & Blu-ray", "Video Games", "Watches"];
|
|
116
140
|
let slugs = ['/sale/', '/featured/', '/home/', '/search/', '/wishlist/', '/'];
|
|
@@ -207,12 +231,18 @@ function person(bornDaysAgo = 30) {
|
|
|
207
231
|
const avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
|
|
208
232
|
const $avatar = avatarPrefix + avPath;
|
|
209
233
|
const $created = date(bornDaysAgo, true, null)();
|
|
234
|
+
const anonymousIds = [];
|
|
235
|
+
const clusterSize = integer(2, 10);
|
|
236
|
+
for (let i = 0; i < clusterSize; i++) {
|
|
237
|
+
anonymousIds.push(uid(42));
|
|
238
|
+
}
|
|
210
239
|
|
|
211
240
|
return {
|
|
212
241
|
$name,
|
|
213
242
|
$email,
|
|
214
243
|
$avatar,
|
|
215
|
-
$created
|
|
244
|
+
$created,
|
|
245
|
+
anonymousIds
|
|
216
246
|
};
|
|
217
247
|
}
|
|
218
248
|
|
|
@@ -255,9 +285,77 @@ function generateEmoji(max = 10, array = false) {
|
|
|
255
285
|
}
|
|
256
286
|
if (array) return arr;
|
|
257
287
|
if (!array) return arr.join(', ');
|
|
288
|
+
return "🤷";
|
|
258
289
|
};
|
|
259
290
|
}
|
|
260
291
|
|
|
292
|
+
function generateName() {
|
|
293
|
+
var adjs = [
|
|
294
|
+
"autumn", "hidden", "bitter", "misty", "silent", "empty", "dry", "dark",
|
|
295
|
+
"summer", "icy", "delicate", "quiet", "white", "cool", "spring", "winter",
|
|
296
|
+
"patient", "twilight", "dawn", "crimson", "wispy", "weathered", "blue",
|
|
297
|
+
"billowing", "broken", "cold", "damp", "falling", "frosty", "green",
|
|
298
|
+
"long", "late", "lingering", "bold", "little", "morning", "muddy", "old",
|
|
299
|
+
"red", "rough", "still", "small", "sparkling", "throbbing", "shy",
|
|
300
|
+
"wandering", "withered", "wild", "black", "young", "holy", "solitary",
|
|
301
|
+
"fragrant", "aged", "snowy", "proud", "floral", "restless", "divine",
|
|
302
|
+
"polished", "ancient", "purple", "lively", "nameless", "gentle", "gleaming", "furious", "luminous", "obscure", "poised", "shimmering", "swirling",
|
|
303
|
+
"sombre", "steamy", "whispering", "jagged", "melodic", "moonlit", "starry", "forgotten",
|
|
304
|
+
"peaceful", "restive", "rustling", "sacred", "ancient", "haunting", "solitary", "mysterious",
|
|
305
|
+
"silver", "dusky", "earthy", "golden", "hallowed", "misty", "roaring", "serene", "vibrant",
|
|
306
|
+
"stalwart", "whimsical", "timid", "tranquil", "vast", "youthful", "zephyr", "raging",
|
|
307
|
+
"sapphire", "turbulent", "whirling", "sleepy", "ethereal", "tender", "unseen", "wistful"
|
|
308
|
+
];
|
|
309
|
+
|
|
310
|
+
var nouns = [
|
|
311
|
+
"waterfall", "river", "breeze", "moon", "rain", "wind", "sea", "morning",
|
|
312
|
+
"snow", "lake", "sunset", "pine", "shadow", "leaf", "dawn", "glitter",
|
|
313
|
+
"forest", "hill", "cloud", "meadow", "sun", "glade", "bird", "brook",
|
|
314
|
+
"butterfly", "bush", "dew", "dust", "field", "fire", "flower", "firefly",
|
|
315
|
+
"feather", "grass", "haze", "mountain", "night", "pond", "darkness",
|
|
316
|
+
"snowflake", "silence", "sound", "sky", "shape", "surf", "thunder",
|
|
317
|
+
"violet", "water", "wildflower", "wave", "water", "resonance", "sun",
|
|
318
|
+
"wood", "dream", "cherry", "tree", "fog", "frost", "voice", "paper",
|
|
319
|
+
"frog", "smoke", "star", "glow", "wave", "riverbed", "cliff", "deluge", "prairie", "creek", "ocean",
|
|
320
|
+
"peak", "valley", "starlight", "quartz", "woodland", "marsh", "earth", "canopy",
|
|
321
|
+
"petal", "stone", "orb", "gale", "bay", "canyon", "watercourse", "vista", "raindrop",
|
|
322
|
+
"boulder", "grove", "plateau", "sand", "mist", "tide", "blossom", "leaf", "flame",
|
|
323
|
+
"shade", "coil", "grotto", "pinnacle", "scallop", "serenity", "abyss", "skyline",
|
|
324
|
+
"drift", "echo", "nebula", "horizon", "crest", "wreath", "twilight", "balm", "glimmer"
|
|
325
|
+
];
|
|
326
|
+
|
|
327
|
+
var verbs = [
|
|
328
|
+
"dancing", "whispering", "flowing", "shimmering", "swirling", "echoing", "sparkling", "glistening",
|
|
329
|
+
"cascading", "drifting", "glowing", "rippling", "quivering", "singing", "twinkling", "radiating",
|
|
330
|
+
"enveloping", "enchanting", "captivating", "embracing", "embracing", "illuminating", "pulsating", "gliding",
|
|
331
|
+
"soaring", "wandering", "meandering", "dazzling", "cuddling", "embracing", "caressing", "twisting",
|
|
332
|
+
"twirling", "tumbling", "surging", "glimmering", "gushing", "splashing", "rolling", "splintering",
|
|
333
|
+
"splintering", "crescendoing", "whirling", "bursting", "shining", "gushing", "emerging", "revealing",
|
|
334
|
+
"emerging", "unfolding", "unveiling", "emerging", "surrounding", "unveiling", "materializing", "revealing"
|
|
335
|
+
];
|
|
336
|
+
|
|
337
|
+
var adverbs = [
|
|
338
|
+
"gracefully", "softly", "smoothly", "gently", "tenderly", "quietly", "serenely", "peacefully",
|
|
339
|
+
"delicately", "effortlessly", "subtly", "tranquilly", "majestically", "silently", "calmly", "harmoniously",
|
|
340
|
+
"elegantly", "luminously", "ethereally", "mysteriously", "sublimely", "radiantly", "dreamily", "ethereally",
|
|
341
|
+
"mesmerizingly", "hypnotically", "mystically", "enigmatically", "spellbindingly", "enchantingly", "fascinatingly",
|
|
342
|
+
"bewitchingly", "captivatingly", "entrancingly", "alluringly", "rapturously", "seductively", "charismatically",
|
|
343
|
+
"seductively", "envelopingly", "ensnaringly", "entrancingly", "intoxicatingly", "irresistibly", "transcendentally",
|
|
344
|
+
"envelopingly", "rapturously", "intimately", "intensely", "tangibly", "vividly", "intensely", "deeply"
|
|
345
|
+
];
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
// ? http://stackoverflow.com/a/17516862/103058
|
|
349
|
+
var adj = adjs[Math.floor(Math.random() * adjs.length)];
|
|
350
|
+
var noun = nouns[Math.floor(Math.random() * nouns.length)];
|
|
351
|
+
var verb = verbs[Math.floor(Math.random() * verbs.length)];
|
|
352
|
+
var adverb = adverbs[Math.floor(Math.random() * adverbs.length)];
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
return adj + '-' + noun + '-' + verb + '-' + adverb
|
|
356
|
+
|
|
357
|
+
}
|
|
358
|
+
|
|
261
359
|
module.exports = {
|
|
262
360
|
weightedRange,
|
|
263
361
|
pick,
|
|
@@ -274,5 +372,7 @@ module.exports = {
|
|
|
274
372
|
applySkew,
|
|
275
373
|
boxMullerRandom,
|
|
276
374
|
generateEmoji,
|
|
277
|
-
getUniqueKeys
|
|
375
|
+
getUniqueKeys,
|
|
376
|
+
makeHashTags,
|
|
377
|
+
generateName,
|
|
278
378
|
};
|