make-mp-data 1.0.17 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli.js +6 -3
- package/default.js +26 -4
- package/e2e.test.js +13 -2
- package/index.js +111 -80
- package/package.json +11 -10
- package/scripts/deps.sh +3 -0
- package/scripts/go.sh +2 -0
- package/types.d.ts +14 -22
- package/utils.js +103 -3
package/cli.js
CHANGED
|
@@ -18,10 +18,13 @@ function cliParams() {
|
|
|
18
18
|
.scriptName("make-mp-data")
|
|
19
19
|
.usage(`\nusage:\nnpx $0 [dataModel.js] [options]
|
|
20
20
|
ex:
|
|
21
|
-
npx $0
|
|
22
|
-
npx $0
|
|
21
|
+
npx $0
|
|
22
|
+
npx $0 --token 1234 --u 100 --e 1000 --d 7 --w false
|
|
23
|
+
npx $0 myDataConfig.js
|
|
23
24
|
|
|
24
|
-
DOCS: https://github.com/ak--47/make-mp-data
|
|
25
|
+
DOCS: https://github.com/ak--47/make-mp-data
|
|
26
|
+
DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
|
|
27
|
+
`)
|
|
25
28
|
.command('$0', 'model mixpanel data', () => { })
|
|
26
29
|
.option("token", {
|
|
27
30
|
demandOption: false,
|
package/default.js
CHANGED
|
@@ -1,7 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This is the default configuration file for the data generator
|
|
3
|
+
* notice how the config object is structured, and see it's type definition in ./types.d.ts
|
|
4
|
+
* feel free to modify this file to customize the data you generate
|
|
5
|
+
* see helper functions in utils.js for more ways to generate data
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
|
|
1
9
|
const Chance = require('chance');
|
|
2
10
|
const chance = new Chance();
|
|
3
|
-
const { weightedRange, makeProducts, date, generateEmoji } = require('./utils.js');
|
|
11
|
+
const { weightedRange, makeProducts, date, generateEmoji, makeHashTags } = require('./utils.js');
|
|
4
12
|
|
|
13
|
+
/** @type {import('./types.d.ts').Config} */
|
|
5
14
|
const config = {
|
|
6
15
|
token: "",
|
|
7
16
|
seed: "foo bar baz",
|
|
@@ -40,6 +49,19 @@ const config = {
|
|
|
40
49
|
utm_source: ["$organic", "$organic", "$organic", "$organic", "google", "google", "google", "facebook", "facebook", "twitter", "linkedin"],
|
|
41
50
|
}
|
|
42
51
|
},
|
|
52
|
+
{
|
|
53
|
+
"event": "watch video",
|
|
54
|
+
"weight": 8,
|
|
55
|
+
"properties": {
|
|
56
|
+
category: ["funny", "educational", "inspirational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"],
|
|
57
|
+
hashTags: makeHashTags,
|
|
58
|
+
watchTimeSec: weightedRange(10, 600, 1000, .25),
|
|
59
|
+
quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
|
|
60
|
+
format: ["mp4", "avi", "mov", "mpg"],
|
|
61
|
+
uploader_id: chance.guid.bind(chance)
|
|
62
|
+
|
|
63
|
+
}
|
|
64
|
+
},
|
|
43
65
|
{
|
|
44
66
|
"event": "view item",
|
|
45
67
|
"weight": 8,
|
|
@@ -67,7 +89,7 @@ const config = {
|
|
|
67
89
|
}
|
|
68
90
|
],
|
|
69
91
|
superProps: {
|
|
70
|
-
platform: ["web", "mobile", "web", "mobile", "web", "kiosk"],
|
|
92
|
+
platform: ["web", "mobile", "web", "mobile", "web", "kiosk", "smartTV"],
|
|
71
93
|
emotions: generateEmoji(),
|
|
72
94
|
|
|
73
95
|
},
|
|
@@ -78,7 +100,7 @@ const config = {
|
|
|
78
100
|
userProps: {
|
|
79
101
|
title: chance.profession.bind(chance),
|
|
80
102
|
luckyNumber: weightedRange(42, 420),
|
|
81
|
-
vibe: generateEmoji(),
|
|
103
|
+
vibe: generateEmoji(),
|
|
82
104
|
spiritAnimal: chance.animal.bind(chance)
|
|
83
105
|
},
|
|
84
106
|
|
|
@@ -101,7 +123,7 @@ const config = {
|
|
|
101
123
|
groupProps: {
|
|
102
124
|
company_id: {
|
|
103
125
|
$name: () => { return chance.company(); },
|
|
104
|
-
$email: () => { return `CSM ${chance.pickone(["AK", "Jessica", "Michelle", "Dana", "Brian", "Dave"])}`; },
|
|
126
|
+
$email: () => { return `CSM: ${chance.pickone(["AK", "Jessica", "Michelle", "Dana", "Brian", "Dave"])}`; },
|
|
105
127
|
"# of employees": weightedRange(3, 10000),
|
|
106
128
|
"sector": ["tech", "finance", "healthcare", "education", "government", "non-profit"],
|
|
107
129
|
"segment": ["enterprise", "SMB", "mid-market"],
|
package/e2e.test.js
CHANGED
|
@@ -9,15 +9,16 @@ const { execSync } = require("child_process");
|
|
|
9
9
|
const u = require('ak-tools');
|
|
10
10
|
|
|
11
11
|
const timeout = 60000;
|
|
12
|
+
const testToken = process.env.TEST_TOKEN;
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
describe('e2e', () => {
|
|
15
16
|
|
|
16
17
|
test('works as module', async () => {
|
|
17
18
|
console.log('MODULE TEST');
|
|
18
|
-
const results = await generate({ writeToDisk: false, numEvents:
|
|
19
|
+
const results = await generate({ writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
|
|
19
20
|
const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
|
|
20
|
-
expect(eventData.length).toBeGreaterThan(
|
|
21
|
+
expect(eventData.length).toBeGreaterThan(980);
|
|
21
22
|
expect(groupProfilesData.length).toBe(0);
|
|
22
23
|
expect(lookupTableData.length).toBe(0);
|
|
23
24
|
expect(scdTableData.length).toBeGreaterThan(200);
|
|
@@ -33,6 +34,16 @@ describe('e2e', () => {
|
|
|
33
34
|
expect(csvs.length).toBe(5);
|
|
34
35
|
}, timeout);
|
|
35
36
|
|
|
37
|
+
test('sends data to mixpanel', async () => {
|
|
38
|
+
console.log('NETWORK TEST');
|
|
39
|
+
const results = await generate({ writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it", token: testToken});
|
|
40
|
+
const {events, users, groups } = results.import;
|
|
41
|
+
expect(events.success).toBeGreaterThan(980);
|
|
42
|
+
expect(users.success).toBe(100);
|
|
43
|
+
expect(groups.length).toBe(0);
|
|
44
|
+
|
|
45
|
+
}, timeout);
|
|
46
|
+
|
|
36
47
|
|
|
37
48
|
|
|
38
49
|
});
|
package/index.js
CHANGED
|
@@ -35,11 +35,15 @@ const dayjs = require("dayjs");
|
|
|
35
35
|
const utc = require("dayjs/plugin/utc");
|
|
36
36
|
dayjs.extend(utc);
|
|
37
37
|
const cliParams = require("./cli.js");
|
|
38
|
-
|
|
38
|
+
const { makeName } = require('ak-tools');
|
|
39
|
+
|
|
39
40
|
Array.prototype.pickOne = pick;
|
|
40
41
|
const NOW = dayjs().unix();
|
|
42
|
+
let VERBOSE = false;
|
|
41
43
|
|
|
42
44
|
/** @typedef {import('./types.d.ts').Config} Config */
|
|
45
|
+
/** @typedef {import('./types.d.ts').EventConfig} EventConfig */
|
|
46
|
+
|
|
43
47
|
|
|
44
48
|
const PEAK_DAYS = [
|
|
45
49
|
dayjs().subtract(2, "day").unix(),
|
|
@@ -78,18 +82,10 @@ async function main(config) {
|
|
|
78
82
|
token = null,
|
|
79
83
|
region = "US",
|
|
80
84
|
writeToDisk = false,
|
|
85
|
+
verbose = false,
|
|
81
86
|
} = config;
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
if (require.main === module) {
|
|
85
|
-
if (!token) {
|
|
86
|
-
if (!writeToDisk) {
|
|
87
|
-
writeToDisk = true;
|
|
88
|
-
config.writeToDisk = true;
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
|
|
87
|
+
VERBOSE = verbose;
|
|
88
|
+
config.simulationName = makeName();
|
|
93
89
|
const uuidChance = new Chance(seed);
|
|
94
90
|
|
|
95
91
|
//the function which generates $distinct_id + $created, skewing towards the present
|
|
@@ -115,10 +111,12 @@ async function main(config) {
|
|
|
115
111
|
.reduce((acc, event) => {
|
|
116
112
|
const weight = event.weight || 1;
|
|
117
113
|
for (let i = 0; i < weight; i++) {
|
|
114
|
+
|
|
118
115
|
acc.push(event);
|
|
119
116
|
}
|
|
120
117
|
return acc;
|
|
121
118
|
}, [])
|
|
119
|
+
|
|
122
120
|
.filter((e) => !e.isFirstEvent);
|
|
123
121
|
|
|
124
122
|
const firstEvents = events.filter((e) => e.isFirstEvent);
|
|
@@ -133,18 +131,19 @@ async function main(config) {
|
|
|
133
131
|
for (let i = 1; i < numUsers + 1; i++) {
|
|
134
132
|
progress("users", i);
|
|
135
133
|
const user = uuid();
|
|
136
|
-
const { distinct_id, $created } = user;
|
|
134
|
+
const { distinct_id, $created, anonymousIds } = user;
|
|
137
135
|
userProfilesData.push(makeProfile(userProps, user));
|
|
138
|
-
const mutations = chance.integer({ min: 1, max:
|
|
136
|
+
const mutations = chance.integer({ min: 1, max: 10 });
|
|
139
137
|
scdTableData.push(makeSCD(scdProps, distinct_id, mutations, $created));
|
|
140
138
|
const numEventsThisUser = Math.round(
|
|
141
|
-
chance.normal({ mean: avgEvPerUser, dev: avgEvPerUser /
|
|
139
|
+
chance.normal({ mean: avgEvPerUser, dev: avgEvPerUser / integer(3, 7) })
|
|
142
140
|
);
|
|
143
141
|
|
|
144
142
|
if (firstEvents.length) {
|
|
145
143
|
eventData.push(
|
|
146
144
|
makeEvent(
|
|
147
145
|
distinct_id,
|
|
146
|
+
anonymousIds,
|
|
148
147
|
dayjs($created).unix(),
|
|
149
148
|
firstEvents,
|
|
150
149
|
superProps,
|
|
@@ -159,6 +158,7 @@ async function main(config) {
|
|
|
159
158
|
eventData.push(
|
|
160
159
|
makeEvent(
|
|
161
160
|
distinct_id,
|
|
161
|
+
anonymousIds,
|
|
162
162
|
dayjs($created).unix(),
|
|
163
163
|
weightedEvents,
|
|
164
164
|
superProps,
|
|
@@ -170,7 +170,7 @@ async function main(config) {
|
|
|
170
170
|
//flatten SCD
|
|
171
171
|
scdTableData = scdTableData.flat();
|
|
172
172
|
|
|
173
|
-
|
|
173
|
+
log("\n");
|
|
174
174
|
|
|
175
175
|
// make group profiles
|
|
176
176
|
for (const groupPair of groupKeys) {
|
|
@@ -188,7 +188,7 @@ async function main(config) {
|
|
|
188
188
|
}
|
|
189
189
|
groupProfilesData.push({ key: groupKey, data: groupProfiles });
|
|
190
190
|
}
|
|
191
|
-
|
|
191
|
+
log("\n");
|
|
192
192
|
|
|
193
193
|
// make lookup tables
|
|
194
194
|
for (const lookupTable of lookupTables) {
|
|
@@ -213,7 +213,7 @@ async function main(config) {
|
|
|
213
213
|
[groupFiles, groupProfilesData],
|
|
214
214
|
[lookupFiles, lookupTableData],
|
|
215
215
|
];
|
|
216
|
-
|
|
216
|
+
log("\n");
|
|
217
217
|
|
|
218
218
|
if (!writeToDisk && !token)
|
|
219
219
|
return {
|
|
@@ -224,82 +224,89 @@ async function main(config) {
|
|
|
224
224
|
lookupTableData,
|
|
225
225
|
};
|
|
226
226
|
//write the files
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
for (const
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
227
|
+
if (writeToDisk) {
|
|
228
|
+
if (verbose) log(`writing files... for ${config.simulationName}`);
|
|
229
|
+
for (const pair of pairs) {
|
|
230
|
+
const [paths, data] = pair;
|
|
231
|
+
for (const path of paths) {
|
|
232
|
+
let datasetsToWrite;
|
|
233
|
+
if (data?.[0]?.["key"]) datasetsToWrite = data.map((d) => d.data);
|
|
234
|
+
else datasetsToWrite = [data];
|
|
235
|
+
for (const writeData of datasetsToWrite) {
|
|
236
|
+
if (format === "csv") {
|
|
237
|
+
log(`writing ${path}`);
|
|
238
|
+
const columns = getUniqueKeys(writeData);
|
|
239
|
+
//papa parse needs nested JSON stringified
|
|
240
|
+
writeData.forEach((e) => {
|
|
241
|
+
for (const key in e) {
|
|
242
|
+
if (typeof e[key] === "object") e[key] = JSON.stringify(e[key]);
|
|
243
|
+
}
|
|
244
|
+
});
|
|
245
|
+
const csv = Papa.unparse(writeData, { columns });
|
|
246
|
+
await touch(path, csv);
|
|
247
|
+
log(`\tdone\n`);
|
|
248
|
+
} else {
|
|
249
|
+
await touch(path, data, true);
|
|
250
|
+
}
|
|
248
251
|
}
|
|
249
252
|
}
|
|
250
253
|
}
|
|
251
254
|
}
|
|
252
255
|
|
|
253
256
|
const importResults = { events: {}, users: {}, groups: [] };
|
|
254
|
-
|
|
255
|
-
const creds = { token };
|
|
256
|
-
/** @type {import('mixpanel-import').Options} */
|
|
257
|
-
const importOpts = {
|
|
258
|
-
region,
|
|
259
|
-
fixData: true,
|
|
260
|
-
verbose: false,
|
|
261
|
-
forceStream: true,
|
|
262
|
-
strict: false,
|
|
263
|
-
dryRun: false,
|
|
264
|
-
abridged: false,
|
|
265
|
-
};
|
|
257
|
+
|
|
266
258
|
//send to mixpanel
|
|
267
259
|
if (token) {
|
|
260
|
+
/** @type {import('mixpanel-import').Creds} */
|
|
261
|
+
const creds = { token };
|
|
262
|
+
/** @type {import('mixpanel-import').Options} */
|
|
263
|
+
const commonOpts = {
|
|
264
|
+
|
|
265
|
+
region,
|
|
266
|
+
fixData: true,
|
|
267
|
+
verbose: false,
|
|
268
|
+
forceStream: true,
|
|
269
|
+
strict: false,
|
|
270
|
+
dryRun: false,
|
|
271
|
+
abridged: false,
|
|
272
|
+
};
|
|
273
|
+
|
|
268
274
|
if (eventData) {
|
|
269
|
-
|
|
275
|
+
log(`importing events to mixpanel...`);
|
|
270
276
|
const imported = await mp(creds, eventData, {
|
|
271
277
|
recordType: "event",
|
|
272
278
|
fixData: true,
|
|
273
279
|
fixJson: true,
|
|
274
280
|
strict: false,
|
|
275
|
-
...
|
|
281
|
+
...commonOpts,
|
|
276
282
|
});
|
|
277
|
-
|
|
283
|
+
log(`\tsent ${comma(imported.success)} events\n`);
|
|
278
284
|
importResults.events = imported;
|
|
279
285
|
}
|
|
280
286
|
if (userProfilesData) {
|
|
281
|
-
|
|
287
|
+
log(`importing user profiles to mixpanel...`);
|
|
282
288
|
const imported = await mp(creds, userProfilesData, {
|
|
283
289
|
recordType: "user",
|
|
284
|
-
...
|
|
290
|
+
...commonOpts,
|
|
285
291
|
});
|
|
286
|
-
|
|
292
|
+
log(`\tsent ${comma(imported.success)} user profiles\n`);
|
|
287
293
|
importResults.users = imported;
|
|
288
294
|
}
|
|
289
295
|
if (groupProfilesData) {
|
|
290
296
|
for (const groupProfiles of groupProfilesData) {
|
|
291
297
|
const groupKey = groupProfiles.key;
|
|
292
298
|
const data = groupProfiles.data;
|
|
293
|
-
|
|
299
|
+
log(`importing ${groupKey} profiles to mixpanel...`);
|
|
294
300
|
const imported = await mp({ token, groupKey }, data, {
|
|
295
301
|
recordType: "group",
|
|
296
|
-
...
|
|
302
|
+
...commonOpts,
|
|
297
303
|
});
|
|
298
|
-
|
|
304
|
+
log(`\tsent ${comma(imported.success)} ${groupKey} profiles\n`);
|
|
305
|
+
|
|
299
306
|
importResults.groups.push(imported);
|
|
300
307
|
}
|
|
301
308
|
}
|
|
302
|
-
|
|
309
|
+
log(`\n\n`);
|
|
303
310
|
}
|
|
304
311
|
return {
|
|
305
312
|
import: importResults,
|
|
@@ -344,19 +351,33 @@ function makeSCD(props, distinct_id, mutations, $created) {
|
|
|
344
351
|
return scdEntries;
|
|
345
352
|
}
|
|
346
353
|
|
|
347
|
-
|
|
354
|
+
/**
|
|
355
|
+
* creates a random event
|
|
356
|
+
* @param {string} distinct_id
|
|
357
|
+
* @param {string[]} anonymousIds
|
|
358
|
+
* @param {number} earliestTime
|
|
359
|
+
* @param {Object[]} events
|
|
360
|
+
* @param {Object} superProps
|
|
361
|
+
* @param {Object} groupKeys
|
|
362
|
+
* @param {Boolean} isFirstEvent=false
|
|
363
|
+
*/
|
|
364
|
+
function makeEvent(distinct_id, anonymousIds, earliestTime, events, superProps, groupKeys, isFirstEvent = false) {
|
|
365
|
+
|
|
348
366
|
let chosenEvent = events.pickOne();
|
|
349
367
|
if (typeof chosenEvent === "string")
|
|
350
368
|
chosenEvent = { event: chosenEvent, properties: {} };
|
|
351
369
|
const event = {
|
|
352
370
|
event: chosenEvent.event,
|
|
353
|
-
|
|
371
|
+
$device_id: chance.pickone(anonymousIds), // always have a $device_id
|
|
354
372
|
$source: "AKsTimeSoup",
|
|
355
373
|
};
|
|
356
374
|
|
|
357
375
|
if (isFirstEvent) event.time = dayjs.unix(earliestTime).toISOString();
|
|
358
376
|
if (!isFirstEvent) event.time = AKsTimeSoup(earliestTime, NOW, PEAK_DAYS);
|
|
359
377
|
|
|
378
|
+
//sometimes have a $user_id
|
|
379
|
+
if (!isFirstEvent && chance.bool({ likelihood: 42 })) event.$user_id = distinct_id;
|
|
380
|
+
|
|
360
381
|
const props = { ...chosenEvent.properties, ...superProps };
|
|
361
382
|
|
|
362
383
|
//iterate through custom properties
|
|
@@ -364,6 +385,7 @@ function makeEvent(distinct_id, earliestTime, events, superProps, groupKeys, isF
|
|
|
364
385
|
try {
|
|
365
386
|
event[key] = choose(props[key]);
|
|
366
387
|
} catch (e) {
|
|
388
|
+
console.error(`error with ${key} in ${chosenEvent.event} event`, e);
|
|
367
389
|
debugger;
|
|
368
390
|
}
|
|
369
391
|
}
|
|
@@ -372,6 +394,7 @@ function makeEvent(distinct_id, earliestTime, events, superProps, groupKeys, isF
|
|
|
372
394
|
for (const groupPair of groupKeys) {
|
|
373
395
|
const groupKey = groupPair[0];
|
|
374
396
|
const groupCardinality = groupPair[1];
|
|
397
|
+
|
|
375
398
|
event[groupKey] = weightedRange(1, groupCardinality).pickOne();
|
|
376
399
|
}
|
|
377
400
|
|
|
@@ -381,14 +404,15 @@ function makeEvent(distinct_id, earliestTime, events, superProps, groupKeys, isF
|
|
|
381
404
|
function buildFileNames(config) {
|
|
382
405
|
const { format = "csv", groupKeys = [], lookupTables = [] } = config;
|
|
383
406
|
const extension = format === "csv" ? "csv" : "json";
|
|
384
|
-
const current = dayjs.utc().format("MM-DD-HH");
|
|
407
|
+
// const current = dayjs.utc().format("MM-DD-HH");
|
|
408
|
+
const simName = config.simulationName;
|
|
385
409
|
let writeDir = "./";
|
|
386
410
|
if (config.writeToDisk) writeDir = mkdir("./data");
|
|
387
411
|
|
|
388
412
|
const writePaths = {
|
|
389
|
-
eventFiles: [path.join(writeDir, `events-${
|
|
390
|
-
userFiles: [path.join(writeDir, `users-${
|
|
391
|
-
scdFiles: [path.join(writeDir, `scd-${
|
|
413
|
+
eventFiles: [path.join(writeDir, `events-${simName}.${extension}`)],
|
|
414
|
+
userFiles: [path.join(writeDir, `users-${simName}.${extension}`)],
|
|
415
|
+
scdFiles: [path.join(writeDir, `scd-${simName}.${extension}`)],
|
|
392
416
|
groupFiles: [],
|
|
393
417
|
lookupFiles: [],
|
|
394
418
|
folder: writeDir,
|
|
@@ -397,14 +421,14 @@ function buildFileNames(config) {
|
|
|
397
421
|
for (const groupPair of groupKeys) {
|
|
398
422
|
const groupKey = groupPair[0];
|
|
399
423
|
writePaths.groupFiles.push(
|
|
400
|
-
path.join(writeDir, `group-${groupKey}-${
|
|
424
|
+
path.join(writeDir, `group-${groupKey}-${simName}.${extension}`)
|
|
401
425
|
);
|
|
402
426
|
}
|
|
403
427
|
|
|
404
428
|
for (const lookupTable of lookupTables) {
|
|
405
429
|
const { key } = lookupTable;
|
|
406
430
|
writePaths.lookupFiles.push(
|
|
407
|
-
path.join(writeDir, `lookup-${key}-${
|
|
431
|
+
path.join(writeDir, `lookup-${key}-${simName}.${extension}`)
|
|
408
432
|
);
|
|
409
433
|
}
|
|
410
434
|
|
|
@@ -452,7 +476,7 @@ function AKsTimeSoup(earliestTime, latestTime = NOW, peakDays = PEAK_DAYS) {
|
|
|
452
476
|
|
|
453
477
|
// usually, ensure the event time is within business hours
|
|
454
478
|
if (chance.bool({ likelihood: 42 })) eventTime = Math.min(Math.max(eventTime, businessStart), businessEnd);
|
|
455
|
-
|
|
479
|
+
|
|
456
480
|
return dayjs.unix(eventTime).toISOString();
|
|
457
481
|
}
|
|
458
482
|
|
|
@@ -467,10 +491,10 @@ if (require.main === module) {
|
|
|
467
491
|
//if the user specifics an separate config file
|
|
468
492
|
let config = null;
|
|
469
493
|
if (suppliedConfig) {
|
|
470
|
-
|
|
494
|
+
log(`using ${suppliedConfig} for data\n`);
|
|
471
495
|
config = require(path.resolve(suppliedConfig));
|
|
472
496
|
} else {
|
|
473
|
-
|
|
497
|
+
log(`... using default configuration ...\n`);
|
|
474
498
|
config = require("./default.js");
|
|
475
499
|
}
|
|
476
500
|
|
|
@@ -483,13 +507,15 @@ if (require.main === module) {
|
|
|
483
507
|
if (numEvents) config.numEvents = numEvents;
|
|
484
508
|
if (region) config.region = region;
|
|
485
509
|
if (writeToDisk) config.writeToDisk = writeToDisk;
|
|
510
|
+
if (writeToDisk === 'false') config.writeToDisk = false;
|
|
511
|
+
config.verbose = true;
|
|
486
512
|
|
|
487
513
|
main(config)
|
|
488
514
|
.then((data) => {
|
|
489
|
-
|
|
515
|
+
log(`------------------SUMMARY------------------`);
|
|
490
516
|
const { events, groups, users } = data.import;
|
|
491
517
|
const files = data.files;
|
|
492
|
-
const folder = files
|
|
518
|
+
const folder = files?.pop();
|
|
493
519
|
const groupBytes = groups.reduce((acc, group) => {
|
|
494
520
|
return acc + group.bytes;
|
|
495
521
|
}, 0);
|
|
@@ -504,18 +530,18 @@ if (require.main === module) {
|
|
|
504
530
|
bytes: bytesHuman(bytes || 0),
|
|
505
531
|
};
|
|
506
532
|
if (bytes > 0) console.table(stats);
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
533
|
+
log(`\nfiles written to ${folder}...`);
|
|
534
|
+
log("\t" + files?.flat().join("\n\t"));
|
|
535
|
+
log(`\n------------------SUMMARY------------------\n\n\n`);
|
|
510
536
|
})
|
|
511
537
|
.catch((e) => {
|
|
512
|
-
|
|
538
|
+
log(`------------------ERROR------------------`);
|
|
513
539
|
console.error(e);
|
|
514
|
-
|
|
540
|
+
log(`------------------ERROR------------------`);
|
|
515
541
|
debugger;
|
|
516
542
|
})
|
|
517
543
|
.finally(() => {
|
|
518
|
-
|
|
544
|
+
log("have a wonderful day :)");
|
|
519
545
|
openFinder(path.resolve("./data"));
|
|
520
546
|
});
|
|
521
547
|
} else {
|
|
@@ -535,3 +561,8 @@ if (require.main === module) {
|
|
|
535
561
|
openFinder,
|
|
536
562
|
};
|
|
537
563
|
}
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
function log(...args) {
|
|
567
|
+
if (VERBOSE) console.log(...args);
|
|
568
|
+
}
|
package/package.json
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "make-mp-data",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "builds all mixpanel primitives for a given project",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"types": "types.d.ts",
|
|
7
7
|
"scripts": {
|
|
8
8
|
"start": "node index.js",
|
|
9
9
|
"prune": "rm ./data/*",
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
"go": "sh ./scripts/go.sh",
|
|
11
|
+
"post": "npm publish",
|
|
12
|
+
"test": "jest",
|
|
13
|
+
"deps": "sh ./scripts/deps.sh"
|
|
13
14
|
},
|
|
14
15
|
"repository": {
|
|
15
16
|
"type": "git",
|
|
@@ -25,10 +26,10 @@
|
|
|
25
26
|
"tracking",
|
|
26
27
|
"server",
|
|
27
28
|
"CLI",
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
"datamart",
|
|
30
|
+
"scd 2",
|
|
31
|
+
"dummy data",
|
|
32
|
+
"fake data"
|
|
32
33
|
],
|
|
33
34
|
"author": "ak@mixpanel.com",
|
|
34
35
|
"license": "ISC",
|
|
@@ -37,10 +38,10 @@
|
|
|
37
38
|
},
|
|
38
39
|
"homepage": "https://github.com/ak--47/make-mp-data#readme",
|
|
39
40
|
"dependencies": {
|
|
40
|
-
"ak-tools": "^1.0.
|
|
41
|
+
"ak-tools": "^1.0.52",
|
|
41
42
|
"chance": "^1.1.7",
|
|
42
43
|
"dayjs": "^1.11.10",
|
|
43
|
-
"mixpanel-import": "^2.5.
|
|
44
|
+
"mixpanel-import": "^2.5.5",
|
|
44
45
|
"yargs": "^17.7.2"
|
|
45
46
|
}
|
|
46
47
|
}
|
package/scripts/deps.sh
ADDED
package/scripts/go.sh
ADDED
package/types.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// types.d.ts
|
|
2
|
-
|
|
2
|
+
|
|
3
|
+
type primitives = string | number | boolean | Date | Object;
|
|
4
|
+
type valueValid = primitives | primitives[] | (() => primitives | primitives[]);
|
|
3
5
|
|
|
4
6
|
export interface Config {
|
|
5
7
|
token?: string;
|
|
@@ -10,44 +12,34 @@ export interface Config {
|
|
|
10
12
|
format?: "csv" | "json";
|
|
11
13
|
region?: string;
|
|
12
14
|
events?: EventConfig[];
|
|
13
|
-
superProps?: Record<string,
|
|
14
|
-
userProps?: Record<string,
|
|
15
|
-
scdProps?:
|
|
16
|
-
plan?: string[];
|
|
17
|
-
MRR?: number;
|
|
18
|
-
NPS?: number;
|
|
19
|
-
marketingOptIn?: boolean[];
|
|
20
|
-
dateOfRenewal?: Date;
|
|
21
|
-
};
|
|
15
|
+
superProps?: Record<string, valueValid>;
|
|
16
|
+
userProps?: Record<string, valueValid>;
|
|
17
|
+
scdProps?: Record<string, valueValid>;
|
|
22
18
|
groupKeys?: [string, number][];
|
|
23
19
|
groupProps?: Record<string, GroupProperty>; // Adjust according to usage
|
|
24
20
|
lookupTables?: LookupTable[];
|
|
25
21
|
writeToDisk?: boolean;
|
|
22
|
+
simulationName?: string;
|
|
23
|
+
verbose?: boolean;
|
|
26
24
|
}
|
|
27
25
|
|
|
28
26
|
interface EventConfig {
|
|
29
27
|
event?: string;
|
|
30
28
|
weight?: number;
|
|
31
29
|
properties?: {
|
|
32
|
-
[key: string]:
|
|
30
|
+
[key: string]: valueValid; // Consider refining based on actual properties used
|
|
33
31
|
};
|
|
34
32
|
isFirstEvent?: boolean;
|
|
35
33
|
}
|
|
36
34
|
|
|
37
35
|
interface GroupProperty {
|
|
38
|
-
[key?: string]:
|
|
36
|
+
[key?: string]: valueValid;
|
|
39
37
|
}
|
|
40
38
|
|
|
41
39
|
interface LookupTable {
|
|
42
|
-
key
|
|
43
|
-
entries
|
|
44
|
-
attributes
|
|
45
|
-
|
|
46
|
-
demand?: string[];
|
|
47
|
-
supply?: string[];
|
|
48
|
-
manufacturer?: () => string;
|
|
49
|
-
price?: number;
|
|
50
|
-
rating?: number;
|
|
51
|
-
reviews?: number;
|
|
40
|
+
key: string;
|
|
41
|
+
entries: number;
|
|
42
|
+
attributes: {
|
|
43
|
+
[key?: string]: valueValid;
|
|
52
44
|
};
|
|
53
45
|
}
|
package/utils.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
const Chance = require('chance');
|
|
2
2
|
const chance = new Chance();
|
|
3
3
|
const readline = require('readline');
|
|
4
|
-
const { comma } = require('ak-tools');
|
|
4
|
+
const { comma, uid } = require('ak-tools');
|
|
5
5
|
const { spawn } = require('child_process');
|
|
6
6
|
const dayjs = require('dayjs');
|
|
7
7
|
const utc = require('dayjs/plugin/utc');
|
|
@@ -111,6 +111,30 @@ function integer(min, max) {
|
|
|
111
111
|
return 0;
|
|
112
112
|
}
|
|
113
113
|
|
|
114
|
+
function makeHashTags() {
|
|
115
|
+
const popularHashtags = [
|
|
116
|
+
'#GalacticAdventures',
|
|
117
|
+
'#EnchantedExplorations',
|
|
118
|
+
'#MagicalMoments',
|
|
119
|
+
'#EpicQuests',
|
|
120
|
+
'#WonderfulWorlds',
|
|
121
|
+
'#FantasyFrenzy',
|
|
122
|
+
'#MysticalMayhem',
|
|
123
|
+
'#MythicalMarvels',
|
|
124
|
+
'#LegendaryLegends',
|
|
125
|
+
'#DreamlandDiaries',
|
|
126
|
+
'#WhimsicalWonders',
|
|
127
|
+
'#FabledFables'
|
|
128
|
+
];
|
|
129
|
+
|
|
130
|
+
const numHashtags = integer(integer(1, 5), integer(5, 10));
|
|
131
|
+
const hashtags = [];
|
|
132
|
+
for (let i = 0; i < numHashtags; i++) {
|
|
133
|
+
hashtags.push(chance.pickone(popularHashtags));
|
|
134
|
+
}
|
|
135
|
+
return hashtags;
|
|
136
|
+
}
|
|
137
|
+
|
|
114
138
|
function makeProducts() {
|
|
115
139
|
let categories = ["Device Accessories", "eBooks", "Automotive", "Baby Products", "Beauty", "Books", "Camera & Photo", "Cell Phones & Accessories", "Collectible Coins", "Consumer Electronics", "Entertainment Collectibles", "Fine Art", "Grocery & Gourmet Food", "Health & Personal Care", "Home & Garden", "Independent Design", "Industrial & Scientific", "Accessories", "Major Appliances", "Music", "Musical Instruments", "Office Products", "Outdoors", "Personal Computers", "Pet Supplies", "Software", "Sports", "Sports Collectibles", "Tools & Home Improvement", "Toys & Games", "Video, DVD & Blu-ray", "Video Games", "Watches"];
|
|
116
140
|
let slugs = ['/sale/', '/featured/', '/home/', '/search/', '/wishlist/', '/'];
|
|
@@ -207,12 +231,18 @@ function person(bornDaysAgo = 30) {
|
|
|
207
231
|
const avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
|
|
208
232
|
const $avatar = avatarPrefix + avPath;
|
|
209
233
|
const $created = date(bornDaysAgo, true, null)();
|
|
234
|
+
const anonymousIds = [];
|
|
235
|
+
const clusterSize = integer(2, 10);
|
|
236
|
+
for (let i = 0; i < clusterSize; i++) {
|
|
237
|
+
anonymousIds.push(uid(42));
|
|
238
|
+
}
|
|
210
239
|
|
|
211
240
|
return {
|
|
212
241
|
$name,
|
|
213
242
|
$email,
|
|
214
243
|
$avatar,
|
|
215
|
-
$created
|
|
244
|
+
$created,
|
|
245
|
+
anonymousIds
|
|
216
246
|
};
|
|
217
247
|
}
|
|
218
248
|
|
|
@@ -255,9 +285,77 @@ function generateEmoji(max = 10, array = false) {
|
|
|
255
285
|
}
|
|
256
286
|
if (array) return arr;
|
|
257
287
|
if (!array) return arr.join(', ');
|
|
288
|
+
return "🤷";
|
|
258
289
|
};
|
|
259
290
|
}
|
|
260
291
|
|
|
292
|
+
function generateName() {
|
|
293
|
+
var adjs = [
|
|
294
|
+
"autumn", "hidden", "bitter", "misty", "silent", "empty", "dry", "dark",
|
|
295
|
+
"summer", "icy", "delicate", "quiet", "white", "cool", "spring", "winter",
|
|
296
|
+
"patient", "twilight", "dawn", "crimson", "wispy", "weathered", "blue",
|
|
297
|
+
"billowing", "broken", "cold", "damp", "falling", "frosty", "green",
|
|
298
|
+
"long", "late", "lingering", "bold", "little", "morning", "muddy", "old",
|
|
299
|
+
"red", "rough", "still", "small", "sparkling", "throbbing", "shy",
|
|
300
|
+
"wandering", "withered", "wild", "black", "young", "holy", "solitary",
|
|
301
|
+
"fragrant", "aged", "snowy", "proud", "floral", "restless", "divine",
|
|
302
|
+
"polished", "ancient", "purple", "lively", "nameless", "gentle", "gleaming", "furious", "luminous", "obscure", "poised", "shimmering", "swirling",
|
|
303
|
+
"sombre", "steamy", "whispering", "jagged", "melodic", "moonlit", "starry", "forgotten",
|
|
304
|
+
"peaceful", "restive", "rustling", "sacred", "ancient", "haunting", "solitary", "mysterious",
|
|
305
|
+
"silver", "dusky", "earthy", "golden", "hallowed", "misty", "roaring", "serene", "vibrant",
|
|
306
|
+
"stalwart", "whimsical", "timid", "tranquil", "vast", "youthful", "zephyr", "raging",
|
|
307
|
+
"sapphire", "turbulent", "whirling", "sleepy", "ethereal", "tender", "unseen", "wistful"
|
|
308
|
+
];
|
|
309
|
+
|
|
310
|
+
var nouns = [
|
|
311
|
+
"waterfall", "river", "breeze", "moon", "rain", "wind", "sea", "morning",
|
|
312
|
+
"snow", "lake", "sunset", "pine", "shadow", "leaf", "dawn", "glitter",
|
|
313
|
+
"forest", "hill", "cloud", "meadow", "sun", "glade", "bird", "brook",
|
|
314
|
+
"butterfly", "bush", "dew", "dust", "field", "fire", "flower", "firefly",
|
|
315
|
+
"feather", "grass", "haze", "mountain", "night", "pond", "darkness",
|
|
316
|
+
"snowflake", "silence", "sound", "sky", "shape", "surf", "thunder",
|
|
317
|
+
"violet", "water", "wildflower", "wave", "water", "resonance", "sun",
|
|
318
|
+
"wood", "dream", "cherry", "tree", "fog", "frost", "voice", "paper",
|
|
319
|
+
"frog", "smoke", "star", "glow", "wave", "riverbed", "cliff", "deluge", "prairie", "creek", "ocean",
|
|
320
|
+
"peak", "valley", "starlight", "quartz", "woodland", "marsh", "earth", "canopy",
|
|
321
|
+
"petal", "stone", "orb", "gale", "bay", "canyon", "watercourse", "vista", "raindrop",
|
|
322
|
+
"boulder", "grove", "plateau", "sand", "mist", "tide", "blossom", "leaf", "flame",
|
|
323
|
+
"shade", "coil", "grotto", "pinnacle", "scallop", "serenity", "abyss", "skyline",
|
|
324
|
+
"drift", "echo", "nebula", "horizon", "crest", "wreath", "twilight", "balm", "glimmer"
|
|
325
|
+
];
|
|
326
|
+
|
|
327
|
+
var verbs = [
|
|
328
|
+
"dancing", "whispering", "flowing", "shimmering", "swirling", "echoing", "sparkling", "glistening",
|
|
329
|
+
"cascading", "drifting", "glowing", "rippling", "quivering", "singing", "twinkling", "radiating",
|
|
330
|
+
"enveloping", "enchanting", "captivating", "embracing", "embracing", "illuminating", "pulsating", "gliding",
|
|
331
|
+
"soaring", "wandering", "meandering", "dazzling", "cuddling", "embracing", "caressing", "twisting",
|
|
332
|
+
"twirling", "tumbling", "surging", "glimmering", "gushing", "splashing", "rolling", "splintering",
|
|
333
|
+
"splintering", "crescendoing", "whirling", "bursting", "shining", "gushing", "emerging", "revealing",
|
|
334
|
+
"emerging", "unfolding", "unveiling", "emerging", "surrounding", "unveiling", "materializing", "revealing"
|
|
335
|
+
];
|
|
336
|
+
|
|
337
|
+
var adverbs = [
|
|
338
|
+
"gracefully", "softly", "smoothly", "gently", "tenderly", "quietly", "serenely", "peacefully",
|
|
339
|
+
"delicately", "effortlessly", "subtly", "tranquilly", "majestically", "silently", "calmly", "harmoniously",
|
|
340
|
+
"elegantly", "luminously", "ethereally", "mysteriously", "sublimely", "radiantly", "dreamily", "ethereally",
|
|
341
|
+
"mesmerizingly", "hypnotically", "mystically", "enigmatically", "spellbindingly", "enchantingly", "fascinatingly",
|
|
342
|
+
"bewitchingly", "captivatingly", "entrancingly", "alluringly", "rapturously", "seductively", "charismatically",
|
|
343
|
+
"seductively", "envelopingly", "ensnaringly", "entrancingly", "intoxicatingly", "irresistibly", "transcendentally",
|
|
344
|
+
"envelopingly", "rapturously", "intimately", "intensely", "tangibly", "vividly", "intensely", "deeply"
|
|
345
|
+
];
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
// ? http://stackoverflow.com/a/17516862/103058
|
|
349
|
+
var adj = adjs[Math.floor(Math.random() * adjs.length)];
|
|
350
|
+
var noun = nouns[Math.floor(Math.random() * nouns.length)];
|
|
351
|
+
var verb = verbs[Math.floor(Math.random() * verbs.length)];
|
|
352
|
+
var adverb = adverbs[Math.floor(Math.random() * adverbs.length)];
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
return adj + '-' + noun + '-' + verb + '-' + adverb
|
|
356
|
+
|
|
357
|
+
}
|
|
358
|
+
|
|
261
359
|
module.exports = {
|
|
262
360
|
weightedRange,
|
|
263
361
|
pick,
|
|
@@ -274,5 +372,7 @@ module.exports = {
|
|
|
274
372
|
applySkew,
|
|
275
373
|
boxMullerRandom,
|
|
276
374
|
generateEmoji,
|
|
277
|
-
getUniqueKeys
|
|
375
|
+
getUniqueKeys,
|
|
376
|
+
makeHashTags,
|
|
377
|
+
generateName,
|
|
278
378
|
};
|