make-mp-data 1.5.0 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.vscode/launch.json +25 -13
- package/components/project.js +155 -0
- package/{src → components}/utils.js +41 -36
- package/index.js +41 -24
- package/package.json +9 -6
- package/schemas/adspend.js +1 -1
- package/schemas/anon.js +1 -1
- package/schemas/big.js +1 -1
- package/schemas/business.js +327 -0
- package/schemas/complex.js +1 -1
- package/schemas/foobar.js +1 -1
- package/schemas/funnels.js +1 -1
- package/schemas/mirror.js +1 -1
- package/schemas/sanity.js +118 -0
- package/schemas/session-replay.js +136 -0
- package/schemas/simple.js +2 -2
- package/scripts/jsdoctest.js +1 -1
- package/scripts/{new.sh → new-dungeon.sh} +30 -7
- package/scripts/new-project.mjs +14 -0
- package/scripts/update-deps.sh +4 -0
- package/tests/int.test.js +11 -10
- package/tests/testCases.mjs +1 -1
- package/tests/testSoup.mjs +4 -3
- package/tests/unit.test.js +15 -14
- package/tsconfig.json +1 -1
- package/types.d.ts +1 -0
- package/scripts/deps.sh +0 -3
- /package/{src → components}/chart.js +0 -0
- /package/{src → components}/cli.js +0 -0
- /package/{src → components}/defaults.js +0 -0
- /package/scripts/{go.sh → run-index.sh} +0 -0
package/.vscode/launch.json
CHANGED
|
@@ -23,18 +23,30 @@
|
|
|
23
23
|
"type": "node-terminal"
|
|
24
24
|
},
|
|
25
25
|
{
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
"
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
26
|
+
"type": "node",
|
|
27
|
+
"request": "launch",
|
|
28
|
+
"name": "go",
|
|
29
|
+
"runtimeExecutable": "nodemon",
|
|
30
|
+
"runtimeArgs": ["--inspect"],
|
|
31
|
+
"program": "${workspaceFolder}/index.js",
|
|
32
|
+
"args": ["--ignore", "./data/*", "${file}"],
|
|
33
|
+
"restart": true,
|
|
34
|
+
"console": "integratedTerminal",
|
|
35
|
+
"internalConsoleOptions": "neverOpen",
|
|
36
|
+
"skipFiles": ["<node_internals>/**"],
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"type": "node",
|
|
40
|
+
"request": "launch",
|
|
41
|
+
"name": "sanity",
|
|
42
|
+
"runtimeExecutable": "nodemon",
|
|
43
|
+
"runtimeArgs": ["--inspect"],
|
|
44
|
+
"program": "${workspaceFolder}/index.js",
|
|
45
|
+
"args": ["--ignore", "./data/*", "./schemas/sanity.js"],
|
|
46
|
+
"restart": true,
|
|
47
|
+
"console": "integratedTerminal",
|
|
48
|
+
"internalConsoleOptions": "neverOpen",
|
|
49
|
+
"skipFiles": ["<node_internals>/**"],
|
|
50
|
+
}
|
|
39
51
|
]
|
|
40
52
|
}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
require('dotenv').config();
|
|
2
|
+
const akTools = require('ak-tools');
|
|
3
|
+
const { rand, makeName } = akTools;
|
|
4
|
+
let { OAUTH_TOKEN = "" } = process.env;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Main function to create a project and add group keys to it.
|
|
8
|
+
*
|
|
9
|
+
* @param {Object} params - Parameters for the function.
|
|
10
|
+
* @param {string} [params.oauth=""] - OAuth token for authentication.
|
|
11
|
+
* @param {string} [params.orgId=""] - Organization ID.
|
|
12
|
+
* @param {Array<Object>} [params.groups=[]] - List of groups to add to the project.
|
|
13
|
+
* @param {string} [params.name=""] - Name of the user.
|
|
14
|
+
* @param {string} [params.email=""] - Email of the user.
|
|
15
|
+
* @param {string} [params.projectName=""] - Name of the project.
|
|
16
|
+
* @returns {Promise<Object>} The created project with additional group keys.
|
|
17
|
+
* @throws Will throw an error if OAUTH_TOKEN is not set.
|
|
18
|
+
* @throws Will throw an error if orgId is not found.
|
|
19
|
+
*/
|
|
20
|
+
async function main(params = {}) {
|
|
21
|
+
let { oauth = "", orgId = "", groups = [], name = "", email = "", projectName } = params;
|
|
22
|
+
if (oauth) OAUTH_TOKEN = oauth;
|
|
23
|
+
if (!OAUTH_TOKEN) throw new Error('No OAUTH_TOKEN in .env');
|
|
24
|
+
if (!orgId) {
|
|
25
|
+
({ orgId, name, email } = await getUser());
|
|
26
|
+
}
|
|
27
|
+
if (!orgId) throw new Error('No orgId found');
|
|
28
|
+
if (!projectName) projectName = makeName();
|
|
29
|
+
const project = await makeProject(orgId);
|
|
30
|
+
project.user = name;
|
|
31
|
+
project.email = email;
|
|
32
|
+
project.groups = groups;
|
|
33
|
+
project.orgId = orgId;
|
|
34
|
+
const groupKeys = [
|
|
35
|
+
// { display_name: 'Account', property_name: 'account_id' },
|
|
36
|
+
];
|
|
37
|
+
groupKeys.push(...groups);
|
|
38
|
+
const addedGroupKeys = await addGroupKeys(groupKeys, project.id);
|
|
39
|
+
project.groupsAdded = addedGroupKeys;
|
|
40
|
+
|
|
41
|
+
return project;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
async function makeProject(orgId, oauthToken = OAUTH_TOKEN) {
|
|
46
|
+
const excludedOrgs = [
|
|
47
|
+
1, // Mixpanel
|
|
48
|
+
328203, // Mixpanel Demo
|
|
49
|
+
1673847, // SE Demo
|
|
50
|
+
1866253 // Demo Projects
|
|
51
|
+
];
|
|
52
|
+
if (!orgId || !oauthToken) throw new Error('Missing orgId or oauthToken');
|
|
53
|
+
const url = `https://mixpanel.com/api/app/organizations/${orgId}/create-project`;
|
|
54
|
+
const projectPayload = {
|
|
55
|
+
"cluster_id": 1,
|
|
56
|
+
"project_name": `GTM Metrics: Test Env ${rand(1000, 9999)}`,
|
|
57
|
+
"timezone_id": 404
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const payload = {
|
|
61
|
+
method: 'POST',
|
|
62
|
+
|
|
63
|
+
headers: {
|
|
64
|
+
Authorization: `Bearer ${oauthToken}`,
|
|
65
|
+
},
|
|
66
|
+
body: JSON.stringify(projectPayload)
|
|
67
|
+
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
const projectsReq = await fetch(url, payload);
|
|
71
|
+
const projectsRes = await projectsReq.json();
|
|
72
|
+
const { api_secret, id, name, token } = projectsRes.results;
|
|
73
|
+
|
|
74
|
+
const data = {
|
|
75
|
+
api_secret,
|
|
76
|
+
id,
|
|
77
|
+
name,
|
|
78
|
+
token,
|
|
79
|
+
url: `https://mixpanel.com/project/${id}/app/settings#project/${id}`
|
|
80
|
+
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
return data;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function getUser(oauthToken = OAUTH_TOKEN) {
|
|
87
|
+
const user = {};
|
|
88
|
+
try {
|
|
89
|
+
if (oauthToken) {
|
|
90
|
+
const info = await fetch(`https://mixpanel.com/api/app/me/?include_workspace_users=false`, { headers: { Authorization: `Bearer ${oauthToken}` } });
|
|
91
|
+
const data = await info.json();
|
|
92
|
+
if (data?.results) {
|
|
93
|
+
const { user_name = "", user_email = "" } = data.results;
|
|
94
|
+
if (user_name) user.name = user_name;
|
|
95
|
+
if (user_email) user.email = user_email;
|
|
96
|
+
const foundOrg = Object.values(data.results.organizations).filter(o => o.name.includes(user_name))?.pop();
|
|
97
|
+
if (foundOrg) {
|
|
98
|
+
user.orgId = foundOrg.id?.toString();
|
|
99
|
+
user.orgName = foundOrg.name;
|
|
100
|
+
}
|
|
101
|
+
if (!foundOrg) {
|
|
102
|
+
// the name is not in the orgs, so we need to find the org in which the user is the owner
|
|
103
|
+
const ignoreProjects = [1673847, 1866253, 328203];
|
|
104
|
+
const possibleOrg = Object.values(data.results.organizations)
|
|
105
|
+
.filter(o => o.role === 'owner')
|
|
106
|
+
.filter(o => !ignoreProjects.includes(o.id))?.pop();
|
|
107
|
+
if (possibleOrg) {
|
|
108
|
+
user.orgId = possibleOrg?.id?.toString();
|
|
109
|
+
user.orgName = possibleOrg.name;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
console.error('get user err', err);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return user;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
async function addGroupKeys(groupKeyDfns = [], projectId, oauthToken = OAUTH_TOKEN) {
|
|
124
|
+
const url = `https://mixpanel.com/api/app/projects/${projectId}/data-groups/`;
|
|
125
|
+
const results = [];
|
|
126
|
+
loopKeys: for (const { display_name, property_name } of groupKeyDfns) {
|
|
127
|
+
const body = {
|
|
128
|
+
display_name,
|
|
129
|
+
property_name
|
|
130
|
+
};
|
|
131
|
+
const payload = {
|
|
132
|
+
method: 'POST',
|
|
133
|
+
headers: {
|
|
134
|
+
Authorization: `Bearer ${oauthToken}`,
|
|
135
|
+
'Content-Type': 'application/json'
|
|
136
|
+
},
|
|
137
|
+
body: JSON.stringify(body)
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
try {
|
|
141
|
+
const res = await fetch(url, payload);
|
|
142
|
+
const data = await res.json();
|
|
143
|
+
results.push(data?.results);
|
|
144
|
+
}
|
|
145
|
+
catch (err) {
|
|
146
|
+
console.error('add group keys err', err);
|
|
147
|
+
continue loopKeys;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
}
|
|
151
|
+
return results;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
module.exports = main;
|
|
@@ -23,6 +23,7 @@ const { domainSuffix, domainPrefix } = require('./defaults');
|
|
|
23
23
|
let globalChance;
|
|
24
24
|
let chanceInitialized = false;
|
|
25
25
|
|
|
26
|
+
const ACTUAL_NOW = dayjs.utc();
|
|
26
27
|
|
|
27
28
|
|
|
28
29
|
/*
|
|
@@ -96,7 +97,8 @@ function pick(items) {
|
|
|
96
97
|
*/
|
|
97
98
|
function date(inTheLast = 30, isPast = true, format = 'YYYY-MM-DD') {
|
|
98
99
|
const chance = getChance();
|
|
99
|
-
const now = global.
|
|
100
|
+
// const now = global.FIXED_NOW ? dayjs.unix(global.FIXED_NOW) : dayjs();
|
|
101
|
+
const now = ACTUAL_NOW;
|
|
100
102
|
if (Math.abs(inTheLast) > 365 * 10) inTheLast = chance.integer({ min: 1, max: 180 });
|
|
101
103
|
return function () {
|
|
102
104
|
const when = chance.integer({ min: 0, max: Math.abs(inTheLast) });
|
|
@@ -149,14 +151,17 @@ function datesBetween(start, end) {
|
|
|
149
151
|
/**
|
|
150
152
|
* returns a random date
|
|
151
153
|
* @param {any} start
|
|
152
|
-
* @param {any} end
|
|
154
|
+
* @param {any} end
|
|
153
155
|
*/
|
|
154
|
-
function day(start, end
|
|
156
|
+
function day(start, end) {
|
|
157
|
+
// if (!end) end = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
158
|
+
if (!start) start = ACTUAL_NOW.subtract(30, 'd').toISOString();
|
|
159
|
+
if (!end) end = ACTUAL_NOW.toISOString();
|
|
155
160
|
const chance = getChance();
|
|
156
161
|
const format = 'YYYY-MM-DD';
|
|
157
162
|
return function (min, max) {
|
|
158
163
|
start = dayjs(start);
|
|
159
|
-
end = dayjs
|
|
164
|
+
end = dayjs(end);
|
|
160
165
|
const diff = end.diff(start, 'day');
|
|
161
166
|
const delta = chance.integer({ min: min, max: diff });
|
|
162
167
|
const day = start.add(delta, 'day');
|
|
@@ -628,13 +633,13 @@ function validateEventConfig(events) {
|
|
|
628
633
|
}
|
|
629
634
|
|
|
630
635
|
function validTime(chosenTime, earliestTime, latestTime) {
|
|
631
|
-
if (!earliestTime) earliestTime = global.
|
|
632
|
-
if (!latestTime) latestTime = global.
|
|
636
|
+
if (!earliestTime) earliestTime = global.FIXED_BEGIN ? global.FIXED_BEGIN : dayjs().subtract(30, 'd').unix(); // 30 days ago
|
|
637
|
+
if (!latestTime) latestTime = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
633
638
|
|
|
634
639
|
if (typeof chosenTime === 'number') {
|
|
635
640
|
if (chosenTime > 0) {
|
|
636
641
|
if (chosenTime > earliestTime) {
|
|
637
|
-
if (chosenTime < latestTime) {
|
|
642
|
+
if (chosenTime < (latestTime)) {
|
|
638
643
|
return true;
|
|
639
644
|
}
|
|
640
645
|
|
|
@@ -808,8 +813,8 @@ let soupHits = 0;
|
|
|
808
813
|
* @param {number} [peaks=5]
|
|
809
814
|
*/
|
|
810
815
|
function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0) {
|
|
811
|
-
if (!earliestTime) earliestTime = global.
|
|
812
|
-
if (!latestTime) latestTime = global.
|
|
816
|
+
if (!earliestTime) earliestTime = global.FIXED_BEGIN ? global.FIXED_BEGIN : dayjs().subtract(30, 'd').unix(); // 30 days ago
|
|
817
|
+
if (!latestTime) latestTime = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
813
818
|
const chance = getChance();
|
|
814
819
|
const totalRange = latestTime - earliestTime;
|
|
815
820
|
const chunkSize = totalRange / peaks;
|
|
@@ -871,7 +876,7 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false
|
|
|
871
876
|
let randomAvatarNumber = integer(1, 99);
|
|
872
877
|
let avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
|
|
873
878
|
let avatar = avatarPrefix + avPath;
|
|
874
|
-
let created = dayjs
|
|
879
|
+
let created = dayjs().subtract(bornDaysAgo, 'day').format('YYYY-MM-DD');
|
|
875
880
|
// const created = date(bornDaysAgo, true)();
|
|
876
881
|
|
|
877
882
|
|
|
@@ -924,32 +929,32 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false
|
|
|
924
929
|
|
|
925
930
|
//UNUSED
|
|
926
931
|
|
|
927
|
-
function fixFunkyTime(earliestTime, latestTime) {
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
}
|
|
932
|
+
// function fixFunkyTime(earliestTime, latestTime) {
|
|
933
|
+
// if (!earliestTime) earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
|
|
934
|
+
// // if (typeof earliestTime !== "number") {
|
|
935
|
+
// // if (parseInt(earliestTime) > 0) earliestTime = parseInt(earliestTime);
|
|
936
|
+
// // if (dayjs(earliestTime).isValid()) earliestTime = dayjs(earliestTime).unix();
|
|
937
|
+
// // }
|
|
938
|
+
// if (typeof earliestTime !== "number") earliestTime = dayjs.unix(earliestTime).unix();
|
|
939
|
+
// if (typeof latestTime !== "number") latestTime = global.NOW;
|
|
940
|
+
// if (typeof latestTime === "number" && latestTime > global.NOW) latestTime = global.NOW;
|
|
941
|
+
// if (earliestTime > latestTime) {
|
|
942
|
+
// const tempEarlyTime = earliestTime;
|
|
943
|
+
// const tempLateTime = latestTime;
|
|
944
|
+
// earliestTime = tempLateTime;
|
|
945
|
+
// latestTime = tempEarlyTime;
|
|
946
|
+
// }
|
|
947
|
+
// if (earliestTime === latestTime) {
|
|
948
|
+
// earliestTime = dayjs.unix(earliestTime)
|
|
949
|
+
// .subtract(integer(1, 14), "day")
|
|
950
|
+
// .subtract(integer(1, 23), "hour")
|
|
951
|
+
// .subtract(integer(1, 59), "minute")
|
|
952
|
+
// .subtract(integer(1, 59), "second")
|
|
953
|
+
// .unix();
|
|
954
|
+
// }
|
|
955
|
+
// return [earliestTime, latestTime];
|
|
956
|
+
|
|
957
|
+
// }
|
|
953
958
|
|
|
954
959
|
|
|
955
960
|
|
package/index.js
CHANGED
|
@@ -16,12 +16,13 @@ ak@mixpanel.com
|
|
|
16
16
|
const dayjs = require("dayjs");
|
|
17
17
|
const utc = require("dayjs/plugin/utc");
|
|
18
18
|
dayjs.extend(utc);
|
|
19
|
-
const
|
|
20
|
-
global.
|
|
19
|
+
const FIXED_NOW = dayjs('2024-02-02').unix();
|
|
20
|
+
global.FIXED_NOW = FIXED_NOW;
|
|
21
21
|
// ^ this creates a FIXED POINT in time; we will shift it later
|
|
22
|
+
let FIXED_BEGIN = dayjs.unix(FIXED_NOW).subtract(90, 'd').unix();
|
|
22
23
|
const actualNow = dayjs();
|
|
23
|
-
const
|
|
24
|
-
const
|
|
24
|
+
const timeShift = actualNow.diff(dayjs.unix(FIXED_NOW), "seconds");
|
|
25
|
+
const daysShift = actualNow.diff(dayjs.unix(FIXED_NOW), "days");
|
|
25
26
|
|
|
26
27
|
// UTILS
|
|
27
28
|
const { existsSync } = require("fs");
|
|
@@ -30,15 +31,15 @@ const os = require("os");
|
|
|
30
31
|
const path = require("path");
|
|
31
32
|
const { comma, bytesHuman, makeName, md5, clone, tracker, uid, timer, ls, rm } = require("ak-tools");
|
|
32
33
|
const jobTimer = timer('job');
|
|
33
|
-
const { generateLineChart } = require('./
|
|
34
|
+
const { generateLineChart } = require('./components/chart.js');
|
|
34
35
|
const { version } = require('./package.json');
|
|
35
36
|
const mp = require("mixpanel-import");
|
|
36
|
-
const u = require("./
|
|
37
|
-
const getCliParams = require("./
|
|
37
|
+
const u = require("./components/utils.js");
|
|
38
|
+
const getCliParams = require("./components/cli.js");
|
|
38
39
|
const metrics = tracker("make-mp-data", "db99eb8f67ae50949a13c27cacf57d41", os.userInfo().username);
|
|
39
40
|
|
|
40
41
|
// DEFAULTS
|
|
41
|
-
const { campaigns, devices, locations } = require('./
|
|
42
|
+
const { campaigns, devices, locations } = require('./components/defaults.js');
|
|
42
43
|
let CAMPAIGNS;
|
|
43
44
|
let DEFAULTS;
|
|
44
45
|
/** @type {Storage} */
|
|
@@ -52,7 +53,7 @@ require('dotenv').config();
|
|
|
52
53
|
let VERBOSE = false;
|
|
53
54
|
let isCLI = false;
|
|
54
55
|
// if we are running in batch mode, we MUST write to disk before we can send to mixpanel
|
|
55
|
-
let isBATCH_MODE = false;
|
|
56
|
+
let isBATCH_MODE = false;
|
|
56
57
|
let BATCH_SIZE = 500_000;
|
|
57
58
|
|
|
58
59
|
//todo: these should be moved into the hookedArrays
|
|
@@ -75,6 +76,7 @@ async function main(config) {
|
|
|
75
76
|
// ^ this is critical; same seed = same data;
|
|
76
77
|
// ^ seed can be passed in as an env var or in the config
|
|
77
78
|
validateDungeonConfig(config);
|
|
79
|
+
global.FIXED_BEGIN = dayjs.unix(FIXED_NOW).subtract(config.numDays, 'd').unix();
|
|
78
80
|
|
|
79
81
|
//GLOBALS
|
|
80
82
|
CONFIG = config;
|
|
@@ -97,6 +99,7 @@ async function main(config) {
|
|
|
97
99
|
let { funnels } = config;
|
|
98
100
|
trackingParams.runId = runId;
|
|
99
101
|
trackingParams.version = version;
|
|
102
|
+
delete trackingParams.funnels;
|
|
100
103
|
|
|
101
104
|
//STORAGE
|
|
102
105
|
const { simulationName, format } = config;
|
|
@@ -316,12 +319,13 @@ async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, s
|
|
|
316
319
|
|
|
317
320
|
|
|
318
321
|
//event time
|
|
319
|
-
if (earliestTime >
|
|
320
|
-
|
|
321
|
-
};
|
|
322
|
+
// if (earliestTime > FIXED_NOW) {
|
|
323
|
+
// earliestTime = dayjs(u.TimeSoup(global.FIXED_BEGIN)).unix();
|
|
324
|
+
// };
|
|
322
325
|
|
|
323
326
|
if (isFirstEvent) eventTemplate.time = dayjs.unix(earliestTime).toISOString();
|
|
324
|
-
if (!isFirstEvent) eventTemplate.time = u.TimeSoup(earliestTime,
|
|
327
|
+
if (!isFirstEvent) eventTemplate.time = u.TimeSoup(earliestTime, FIXED_NOW, peaks, deviation, mean);
|
|
328
|
+
// eventTemplate.time = u.TimeSoup(earliestTime, FIXED_NOW, peaks, deviation, mean);
|
|
325
329
|
|
|
326
330
|
// anonymous and session ids
|
|
327
331
|
if (anonymousIds.length) eventTemplate.device_id = chance.pickone(anonymousIds);
|
|
@@ -396,10 +400,9 @@ async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, s
|
|
|
396
400
|
//make $insert_id
|
|
397
401
|
eventTemplate.insert_id = md5(JSON.stringify(eventTemplate));
|
|
398
402
|
|
|
399
|
-
//
|
|
400
|
-
const
|
|
401
|
-
eventTemplate.time =
|
|
402
|
-
|
|
403
|
+
// move time forward
|
|
404
|
+
const timeShifted = dayjs(eventTemplate.time).add(timeShift, "seconds").toISOString();
|
|
405
|
+
eventTemplate.time = timeShifted;
|
|
403
406
|
|
|
404
407
|
|
|
405
408
|
return eventTemplate;
|
|
@@ -807,14 +810,17 @@ async function userLoop(config, storage, concurrency = 1) {
|
|
|
807
810
|
const { eventData, userProfilesData, scdTableData } = storage;
|
|
808
811
|
const avgEvPerUser = numEvents / numUsers;
|
|
809
812
|
|
|
810
|
-
for (let i =
|
|
813
|
+
for (let i = 0; i < numUsers; i++) {
|
|
811
814
|
await USER_CONN(async () => {
|
|
812
815
|
userCount++;
|
|
813
816
|
if (verbose) u.progress([["users", userCount], ["events", eventCount]]);
|
|
814
817
|
const userId = chance.guid();
|
|
815
818
|
const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds });
|
|
816
819
|
const { distinct_id, created } = user;
|
|
820
|
+
const userIsBornInDataset = chance.bool({ likelihood: 5 });
|
|
817
821
|
let numEventsPreformed = 0;
|
|
822
|
+
if (!userIsBornInDataset) delete user.created;
|
|
823
|
+
const adjustedCreated = userIsBornInDataset ? dayjs(created).subtract(daysShift, 'd') : dayjs.unix(global.FIXED_BEGIN);
|
|
818
824
|
|
|
819
825
|
if (hasLocation) {
|
|
820
826
|
const location = u.choose(DEFAULTS.locationsUsers);
|
|
@@ -822,7 +828,7 @@ async function userLoop(config, storage, concurrency = 1) {
|
|
|
822
828
|
user[key] = location[key];
|
|
823
829
|
}
|
|
824
830
|
}
|
|
825
|
-
|
|
831
|
+
|
|
826
832
|
// Profile creation
|
|
827
833
|
const profile = await makeProfile(userProps, user);
|
|
828
834
|
await userProfilesData.hookPush(profile);
|
|
@@ -849,15 +855,18 @@ async function userLoop(config, storage, concurrency = 1) {
|
|
|
849
855
|
|
|
850
856
|
let userFirstEventTime;
|
|
851
857
|
|
|
852
|
-
// First funnel logic...
|
|
853
858
|
const firstFunnels = funnels.filter((f) => f.isFirstFunnel).reduce(u.weighFunnels, []);
|
|
854
859
|
const usageFunnels = funnels.filter((f) => !f.isFirstFunnel).reduce(u.weighFunnels, []);
|
|
855
|
-
|
|
860
|
+
|
|
861
|
+
const secondsInDay = 86400;
|
|
862
|
+
const noise = () => chance.integer({ min: 0, max: secondsInDay });
|
|
856
863
|
|
|
857
864
|
if (firstFunnels.length && userIsBornInDataset) {
|
|
858
865
|
const firstFunnel = chance.pickone(firstFunnels, user);
|
|
859
|
-
|
|
860
|
-
|
|
866
|
+
|
|
867
|
+
const firstTime = adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
868
|
+
const [data, userConverted] = await makeFunnel(firstFunnel, user, firstTime, profile, userSCD, config);
|
|
869
|
+
userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
|
|
861
870
|
numEventsPreformed += data.length;
|
|
862
871
|
await eventData.hookPush(data);
|
|
863
872
|
if (!userConverted) {
|
|
@@ -865,7 +874,9 @@ async function userLoop(config, storage, concurrency = 1) {
|
|
|
865
874
|
return;
|
|
866
875
|
}
|
|
867
876
|
} else {
|
|
868
|
-
userFirstEventTime = dayjs(created).unix();
|
|
877
|
+
// userFirstEventTime = dayjs(created).unix();
|
|
878
|
+
// userFirstEventTime = global.FIXED_BEGIN;
|
|
879
|
+
userFirstEventTime = adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
869
880
|
}
|
|
870
881
|
|
|
871
882
|
while (numEventsPreformed < numEventsThisUserWillPreform) {
|
|
@@ -1048,6 +1059,7 @@ function validateDungeonConfig(config) {
|
|
|
1048
1059
|
hasAndroidDevices = false,
|
|
1049
1060
|
hasDesktopDevices = false,
|
|
1050
1061
|
hasIOSDevices = false,
|
|
1062
|
+
alsoInferFunnels = false,
|
|
1051
1063
|
name = "",
|
|
1052
1064
|
batchSize = 500_000,
|
|
1053
1065
|
concurrency = 500
|
|
@@ -1076,6 +1088,11 @@ function validateDungeonConfig(config) {
|
|
|
1076
1088
|
funnels = inferFunnels(events);
|
|
1077
1089
|
}
|
|
1078
1090
|
|
|
1091
|
+
if (alsoInferFunnels) {
|
|
1092
|
+
const inferredFunnels = inferFunnels(events);
|
|
1093
|
+
funnels = [...funnels, ...inferredFunnels];
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1079
1096
|
config.concurrency = concurrency;
|
|
1080
1097
|
config.funnels = funnels;
|
|
1081
1098
|
config.batchSize = batchSize;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "make-mp-data",
|
|
3
|
-
"version": "1.5.
|
|
3
|
+
"version": "1.5.01",
|
|
4
4
|
"description": "builds all mixpanel primitives for a given project",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"types": "types.d.ts",
|
|
@@ -11,9 +11,11 @@
|
|
|
11
11
|
"post": "npm publish",
|
|
12
12
|
"test": "NODE_ENV=test jest --runInBand",
|
|
13
13
|
"coverage": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --coverage && open ./tests/coverage/lcov-report/index.html",
|
|
14
|
-
"deps": "
|
|
15
|
-
"new": "
|
|
16
|
-
"
|
|
14
|
+
"update:deps": "./scripts/update-deps.sh",
|
|
15
|
+
"new:dungeon": "./scripts/new-dungeon.sh",
|
|
16
|
+
"new:project": "node ./scripts/new-project.mjs",
|
|
17
|
+
"exp:benchmark": "node --no-warnings --experimental-vm-modules ./tests/benchmark/concurrency.mjs",
|
|
18
|
+
"exp:soup": "node ./tests/testSoup.mjs"
|
|
17
19
|
},
|
|
18
20
|
"repository": {
|
|
19
21
|
"type": "git",
|
|
@@ -41,13 +43,14 @@
|
|
|
41
43
|
},
|
|
42
44
|
"homepage": "https://github.com/ak--47/make-mp-data#readme",
|
|
43
45
|
"dependencies": {
|
|
44
|
-
"ak-
|
|
46
|
+
"ak-fetch": "^1.0.21",
|
|
47
|
+
"ak-tools": "^1.0.64",
|
|
45
48
|
"chance": "^1.1.11",
|
|
46
49
|
"chart.js": "^3.9.1",
|
|
47
50
|
"chartjs-node-canvas": "^4.1.6",
|
|
48
51
|
"dayjs": "^1.11.11",
|
|
49
52
|
"dotenv": "^16.4.5",
|
|
50
|
-
"mixpanel-import": "^2.5.
|
|
53
|
+
"mixpanel-import": "^2.5.554",
|
|
51
54
|
"p-limit": "^3.1.0",
|
|
52
55
|
"yargs": "^17.7.2"
|
|
53
56
|
},
|
package/schemas/adspend.js
CHANGED
|
@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
|
|
|
14
14
|
const utc = require("dayjs/plugin/utc");
|
|
15
15
|
dayjs.extend(utc);
|
|
16
16
|
const { uid, comma } = require('ak-tools');
|
|
17
|
-
const { pickAWinner, weighNumRange, date, integer } = require('../
|
|
17
|
+
const { pickAWinner, weighNumRange, date, integer } = require('../components/utils');
|
|
18
18
|
|
|
19
19
|
/** @type {import('../types').Config} */
|
|
20
20
|
const config = {
|
package/schemas/anon.js
CHANGED
|
@@ -14,7 +14,7 @@ const dayjs = require("dayjs");
|
|
|
14
14
|
const utc = require("dayjs/plugin/utc");
|
|
15
15
|
dayjs.extend(utc);
|
|
16
16
|
const { uid, comma } = require('ak-tools');
|
|
17
|
-
const { pickAWinner, weighNumRange, date, integer } = require('../
|
|
17
|
+
const { pickAWinner, weighNumRange, date, integer } = require('../components/utils');
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
|
package/schemas/big.js
CHANGED
|
@@ -16,7 +16,7 @@ const dayjs = require("dayjs");
|
|
|
16
16
|
const utc = require("dayjs/plugin/utc");
|
|
17
17
|
dayjs.extend(utc);
|
|
18
18
|
const { uid, comma } = require('ak-tools');
|
|
19
|
-
const { pickAWinner, weighNumRange, date, integer } = require('../
|
|
19
|
+
const { pickAWinner, weighNumRange, date, integer } = require('../components/utils');
|
|
20
20
|
|
|
21
21
|
/** @type {import('../types').Config} */
|
|
22
22
|
const config = {
|