make-mp-data 1.5.56 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +20 -0
- package/.gcloudignore +2 -1
- package/.vscode/launch.json +6 -17
- package/.vscode/settings.json +31 -2
- package/dungeons/media.js +371 -0
- package/index.js +353 -1766
- package/{components → lib/cli}/cli.js +21 -6
- package/lib/cloud-function.js +20 -0
- package/lib/core/config-validator.js +248 -0
- package/lib/core/context.js +180 -0
- package/lib/core/storage.js +268 -0
- package/{components → lib/data}/defaults.js +17 -14
- package/lib/generators/adspend.js +133 -0
- package/lib/generators/events.js +242 -0
- package/lib/generators/funnels.js +330 -0
- package/lib/generators/mirror.js +168 -0
- package/lib/generators/profiles.js +93 -0
- package/lib/generators/scd.js +102 -0
- package/lib/orchestrators/mixpanel-sender.js +222 -0
- package/lib/orchestrators/user-loop.js +194 -0
- package/lib/orchestrators/worker-manager.js +200 -0
- package/{components → lib/utils}/ai.js +8 -36
- package/{components → lib/utils}/chart.js +9 -9
- package/{components → lib/utils}/project.js +4 -4
- package/{components → lib/utils}/utils.js +35 -23
- package/package.json +15 -15
- package/scripts/dana.mjs +137 -0
- package/scripts/new-dungeon.sh +7 -6
- package/scripts/update-deps.sh +2 -1
- package/tests/cli.test.js +28 -25
- package/tests/e2e.test.js +38 -36
- package/tests/int.test.js +151 -56
- package/tests/testSoup.mjs +1 -1
- package/tests/unit.test.js +15 -14
- package/tsconfig.json +1 -1
- package/types.d.ts +68 -11
- package/vitest.config.js +47 -0
- package/log.json +0 -1678
- package/tests/jest.config.js +0 -47
- /package/{components → lib/utils}/prompt.txt +0 -0
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* User Loop Orchestrator module
|
|
3
|
+
* Manages user generation and event creation workflow
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/** @typedef {import('../../types').Context} Context */
|
|
7
|
+
|
|
8
|
+
import dayjs from "dayjs";
|
|
9
|
+
import pLimit from 'p-limit';
|
|
10
|
+
import * as u from "../utils/utils.js";
|
|
11
|
+
import * as t from 'ak-tools';
|
|
12
|
+
import { makeEvent } from "../generators/events.js";
|
|
13
|
+
import { makeFunnel } from "../generators/funnels.js";
|
|
14
|
+
import { makeUserProfile } from "../generators/profiles.js";
|
|
15
|
+
import { makeSCD } from "../generators/scd.js";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Main user generation loop that creates users, their profiles, events, and SCDs
|
|
19
|
+
* @param {Context} context - Context object containing config, defaults, storage, etc.
|
|
20
|
+
* @returns {Promise<void>}
|
|
21
|
+
*/
|
|
22
|
+
export async function userLoop(context) {
|
|
23
|
+
const { config, storage, defaults } = context;
|
|
24
|
+
const chance = u.getChance();
|
|
25
|
+
const concurrency = config?.concurrency || 1;
|
|
26
|
+
const USER_CONN = pLimit(concurrency);
|
|
27
|
+
|
|
28
|
+
const {
|
|
29
|
+
verbose,
|
|
30
|
+
numUsers,
|
|
31
|
+
numEvents,
|
|
32
|
+
isAnonymous,
|
|
33
|
+
hasAvatar,
|
|
34
|
+
hasAnonIds,
|
|
35
|
+
hasSessionIds,
|
|
36
|
+
hasLocation,
|
|
37
|
+
funnels,
|
|
38
|
+
userProps,
|
|
39
|
+
scdProps,
|
|
40
|
+
numDays,
|
|
41
|
+
percentUsersBornInDataset = 5,
|
|
42
|
+
} = config;
|
|
43
|
+
|
|
44
|
+
const { eventData, userProfilesData, scdTableData } = storage;
|
|
45
|
+
const avgEvPerUser = numEvents / numUsers;
|
|
46
|
+
const startTime = Date.now();
|
|
47
|
+
|
|
48
|
+
for (let i = 0; i < numUsers; i++) {
|
|
49
|
+
await USER_CONN(async () => {
|
|
50
|
+
context.incrementUserCount();
|
|
51
|
+
const eps = Math.floor(context.getEventCount() / ((Date.now() - startTime) / 1000));
|
|
52
|
+
|
|
53
|
+
if (verbose) {
|
|
54
|
+
u.progress([
|
|
55
|
+
["users", context.getUserCount()],
|
|
56
|
+
["events", context.getEventCount()],
|
|
57
|
+
["eps", eps]
|
|
58
|
+
]);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const userId = chance.guid();
|
|
62
|
+
const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds });
|
|
63
|
+
const { distinct_id, created } = user;
|
|
64
|
+
const userIsBornInDataset = chance.bool({ likelihood: percentUsersBornInDataset });
|
|
65
|
+
let numEventsPreformed = 0;
|
|
66
|
+
|
|
67
|
+
if (!userIsBornInDataset) delete user.created;
|
|
68
|
+
|
|
69
|
+
// Calculate time adjustments
|
|
70
|
+
const daysShift = context.getDaysShift();
|
|
71
|
+
const adjustedCreated = userIsBornInDataset
|
|
72
|
+
? dayjs(created).subtract(daysShift, 'd')
|
|
73
|
+
: dayjs.unix(global.FIXED_BEGIN);
|
|
74
|
+
|
|
75
|
+
if (hasLocation) {
|
|
76
|
+
const location = u.shuffleArray(u.choose(defaults.locationsUsers)).pop();
|
|
77
|
+
for (const key in location) {
|
|
78
|
+
user[key] = location[key];
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Profile creation
|
|
83
|
+
const profile = await makeUserProfile(context, userProps, user);
|
|
84
|
+
|
|
85
|
+
// SCD creation
|
|
86
|
+
const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user' || !scd.type);
|
|
87
|
+
const scdTableKeys = Object.keys(scdUserTables);
|
|
88
|
+
|
|
89
|
+
const userSCD = {};
|
|
90
|
+
for (const [index, key] of scdTableKeys.entries()) {
|
|
91
|
+
const { max = 100 } = scdProps[key];
|
|
92
|
+
const mutations = chance.integer({ min: 1, max });
|
|
93
|
+
const changes = await makeSCD(context, scdProps[key], key, distinct_id, mutations, created);
|
|
94
|
+
userSCD[key] = changes;
|
|
95
|
+
|
|
96
|
+
await config.hook(changes, "scd-pre", {
|
|
97
|
+
profile,
|
|
98
|
+
type: 'user',
|
|
99
|
+
scd: { [key]: scdProps[key] },
|
|
100
|
+
config,
|
|
101
|
+
allSCDs: userSCD
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
let numEventsThisUserWillPreform = Math.floor(chance.normal({
|
|
106
|
+
mean: avgEvPerUser,
|
|
107
|
+
dev: avgEvPerUser / u.integer(u.integer(2, 5), u.integer(2, 7))
|
|
108
|
+
}) * 0.714159265359);
|
|
109
|
+
|
|
110
|
+
// Power users and low-activity users logic
|
|
111
|
+
chance.bool({ likelihood: 20 }) ? numEventsThisUserWillPreform *= 5 : null;
|
|
112
|
+
chance.bool({ likelihood: 15 }) ? numEventsThisUserWillPreform *= 0.333 : null;
|
|
113
|
+
numEventsThisUserWillPreform = Math.round(numEventsThisUserWillPreform);
|
|
114
|
+
|
|
115
|
+
let userFirstEventTime;
|
|
116
|
+
|
|
117
|
+
const firstFunnels = funnels.filter((f) => f.isFirstFunnel).reduce(u.weighFunnels, []);
|
|
118
|
+
const usageFunnels = funnels.filter((f) => !f.isFirstFunnel).reduce(u.weighFunnels, []);
|
|
119
|
+
|
|
120
|
+
const secondsInDay = 86400;
|
|
121
|
+
const noise = () => chance.integer({ min: 0, max: secondsInDay });
|
|
122
|
+
let usersEvents = [];
|
|
123
|
+
|
|
124
|
+
if (firstFunnels.length && userIsBornInDataset) {
|
|
125
|
+
const firstFunnel = chance.pickone(firstFunnels, user);
|
|
126
|
+
const firstTime = adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
127
|
+
const [data, userConverted] = await makeFunnel(context, firstFunnel, user, firstTime, profile, userSCD);
|
|
128
|
+
|
|
129
|
+
const timeShift = context.getTimeShift();
|
|
130
|
+
userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
|
|
131
|
+
numEventsPreformed += data.length;
|
|
132
|
+
usersEvents.push(...data);
|
|
133
|
+
|
|
134
|
+
if (!userConverted) {
|
|
135
|
+
if (verbose) {
|
|
136
|
+
u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
|
|
137
|
+
}
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
} else {
|
|
141
|
+
userFirstEventTime = adjustedCreated.subtract(noise(), 'seconds').unix();
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
while (numEventsPreformed < numEventsThisUserWillPreform) {
|
|
145
|
+
if (usageFunnels.length) {
|
|
146
|
+
const currentFunnel = chance.pickone(usageFunnels);
|
|
147
|
+
const [data, userConverted] = await makeFunnel(context, currentFunnel, user, userFirstEventTime, profile, userSCD);
|
|
148
|
+
numEventsPreformed += data.length;
|
|
149
|
+
usersEvents.push(...data);
|
|
150
|
+
} else {
|
|
151
|
+
const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(config.events), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true);
|
|
152
|
+
numEventsPreformed++;
|
|
153
|
+
usersEvents.push(data);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Hook for processing all user events
|
|
158
|
+
if (config.hook) {
|
|
159
|
+
const newEvents = await config.hook(usersEvents, "everything", {
|
|
160
|
+
profile,
|
|
161
|
+
scd: userSCD,
|
|
162
|
+
config,
|
|
163
|
+
userIsBornInDataset
|
|
164
|
+
});
|
|
165
|
+
if (Array.isArray(newEvents)) usersEvents = newEvents;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// Store all user data
|
|
169
|
+
await userProfilesData.hookPush(profile);
|
|
170
|
+
|
|
171
|
+
if (Object.keys(userSCD).length) {
|
|
172
|
+
for (const [key, changesArray] of Object.entries(userSCD)) {
|
|
173
|
+
for (const changes of changesArray) {
|
|
174
|
+
try {
|
|
175
|
+
const target = scdTableData.filter(arr => arr.scdKey === key).pop();
|
|
176
|
+
await target.hookPush(changes, { profile, type: 'user' });
|
|
177
|
+
}
|
|
178
|
+
catch (e) {
|
|
179
|
+
// This is probably a test
|
|
180
|
+
const target = scdTableData[0];
|
|
181
|
+
await target.hookPush(changes, { profile, type: 'user' });
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
await eventData.hookPush(usersEvents, { profile });
|
|
188
|
+
|
|
189
|
+
if (verbose) {
|
|
190
|
+
u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
|
|
191
|
+
}
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
}
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cloud Worker Manager module
|
|
3
|
+
* Handles distributed processing across multiple cloud function workers
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import pLimit from 'p-limit';
|
|
7
|
+
import { GoogleAuth } from 'google-auth-library';
|
|
8
|
+
import { timer, uid, sLog } from 'ak-tools';
|
|
9
|
+
|
|
10
|
+
const CONCURRENCY = 1_000;
|
|
11
|
+
let RUNTIME_URL = "https://dm4-lmozz6xkha-uc.a.run.app";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Spawn multiple cloud function workers to process data generation in parallel
|
|
15
|
+
* @param {number} numberWorkers - Number of worker instances to spawn
|
|
16
|
+
* @param {string} payload - Script payload to execute on each worker
|
|
17
|
+
* @param {Object} params - Parameters for the job execution
|
|
18
|
+
* @returns {Promise<Object>} Results summary with success/failure counts
|
|
19
|
+
*/
|
|
20
|
+
export async function spawnFileWorkers(numberWorkers, payload, params) {
|
|
21
|
+
const auth = new GoogleAuth();
|
|
22
|
+
let client;
|
|
23
|
+
|
|
24
|
+
if (RUNTIME_URL.includes('localhost')) {
|
|
25
|
+
client = await auth.getClient();
|
|
26
|
+
} else {
|
|
27
|
+
client = await auth.getIdTokenClient(RUNTIME_URL);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const limit = pLimit(CONCURRENCY);
|
|
31
|
+
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
32
|
+
|
|
33
|
+
const requestPromises = Array.from({ length: numberWorkers }, async (_, index) => {
|
|
34
|
+
index = index + 1;
|
|
35
|
+
await delay(index * 108); // Stagger requests to avoid thundering herd
|
|
36
|
+
return limit(() => buildRequest(client, payload, index, params, numberWorkers));
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
const complete = await Promise.allSettled(requestPromises);
|
|
40
|
+
const results = {
|
|
41
|
+
jobs_success: complete.filter((p) => p.status === "fulfilled").length,
|
|
42
|
+
jobs_fail: complete.filter((p) => p.status === "rejected").length
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
return results;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Build and execute a single worker request
|
|
50
|
+
* @param {Object} client - Authenticated Google Cloud client
|
|
51
|
+
* @param {string} payload - Script payload to send
|
|
52
|
+
* @param {number} index - Worker index number
|
|
53
|
+
* @param {Object} params - Job parameters
|
|
54
|
+
* @param {number} total - Total number of workers
|
|
55
|
+
* @returns {Promise<Object>} Worker response data
|
|
56
|
+
*/
|
|
57
|
+
async function buildRequest(client, payload, index, params, total) {
|
|
58
|
+
let retryAttempt = 0;
|
|
59
|
+
sLog(`DM4: summoning worker #${index} of ${total}`, params);
|
|
60
|
+
|
|
61
|
+
try {
|
|
62
|
+
const req = await client.request({
|
|
63
|
+
url: RUNTIME_URL + `?replicate=1&is_replica=true&runId=${params.runId || "no run id"}`,
|
|
64
|
+
method: "POST",
|
|
65
|
+
data: payload,
|
|
66
|
+
headers: {
|
|
67
|
+
"Content-Type": "text/plain",
|
|
68
|
+
},
|
|
69
|
+
timeout: 3600 * 1000 * 10, // 10 hours timeout
|
|
70
|
+
retryConfig: {
|
|
71
|
+
retry: 3,
|
|
72
|
+
onRetryAttempt: (error) => {
|
|
73
|
+
const statusCode = error?.response?.status?.toString() || "";
|
|
74
|
+
retryAttempt++;
|
|
75
|
+
sLog(`DM4: summon worker ${index} retry #${retryAttempt}`, {
|
|
76
|
+
statusCode,
|
|
77
|
+
message: error.message,
|
|
78
|
+
stack: error.stack,
|
|
79
|
+
...params
|
|
80
|
+
}, "DEBUG");
|
|
81
|
+
},
|
|
82
|
+
retryDelay: 1000,
|
|
83
|
+
shouldRetry: (error) => {
|
|
84
|
+
if (error.code === 'ECONNRESET') return true;
|
|
85
|
+
const statusCode = error?.response?.status;
|
|
86
|
+
if (statusCode >= 500) return true;
|
|
87
|
+
if (statusCode === 429) return true;
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
},
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
sLog(`DM4: worker #${index} responded`, params);
|
|
94
|
+
const { data } = req;
|
|
95
|
+
return data;
|
|
96
|
+
} catch (error) {
|
|
97
|
+
sLog(`DM4: worker #${index} failed to respond`, {
|
|
98
|
+
message: error.message,
|
|
99
|
+
stack: error.stack,
|
|
100
|
+
code: error.code,
|
|
101
|
+
retries: retryAttempt,
|
|
102
|
+
...params
|
|
103
|
+
}, "ERROR");
|
|
104
|
+
return {};
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Handle cloud function HTTP entry point
|
|
110
|
+
* @param {Object} req - HTTP request object
|
|
111
|
+
* @param {Object} res - HTTP response object
|
|
112
|
+
* @param {Function} mainFunction - Main data generation function to execute
|
|
113
|
+
* @returns {Promise<void>}
|
|
114
|
+
*/
|
|
115
|
+
export async function handleCloudFunctionEntry(req, res, mainFunction) {
|
|
116
|
+
const reqTimer = timer('request');
|
|
117
|
+
reqTimer.start();
|
|
118
|
+
let response = {};
|
|
119
|
+
let script = req.body || "";
|
|
120
|
+
const params = {
|
|
121
|
+
replicate: 1,
|
|
122
|
+
is_replica: "false",
|
|
123
|
+
runId: "",
|
|
124
|
+
seed: "",
|
|
125
|
+
...req.query
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
const replicate = Number(params.replicate);
|
|
129
|
+
|
|
130
|
+
// Parse boolean parameters
|
|
131
|
+
if (params?.is_replica === "true") params.is_replica = true;
|
|
132
|
+
else params.is_replica = false;
|
|
133
|
+
|
|
134
|
+
const isReplica = params.is_replica;
|
|
135
|
+
|
|
136
|
+
if (!params.runId) params.runId = uid(42);
|
|
137
|
+
|
|
138
|
+
try {
|
|
139
|
+
if (!script) throw new Error("no script");
|
|
140
|
+
|
|
141
|
+
/** @type {Config} */
|
|
142
|
+
const config = eval(script);
|
|
143
|
+
|
|
144
|
+
if (isReplica) {
|
|
145
|
+
// Generate unique seed for replica workers
|
|
146
|
+
const newSeed = (Math.random() / Math.random() / Math.random() / Math.random() / Math.random() / Math.random()).toString();
|
|
147
|
+
config.seed = newSeed;
|
|
148
|
+
params.seed = newSeed;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** @type {Config} */
|
|
152
|
+
const optionsYouCantChange = {
|
|
153
|
+
verbose: false
|
|
154
|
+
};
|
|
155
|
+
|
|
156
|
+
if (replicate <= 1 || isReplica) {
|
|
157
|
+
if (isReplica) sLog("DM4: worker start", params);
|
|
158
|
+
|
|
159
|
+
const { files = [], operations = 0, eventCount = 0, userCount = 0 } = await mainFunction({
|
|
160
|
+
...config,
|
|
161
|
+
...optionsYouCantChange,
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
reqTimer.stop(false);
|
|
165
|
+
response = { files, operations, eventCount, userCount };
|
|
166
|
+
} else {
|
|
167
|
+
sLog(`DM4: job start (${replicate} workers)`, params);
|
|
168
|
+
const results = await spawnFileWorkers(replicate, script, params);
|
|
169
|
+
response = results;
|
|
170
|
+
}
|
|
171
|
+
} catch (e) {
|
|
172
|
+
sLog("DM4: error", { error: e.message, stack: e.stack }, "ERROR");
|
|
173
|
+
response = { error: e.message };
|
|
174
|
+
res.status(500);
|
|
175
|
+
} finally {
|
|
176
|
+
reqTimer.stop(false);
|
|
177
|
+
const { start, end, delta, human } = reqTimer.report(false);
|
|
178
|
+
|
|
179
|
+
if (!isReplica) {
|
|
180
|
+
sLog(`DM4: job end (${human})`, { human, delta, ...params, ...response });
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
if (isReplica) {
|
|
184
|
+
const eps = Math.floor(((response?.eventCount || 0) / delta) * 1000);
|
|
185
|
+
sLog(`DM4: worker end (${human})`, { human, delta, eps, ...params, ...response });
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
response = { ...response, start, end, delta, human, ...params };
|
|
189
|
+
res.send(response);
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Set the runtime URL for the cloud function service
|
|
196
|
+
* @param {string} url - The runtime URL
|
|
197
|
+
*/
|
|
198
|
+
export function setRuntimeUrl(url) {
|
|
199
|
+
RUNTIME_URL = url;
|
|
200
|
+
}
|
|
@@ -1,16 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
import { GoogleGenerativeAI } from "@google/generative-ai";
|
|
2
|
+
import * as u from "ak-tools";
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
const dotenv = require("dotenv");
|
|
6
|
-
dotenv.config();
|
|
4
|
+
import 'dotenv/config';
|
|
7
5
|
|
|
8
6
|
const { GEMINI_API_KEY: API_KEY, NODE_ENV = "unknown" } = process.env;
|
|
9
7
|
if (!API_KEY) throw new Error("Please provide a Gemini API key");
|
|
10
8
|
|
|
11
9
|
async function generateSchema(userInput) {
|
|
12
10
|
const gemini = new GoogleGenerativeAI(API_KEY);
|
|
13
|
-
const model = gemini.getGenerativeModel({ model: "gemini-
|
|
11
|
+
const model = gemini.getGenerativeModel({ model: "gemini-2.0-flash" });
|
|
14
12
|
const PROOMPTY = await u.load("./components/prompt.txt");
|
|
15
13
|
const prompt = `
|
|
16
14
|
Given the following information about a website or app:
|
|
@@ -82,36 +80,10 @@ function validator(schema) {
|
|
|
82
80
|
}
|
|
83
81
|
|
|
84
82
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
Onboarding Started
|
|
91
|
-
Onboarding Completed (Basic)
|
|
92
|
-
Onboarding Completed (Premium)
|
|
93
|
-
Page Views (all "screens" within the app")
|
|
94
|
-
Enable/Disable High Risk Blocking
|
|
95
|
-
Enable/Disable Medium Risk Blocking
|
|
96
|
-
Enable/Disable Neighborhood Spoof Blocking
|
|
97
|
-
Call Blocked (Spam)
|
|
98
|
-
Call Blocked (Custom List)
|
|
99
|
-
Branded Call w/o Logo Received
|
|
100
|
-
Branded Call w/ Logo Received
|
|
101
|
-
Branded Call Answered
|
|
102
|
-
Branded Call Blocked
|
|
103
|
-
Enable/Disable Text Spam
|
|
104
|
-
Reverse Number Lookup
|
|
105
|
-
Report as Spam
|
|
106
|
-
Report as Not Spam
|
|
107
|
-
Custom Block List Number Add
|
|
108
|
-
Custom Block List Number Remove
|
|
109
|
-
Call Arrives Before Push
|
|
110
|
-
Error Scenarios
|
|
111
|
-
User Can't Authenticate
|
|
112
|
-
Xfinity Services Can't Connect
|
|
113
|
-
Verizon Services Can't Connect
|
|
114
|
-
Deep Links into app`)
|
|
83
|
+
export { generateSchema };
|
|
84
|
+
|
|
85
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
86
|
+
generateSchema(`metube, a video streaming company like youutube, where users watch videos, search, like, comment, subscribe, share, create playlists, etc...`)
|
|
115
87
|
.then((result) => {
|
|
116
88
|
if (NODE_ENV === "dev") debugger;
|
|
117
89
|
})
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
import { ChartJSNodeCanvas } from 'chartjs-node-canvas';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import * as u from 'ak-tools';
|
|
4
|
+
import dayjs from 'dayjs';
|
|
5
|
+
import { openFinder } from './utils.js';
|
|
6
6
|
const { existsSync } = fs;
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
import path from 'path';
|
|
8
|
+
import 'dotenv/config';
|
|
9
9
|
const { NODE_ENV = "unknown" } = process.env;
|
|
10
10
|
|
|
11
11
|
|
|
@@ -186,11 +186,11 @@ async function generateLineChart(rawData, signupEvents = ["sign up"], fileName)
|
|
|
186
186
|
return file;
|
|
187
187
|
}
|
|
188
188
|
|
|
189
|
-
|
|
189
|
+
export { generateLineChart };
|
|
190
190
|
|
|
191
191
|
|
|
192
192
|
|
|
193
|
-
if (
|
|
193
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
194
194
|
generateLineChart()
|
|
195
195
|
.then((result)=>{
|
|
196
196
|
if (NODE_ENV === "dev") debugger;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
import 'dotenv/config';
|
|
2
|
+
import * as akTools from 'ak-tools';
|
|
3
3
|
const { rand, makeName } = akTools;
|
|
4
4
|
let { OAUTH_TOKEN = "" } = process.env;
|
|
5
5
|
const { NODE_ENV = "unknown" } = process.env;
|
|
@@ -153,7 +153,7 @@ async function addGroupKeys(groupKeyDfns = [], projectId, oauthToken = OAUTH_TOK
|
|
|
153
153
|
}
|
|
154
154
|
|
|
155
155
|
|
|
156
|
-
if (
|
|
156
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
157
157
|
main()
|
|
158
158
|
.then((result)=>{
|
|
159
159
|
if (NODE_ENV === "dev") debugger;
|
|
@@ -163,4 +163,4 @@ if (require.main === module) {
|
|
|
163
163
|
})
|
|
164
164
|
}
|
|
165
165
|
|
|
166
|
-
|
|
166
|
+
export default main;
|
|
@@ -1,24 +1,24 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import Chance from 'chance';
|
|
3
|
+
import readline from 'readline';
|
|
4
|
+
import { comma, uid, clone } from 'ak-tools';
|
|
5
|
+
import { spawn } from 'child_process';
|
|
6
|
+
import dayjs from 'dayjs';
|
|
7
|
+
import utc from 'dayjs/plugin/utc.js';
|
|
8
|
+
import path from 'path';
|
|
9
|
+
import { mkdir, parseGCSUri } from 'ak-tools';
|
|
10
|
+
import { existsSync } from 'fs';
|
|
11
11
|
dayjs.extend(utc);
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
import 'dotenv/config';
|
|
13
|
+
import { domainSuffix, domainPrefix } from '../data/defaults.js';
|
|
14
14
|
|
|
15
|
-
/** @typedef {import('
|
|
16
|
-
/** @typedef {import('
|
|
17
|
-
/** @typedef {import('
|
|
18
|
-
/** @typedef {import('
|
|
19
|
-
/** @typedef {import('
|
|
20
|
-
/** @typedef {import('
|
|
21
|
-
/** @typedef {import('
|
|
15
|
+
/** @typedef {import('../../types').Dungeon} Config */
|
|
16
|
+
/** @typedef {import('../../types').EventConfig} EventConfig */
|
|
17
|
+
/** @typedef {import('../../types').ValueValid} ValueValid */
|
|
18
|
+
/** @typedef {import('../../types').HookedArray} hookArray */
|
|
19
|
+
/** @typedef {import('../../types').hookArrayOptions} hookArrayOptions */
|
|
20
|
+
/** @typedef {import('../../types').Person} Person */
|
|
21
|
+
/** @typedef {import('../../types').Funnel} Funnel */
|
|
22
22
|
|
|
23
23
|
let globalChance;
|
|
24
24
|
let chanceInitialized = false;
|
|
@@ -26,7 +26,7 @@ let chanceInitialized = false;
|
|
|
26
26
|
const ACTUAL_NOW = dayjs.utc();
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
import { Storage as cloudStorage } from '@google-cloud/storage';
|
|
30
30
|
const projectId = 'YOUR_PROJECT_ID';
|
|
31
31
|
const storage = new cloudStorage({ projectId });
|
|
32
32
|
|
|
@@ -192,6 +192,9 @@ function choose(value) {
|
|
|
192
192
|
value = value();
|
|
193
193
|
}
|
|
194
194
|
|
|
195
|
+
if (Array.isArray(value) && value.length === 0) {
|
|
196
|
+
return ""; // Return empty string if the array is empty
|
|
197
|
+
}
|
|
195
198
|
|
|
196
199
|
// [[],[],[]] should pick one
|
|
197
200
|
if (Array.isArray(value) && Array.isArray(value[0])) {
|
|
@@ -947,7 +950,15 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
|
|
|
947
950
|
if (!earliestTime) earliestTime = global.FIXED_BEGIN ? global.FIXED_BEGIN : dayjs().subtract(30, 'd').unix(); // 30 days ago
|
|
948
951
|
if (!latestTime) latestTime = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
949
952
|
const chance = getChance();
|
|
950
|
-
|
|
953
|
+
let totalRange = latestTime - earliestTime;
|
|
954
|
+
if (totalRange <= 0 || earliestTime > latestTime) {
|
|
955
|
+
//just flip earliest and latest
|
|
956
|
+
let tempEarly = latestTime
|
|
957
|
+
let tempLate = earliestTime;
|
|
958
|
+
earliestTime = tempEarly;
|
|
959
|
+
latestTime = tempLate;
|
|
960
|
+
totalRange = latestTime - earliestTime;
|
|
961
|
+
}
|
|
951
962
|
const chunkSize = totalRange / peaks;
|
|
952
963
|
|
|
953
964
|
// Select a random chunk based on the number of peaks
|
|
@@ -966,6 +977,7 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
|
|
|
966
977
|
offset = chance.normal({ mean: mean, dev: chunkSize / deviation });
|
|
967
978
|
isValidTime = validTime(chunkMid + offset, earliestTime, latestTime);
|
|
968
979
|
if (iterations > 25000) {
|
|
980
|
+
if (process.env?.NODE_ENV === 'dev') debugger;
|
|
969
981
|
throw `${iterations} iterations... exceeded`;
|
|
970
982
|
}
|
|
971
983
|
} while (chunkMid + offset < chunkStart || chunkMid + offset > chunkEnd);
|
|
@@ -1141,7 +1153,7 @@ function generateEmoji(max = 10, array = false) {
|
|
|
1141
1153
|
|
|
1142
1154
|
|
|
1143
1155
|
|
|
1144
|
-
|
|
1156
|
+
export {
|
|
1145
1157
|
pick,
|
|
1146
1158
|
date,
|
|
1147
1159
|
dates,
|
|
@@ -1152,7 +1164,7 @@ module.exports = {
|
|
|
1152
1164
|
TimeSoup,
|
|
1153
1165
|
companyName,
|
|
1154
1166
|
generateEmoji,
|
|
1155
|
-
haveSameKeys
|
|
1167
|
+
hasSameKeys as haveSameKeys,
|
|
1156
1168
|
|
|
1157
1169
|
initChance,
|
|
1158
1170
|
getChance,
|
package/package.json
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "make-mp-data",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "builds all mixpanel primitives for a given project",
|
|
5
|
+
"type": "module",
|
|
5
6
|
"main": "index.js",
|
|
6
7
|
"types": "types.d.ts",
|
|
7
8
|
"scripts": {
|
|
@@ -10,8 +11,10 @@
|
|
|
10
11
|
"prune": "rm -f ./data/* && rm -f ./tmp/* && rm -f vscode-profile-*",
|
|
11
12
|
"post": "npm publish",
|
|
12
13
|
"deps": "./scripts/update-deps.sh",
|
|
13
|
-
"test": "NODE_ENV=test
|
|
14
|
-
"
|
|
14
|
+
"test": "NODE_ENV=test vitest run",
|
|
15
|
+
"test:watch": "NODE_ENV=test vitest",
|
|
16
|
+
"test:ui": "NODE_ENV=test vitest --ui",
|
|
17
|
+
"coverage": "vitest run --coverage && open ./coverage/index.html",
|
|
15
18
|
"new:dungeon": "./scripts/new-dungeon.sh",
|
|
16
19
|
"new:project": "node ./scripts/new-project.mjs",
|
|
17
20
|
"exp:benchmark": "node --no-warnings --experimental-vm-modules ./tests/benchmark/concurrency.mjs",
|
|
@@ -47,32 +50,29 @@
|
|
|
47
50
|
"dependencies": {
|
|
48
51
|
"@google-cloud/functions-framework": "^3.4.2",
|
|
49
52
|
"@google-cloud/storage": "^7.14.0",
|
|
50
|
-
"@google/generative-ai": "^0.
|
|
51
|
-
"ak-fetch": "^
|
|
52
|
-
"ak-
|
|
53
|
+
"@google/generative-ai": "^0.24.1",
|
|
54
|
+
"ak-fetch": "^2.0.1",
|
|
55
|
+
"ak-gemini": "^1.0.55",
|
|
56
|
+
"ak-tools": "^1.1.0",
|
|
53
57
|
"chance": "^1.1.11",
|
|
54
58
|
"chart.js": "^3.9.1",
|
|
55
59
|
"chartjs-node-canvas": "^4.1.6",
|
|
56
60
|
"dayjs": "^1.11.11",
|
|
57
61
|
"dotenv": "^16.4.5",
|
|
58
62
|
"google-auth-library": "^9.15.0",
|
|
59
|
-
"mixpanel-import": "^2.
|
|
63
|
+
"mixpanel-import": "^2.8.14",
|
|
60
64
|
"p-limit": "^3.1.0",
|
|
61
65
|
"yargs": "^17.7.2"
|
|
62
66
|
},
|
|
63
67
|
"devDependencies": {
|
|
64
|
-
"@
|
|
65
|
-
"
|
|
66
|
-
"
|
|
67
|
-
},
|
|
68
|
-
"jest": {
|
|
69
|
-
"preset": "./tests/jest.config.js"
|
|
68
|
+
"@vitest/ui": "^2.1.9",
|
|
69
|
+
"nodemon": "^3.1.3",
|
|
70
|
+
"vitest": "^2.1.9"
|
|
70
71
|
},
|
|
71
72
|
"nodemonConfig": {
|
|
72
73
|
"ignore": [
|
|
73
74
|
"data/",
|
|
74
75
|
"tmp/"
|
|
75
|
-
|
|
76
76
|
]
|
|
77
77
|
}
|
|
78
|
-
}
|
|
78
|
+
}
|