@ak--47/dungeon-master 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +518 -0
- package/dungeons/array-of-object-lookup-schema.json +327 -0
- package/dungeons/array-of-object-lookup.js +220 -0
- package/dungeons/ecommerce-schema.json +462 -0
- package/dungeons/ecommerce.js +447 -0
- package/dungeons/education-schema.json +2409 -0
- package/dungeons/education.js +768 -0
- package/dungeons/fintech-schema.json +14034 -0
- package/dungeons/fintech.js +696 -0
- package/dungeons/foobar-schema.json +403 -0
- package/dungeons/foobar.js +296 -0
- package/dungeons/food-delivery-schema.json +192 -0
- package/dungeons/food-delivery.js +602 -0
- package/dungeons/food-schema.json +1152 -0
- package/dungeons/food.js +754 -0
- package/dungeons/gaming-schema.json +1270 -0
- package/dungeons/gaming.js +508 -0
- package/dungeons/insurance-application-schema.json +204 -0
- package/dungeons/insurance-application.js +605 -0
- package/dungeons/media-schema.json +906 -0
- package/dungeons/media.js +790 -0
- package/dungeons/retention-cadence-schema.json +78 -0
- package/dungeons/retention-cadence.js +244 -0
- package/dungeons/rpg-schema.json +4526 -0
- package/dungeons/rpg.js +919 -0
- package/dungeons/sanity-schema.json +255 -0
- package/dungeons/sanity.js +152 -0
- package/dungeons/sass-schema.json +1291 -0
- package/dungeons/sass.js +795 -0
- package/dungeons/scd-schema.json +919 -0
- package/dungeons/scd.js +277 -0
- package/dungeons/simple-schema.json +608 -0
- package/dungeons/simple.js +285 -0
- package/dungeons/simplest-schema.json +1418 -0
- package/dungeons/simplest.js +392 -0
- package/dungeons/social-schema.json +1118 -0
- package/dungeons/social.js +686 -0
- package/dungeons/text-generation-schema.json +3096 -0
- package/dungeons/text-generation.js +812 -0
- package/index.js +567 -0
- package/lib/core/config-validator.js +395 -0
- package/lib/core/context.js +204 -0
- package/lib/core/dungeon-loader.js +337 -0
- package/lib/core/storage.js +379 -0
- package/lib/generators/adspend.js +132 -0
- package/lib/generators/events.js +271 -0
- package/lib/generators/funnels.js +407 -0
- package/lib/generators/mirror.js +167 -0
- package/lib/generators/product-lookup.js +262 -0
- package/lib/generators/product-names.js +195 -0
- package/lib/generators/profiles.js +93 -0
- package/lib/generators/scd.js +124 -0
- package/lib/generators/text.js +1192 -0
- package/lib/orchestrators/mixpanel-sender.js +266 -0
- package/lib/orchestrators/user-loop.js +335 -0
- package/lib/templates/abbreviated.d.ts +169 -0
- package/lib/templates/defaults.js +1405 -0
- package/lib/templates/phrases.js +2526 -0
- package/lib/templates/schema.d.ts +173 -0
- package/lib/templates/soup-presets.js +188 -0
- package/lib/utils/function-registry.js +302 -0
- package/lib/utils/json-evaluator.js +172 -0
- package/lib/utils/logger.js +34 -0
- package/lib/utils/utils.js +1490 -0
- package/package.json +89 -0
- package/types.d.ts +865 -0
|
@@ -0,0 +1,1490 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import Chance from 'chance';
|
|
3
|
+
import readline from 'readline';
|
|
4
|
+
import { comma, uid } from 'ak-tools';
|
|
5
|
+
import dayjs from 'dayjs';
|
|
6
|
+
import utc from 'dayjs/plugin/utc.js';
|
|
7
|
+
import path from 'path';
|
|
8
|
+
import { mkdir, parseGCSUri } from 'ak-tools';
|
|
9
|
+
import { existsSync } from 'fs';
|
|
10
|
+
import zlib from 'zlib';
|
|
11
|
+
dayjs.extend(utc);
|
|
12
|
+
import 'dotenv/config';
|
|
13
|
+
import { domainSuffix, domainPrefix } from '../templates/defaults.js';
|
|
14
|
+
const { NODE_ENV = "unknown" } = process.env;
|
|
15
|
+
|
|
16
|
+
/** @typedef {import('../../types').Dungeon} Config */
|
|
17
|
+
/** @typedef {import('../../types').EventConfig} EventConfig */
|
|
18
|
+
/** @typedef {import('../../types').ValueValid} ValueValid */
|
|
19
|
+
/** @typedef {import('../../types').HookedArray} hookArray */
|
|
20
|
+
/** @typedef {import('../../types').hookArrayOptions} hookArrayOptions */
|
|
21
|
+
/** @typedef {import('../../types').Person} Person */
|
|
22
|
+
/** @typedef {import('../../types').Funnel} Funnel */
|
|
23
|
+
|
|
24
|
+
let globalChance;
|
|
25
|
+
let chanceInitialized = false;
|
|
26
|
+
|
|
27
|
+
const ACTUAL_NOW = dayjs.utc();
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
import { Storage as cloudStorage } from '@google-cloud/storage';
|
|
31
|
+
const projectId = 'YOUR_PROJECT_ID';
|
|
32
|
+
const storage = new cloudStorage({ projectId });
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
/*
|
|
36
|
+
----
|
|
37
|
+
RNG
|
|
38
|
+
----
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* the random number generator initialization function
|
|
43
|
+
* @param {string} seed
|
|
44
|
+
* @returns {Chance}
|
|
45
|
+
*/
|
|
46
|
+
function initChance(seed) {
|
|
47
|
+
if (process.env.SEED) seed = process.env.SEED; // Override seed with environment variable if available
|
|
48
|
+
if (!chanceInitialized) {
|
|
49
|
+
globalChance = new Chance(seed);
|
|
50
|
+
chanceInitialized = true;
|
|
51
|
+
}
|
|
52
|
+
return globalChance;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* the random number generator getter function
|
|
57
|
+
* @returns {Chance}
|
|
58
|
+
*/
|
|
59
|
+
function getChance() {
|
|
60
|
+
if (!chanceInitialized) {
|
|
61
|
+
const seed = process.env.SEED || "";
|
|
62
|
+
if (!seed) {
|
|
63
|
+
return new Chance(); // this is a new RNG and therefore not deterministic
|
|
64
|
+
}
|
|
65
|
+
return initChance(seed);
|
|
66
|
+
}
|
|
67
|
+
return globalChance;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/*
|
|
71
|
+
----
|
|
72
|
+
PICKERS
|
|
73
|
+
----
|
|
74
|
+
*/
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* choose a value from an array or a function
|
|
78
|
+
* @param {ValueValid} items
|
|
79
|
+
*/
|
|
80
|
+
function pick(items) {
|
|
81
|
+
const chance = getChance();
|
|
82
|
+
if (!Array.isArray(items)) {
|
|
83
|
+
if (typeof items === 'function') {
|
|
84
|
+
const selection = items();
|
|
85
|
+
if (Array.isArray(selection)) {
|
|
86
|
+
return chance.pickone(selection);
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
return selection;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return items;
|
|
93
|
+
|
|
94
|
+
}
|
|
95
|
+
return chance.pickone(items);
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* returns a random date in the past or future
|
|
100
|
+
* @param {number} inTheLast=30
|
|
101
|
+
* @param {boolean} isPast=true
|
|
102
|
+
* @param {string} format='YYYY-MM-DD'
|
|
103
|
+
*/
|
|
104
|
+
function date(inTheLast = 30, isPast = true, format = 'YYYY-MM-DD') {
|
|
105
|
+
const chance = getChance();
|
|
106
|
+
const now = ACTUAL_NOW;
|
|
107
|
+
if (Math.abs(inTheLast) > 365 * 10) inTheLast = chance.integer({ min: 1, max: 180 });
|
|
108
|
+
return function () {
|
|
109
|
+
const when = chance.integer({ min: 0, max: Math.abs(inTheLast) });
|
|
110
|
+
let then;
|
|
111
|
+
if (isPast) {
|
|
112
|
+
then = now.subtract(when, 'day')
|
|
113
|
+
.subtract(integer(0, 23), 'hour')
|
|
114
|
+
.subtract(integer(0, 59), 'minute')
|
|
115
|
+
.subtract(integer(0, 59), 'second');
|
|
116
|
+
} else {
|
|
117
|
+
then = now.add(when, 'day')
|
|
118
|
+
.add(integer(0, 23), 'hour')
|
|
119
|
+
.add(integer(0, 59), 'minute')
|
|
120
|
+
.add(integer(0, 59), 'second');
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
return format ? then.format(format) : then.toISOString();
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* returns pairs of random date in the past or future
|
|
129
|
+
* @param {number} inTheLast=30
|
|
130
|
+
* @param {number} numPairs=5
|
|
131
|
+
* @param {string} format='YYYY-MM-DD'
|
|
132
|
+
*/
|
|
133
|
+
function dates(inTheLast = 30, numPairs = 5, format = 'YYYY-MM-DD') {
|
|
134
|
+
const pairs = [];
|
|
135
|
+
for (let i = 0; i < numPairs; i++) {
|
|
136
|
+
pairs.push([date(inTheLast, true, format), date(inTheLast, true, format)]);
|
|
137
|
+
}
|
|
138
|
+
return pairs;
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
function datesBetween(start, end) {
|
|
142
|
+
const result = [];
|
|
143
|
+
if (typeof start === 'number') start = dayjs.unix(start).utc();
|
|
144
|
+
if (typeof start !== 'number') start = dayjs(start).utc();
|
|
145
|
+
if (typeof end === 'number') end = dayjs.unix(end).utc();
|
|
146
|
+
if (typeof end !== 'number') end = dayjs(end).utc();
|
|
147
|
+
const diff = end.diff(start, 'day');
|
|
148
|
+
for (let i = 0; i < diff; i++) {
|
|
149
|
+
const day = start.add(i, 'day').startOf('day').add(12, 'hour');
|
|
150
|
+
result.push(day.toISOString());
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return result;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* returns a random date
|
|
158
|
+
* @param {any} start
|
|
159
|
+
* @param {any} end
|
|
160
|
+
*/
|
|
161
|
+
function day(start, end) {
|
|
162
|
+
// if (!end) end = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
163
|
+
if (!start) start = ACTUAL_NOW.subtract(30, 'd').toISOString();
|
|
164
|
+
if (!end) end = ACTUAL_NOW.toISOString();
|
|
165
|
+
const chance = getChance();
|
|
166
|
+
const format = 'YYYY-MM-DD';
|
|
167
|
+
return function (min, max) {
|
|
168
|
+
start = dayjs(start);
|
|
169
|
+
end = dayjs(end);
|
|
170
|
+
const diff = end.diff(start, 'day');
|
|
171
|
+
const delta = chance.integer({ min: min, max: diff });
|
|
172
|
+
const day = start.add(delta, 'day');
|
|
173
|
+
return {
|
|
174
|
+
start: start.format(format),
|
|
175
|
+
end: end.format(format),
|
|
176
|
+
day: day.format(format)
|
|
177
|
+
};
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* similar to pick
|
|
184
|
+
* @param {ValueValid} value
|
|
185
|
+
*/
|
|
186
|
+
function choose(value) {
|
|
187
|
+
const chance = getChance();
|
|
188
|
+
|
|
189
|
+
// most of the time this will receive a list of strings;
|
|
190
|
+
// when that is the case, we need to ensure some 'keywords' like 'variant' or 'test' aren't in the array
|
|
191
|
+
// next we want to see if the array is unweighted ... i.e. no dupe strings and each string only occurs once ['a', 'b', 'c', 'd']
|
|
192
|
+
// if all these are true we will pickAWinner(value)()
|
|
193
|
+
if (Array.isArray(value) && value.length > 2 && value.length < 20 && value.every(item => typeof item === 'string')) {
|
|
194
|
+
// ensure terms 'variant' 'group' 'experiment' or 'population' are NOT in any of the items
|
|
195
|
+
if (!value.some(item => item.includes('variant') || item.includes('group') || item.includes('experiment') || item.includes('population'))) {
|
|
196
|
+
// check to make sure that each element in the array only occurs once...
|
|
197
|
+
const uniqueItems = new Set(value);
|
|
198
|
+
if (uniqueItems.size === value.length) {
|
|
199
|
+
// Array has no duplicates, use pickAWinner
|
|
200
|
+
const quickList = pickAWinner(value, 0)();
|
|
201
|
+
const theChosenOne = chance.pickone(quickList);
|
|
202
|
+
return theChosenOne;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// if the thing has a .next() method, call that (e.g., generators/iterators)
|
|
210
|
+
try {
|
|
211
|
+
if (value && typeof /** @type {any} */ (value).next === 'function') {
|
|
212
|
+
return /** @type {any} */ (value).next();
|
|
213
|
+
}
|
|
214
|
+
} catch (e) {
|
|
215
|
+
console.error(`Error occurred while calling next(): ${e}`);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
try {
|
|
219
|
+
// Keep resolving the value if it's a function (with caching)
|
|
220
|
+
while (typeof value === 'function') {
|
|
221
|
+
const funcString = value.toString();
|
|
222
|
+
|
|
223
|
+
// Check cache for weighted array functions
|
|
224
|
+
if (typeof global.weightedArrayCache === 'undefined') {
|
|
225
|
+
global.weightedArrayCache = new Map();
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
if (global.weightedArrayCache.has(funcString)) {
|
|
229
|
+
value = global.weightedArrayCache.get(funcString);
|
|
230
|
+
break;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
const result = value();
|
|
234
|
+
if (Array.isArray(result) && result.length > 10) {
|
|
235
|
+
// Cache large arrays (likely weighted arrays)
|
|
236
|
+
global.weightedArrayCache.set(funcString, result);
|
|
237
|
+
}
|
|
238
|
+
value = result;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
if (Array.isArray(value) && value.length === 0) {
|
|
242
|
+
return ""; // Return empty string if the array is empty
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// [[],[],[]] should pick one
|
|
246
|
+
if (Array.isArray(value) && Array.isArray(value[0])) {
|
|
247
|
+
return chance.pickone(value);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// PERFORMANCE: Optimized array handling - check first item type instead of every()
|
|
251
|
+
if (Array.isArray(value) && value.length > 0) {
|
|
252
|
+
const firstType = typeof value[0];
|
|
253
|
+
if (firstType === 'string' || firstType === 'number') {
|
|
254
|
+
return chance.pickone(value);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
if (Array.isArray(value) && value.every(item => typeof item === 'object')) {
|
|
259
|
+
if (hasSameKeys(value)) return value;
|
|
260
|
+
else {
|
|
261
|
+
if (process.env.NODE_ENV === "dev") debugger;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// ["","",""] should pick-a-winner
|
|
266
|
+
if (Array.isArray(value) && typeof value[0] === "string") {
|
|
267
|
+
value = pickAWinner(value)();
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// [0,1,2] should pick one
|
|
271
|
+
if (Array.isArray(value) && typeof value[0] === "number") {
|
|
272
|
+
return chance.pickone(value);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
if (Array.isArray(value)) {
|
|
276
|
+
return chance.pickone(value);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
if (typeof value === 'string') {
|
|
280
|
+
return value;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
if (typeof value === 'number') {
|
|
284
|
+
return value;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// If it's not a function or array, return it as is
|
|
288
|
+
return value;
|
|
289
|
+
}
|
|
290
|
+
catch (e) {
|
|
291
|
+
console.error(`\n\nerror on value: ${value};\n\n`, e, '\n\n');
|
|
292
|
+
if (process.env?.NODE_ENV === 'dev') debugger;
|
|
293
|
+
throw e;
|
|
294
|
+
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
function hasSameKeys(arr) {
|
|
300
|
+
if (arr.length <= 1) {
|
|
301
|
+
return true; // An empty array or an array with one object always has the same keys
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
const firstKeys = Object.keys(arr[0]);
|
|
305
|
+
|
|
306
|
+
for (let i = 1; i < arr.length; i++) {
|
|
307
|
+
const currentKeys = Object.keys(arr[i]);
|
|
308
|
+
|
|
309
|
+
if (currentKeys.length !== firstKeys.length) {
|
|
310
|
+
return false; // Different number of keys
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
for (const key of firstKeys) {
|
|
314
|
+
if (!currentKeys.includes(key)) {
|
|
315
|
+
return false; // Key missing in current object
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
return true; // All objects have the same keys
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* keeps picking from an array until the array is exhausted
|
|
325
|
+
* @param {Array} arr
|
|
326
|
+
*/
|
|
327
|
+
function exhaust(arr) {
|
|
328
|
+
return function () {
|
|
329
|
+
return arr.shift();
|
|
330
|
+
};
|
|
331
|
+
};
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* returns a random integer between min and max
|
|
335
|
+
* @param {number} min=1
|
|
336
|
+
* @param {number} max=100
|
|
337
|
+
*/
|
|
338
|
+
function integer(min = 1, max = 100) {
|
|
339
|
+
const chance = getChance();
|
|
340
|
+
if (min === max) {
|
|
341
|
+
return min;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
if (min > max) {
|
|
345
|
+
return chance.integer({
|
|
346
|
+
min: max,
|
|
347
|
+
max: min
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
if (min < max) {
|
|
352
|
+
return chance.integer({
|
|
353
|
+
min: min,
|
|
354
|
+
max: max
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
return 0;
|
|
359
|
+
};
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
function decimal(min = 0, max = 1, fixed = 2) {
|
|
363
|
+
const chance = getChance();
|
|
364
|
+
return chance.floating({ min, max, fixed });
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
/*
|
|
369
|
+
----
|
|
370
|
+
GENERATORS
|
|
371
|
+
----
|
|
372
|
+
*/
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* returns a random float between 0 and 1
|
|
376
|
+
* a substitute for Math.random
|
|
377
|
+
*/
|
|
378
|
+
function boxMullerRandom() {
|
|
379
|
+
const chance = getChance();
|
|
380
|
+
let u = 0, v = 0;
|
|
381
|
+
while (u === 0) u = chance.floating({ min: 0, max: 1, fixed: 13 });
|
|
382
|
+
while (v === 0) v = chance.floating({ min: 0, max: 1, fixed: 13 });
|
|
383
|
+
return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
function optimizedBoxMuller() {
|
|
387
|
+
const chance = getChance();
|
|
388
|
+
const u = Math.max(Math.min(chance.normal({ mean: .5, dev: .25 }), 1), 0);
|
|
389
|
+
const v = Math.max(Math.min(chance.normal({ mean: .5, dev: .25 }), 1), 0);
|
|
390
|
+
const result = Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
|
|
391
|
+
//ensure we didn't get infinity
|
|
392
|
+
if (result === Infinity || result === -Infinity) return chance.floating({ min: 0, max: 1 });
|
|
393
|
+
return result;
|
|
394
|
+
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* applies a skew to a value;
|
|
399
|
+
* Skew=0.5: When the skew is 0.5, the distribution becomes more compressed, with values clustering closer to the mean.
|
|
400
|
+
* Skew=1: With a skew of 1, the distribution remains unchanged, as this is equivalent to applying no skew.
|
|
401
|
+
* Skew=2: When the skew is 2, the distribution spreads out, with values extending further from the mean.
|
|
402
|
+
* @param {number} value
|
|
403
|
+
* @param {number} skew
|
|
404
|
+
*/
|
|
405
|
+
function applySkew(value, skew) {
|
|
406
|
+
if (skew === 1) return value;
|
|
407
|
+
// Adjust the value based on skew
|
|
408
|
+
let sign = value < 0 ? -1 : 1;
|
|
409
|
+
return sign * Math.pow(Math.abs(value), skew);
|
|
410
|
+
};
|
|
411
|
+
|
|
412
|
+
// Map standard normal value to our range
|
|
413
|
+
function mapToRange(value, mean, sd) {
|
|
414
|
+
return Math.round(value * sd + mean);
|
|
415
|
+
};
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* generate a range of numbers
|
|
419
|
+
* @param {number} a
|
|
420
|
+
* @param {number} b
|
|
421
|
+
* @param {number} step=1
|
|
422
|
+
*/
|
|
423
|
+
function range(a, b, step = 1) {
|
|
424
|
+
const arr = [];
|
|
425
|
+
step = !step ? 1 : step;
|
|
426
|
+
b = b / step;
|
|
427
|
+
for (var i = a; i <= b; i++) {
|
|
428
|
+
arr.push(i * step);
|
|
429
|
+
}
|
|
430
|
+
return arr;
|
|
431
|
+
};
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
function companyName(words = 2, separator = " ") {
|
|
435
|
+
const industryAdjectives = ["advanced", "premier", "integrated", "optimized", "comprehensive", "expert",
|
|
436
|
+
"visionary", "progressive", "transformative", "pioneering", "streamlined",
|
|
437
|
+
"cutting-edge", "impactful", "purpose-driven", "value-oriented", "future-ready",
|
|
438
|
+
"scalable", "responsive", "data-driven", "cloud-based", "user-friendly",
|
|
439
|
+
"high-performance", "secure", "compliant", "ethical", "inclusive",
|
|
440
|
+
"transparent", "community-focused", "environmentally-conscious", "socially-responsible", "innovative", "dynamic", "global", "leading", "reliable", "trusted",
|
|
441
|
+
"strategic", "efficient", "sustainable", "creative", "agile", "resilient",
|
|
442
|
+
"collaborative", "customer-centric", "forward-thinking", "results-driven", "gizmo", "contraption", "doodle", "whimsy", "quirk", "spark", "zing",
|
|
443
|
+
"zap", "pop", "fizz", "whirl", "twirl", "swirl", "jumble", "tumble",
|
|
444
|
+
"hodgepodge", "mishmash", "kaleidoscope", "labyrinth", "maze", "puzzle",
|
|
445
|
+
"enigma", "conundrum", "paradox", "oxymoron", "chimera", "centaur",
|
|
446
|
+
"griffin", "phoenix", "unicorn", "dragon", "mermaid", "yeti", "bigfoot",
|
|
447
|
+
"loch ness monster", "chupacabra", "kraken", "leviathan", "behemoth",
|
|
448
|
+
"juggernaut", "goliath", "david", "odyssey", "pilgrimage", "crusade",
|
|
449
|
+
"quest", "adventure", "escapade", "frolic", "romp", "lark", "spree",
|
|
450
|
+
"binge", "jag", "bender", "tear", "rampage", "riot", "ruckus", "rumpus",
|
|
451
|
+
"hullabaloo", "brouhaha", "kerfuffle", "shindig", "hootenanny", "jamboree",
|
|
452
|
+
"fiesta", "carnival", "gala", "soiree", "bash", "fete", "jubilee"
|
|
453
|
+
|
|
454
|
+
];
|
|
455
|
+
|
|
456
|
+
const companyNouns = [
|
|
457
|
+
"solutions", "group", "partners", "ventures", "holdings", "enterprises",
|
|
458
|
+
"systems", "technologies", "innovations", "associates", "corporation", "inc.",
|
|
459
|
+
"ltd.", "plc.", "gmbh", "s.a.", "llc.", "network", "alliance", "consortium", "collective", "foundation", "institute",
|
|
460
|
+
"laboratory", "agency", "bureau", "department", "division", "branch",
|
|
461
|
+
"office", "center", "hub", "platform", "ecosystem", "marketplace",
|
|
462
|
+
"exchange", "clearinghouse", "repository", "archive", "registry",
|
|
463
|
+
"database", "framework", "infrastructure", "architecture", "protocol",
|
|
464
|
+
"standard", "specification", "guideline", "blueprint", "roadmap",
|
|
465
|
+
"strategy", "plan", "initiative", "program", "project", "campaign",
|
|
466
|
+
"operation", "mission", "task", "force", "team", "crew", "squad",
|
|
467
|
+
"unit", "cell", "pod", "cohort", "community", "network", "circle",
|
|
468
|
+
"forum", "council", "board", "committee", "panel", "jury", "tribunal"
|
|
469
|
+
];
|
|
470
|
+
|
|
471
|
+
let name = "";
|
|
472
|
+
const cycle = [industryAdjectives, companyNouns];
|
|
473
|
+
for (let i = 0; i < words; i++) {
|
|
474
|
+
const index = i % cycle.length;
|
|
475
|
+
const word = cycle[index][getChance().integer({ min: 0, max: cycle[index].length - 1 })];
|
|
476
|
+
if (name === "") {
|
|
477
|
+
name = word;
|
|
478
|
+
} else {
|
|
479
|
+
name += separator + word;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
return name;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
/*
|
|
488
|
+
----
|
|
489
|
+
STREAMERS
|
|
490
|
+
----
|
|
491
|
+
*/
|
|
492
|
+
|
|
493
|
+
function streamJSON(filePath, data, options = {}) {
|
|
494
|
+
return new Promise((resolve, reject) => {
|
|
495
|
+
let writeStream;
|
|
496
|
+
const { gzip = false } = options;
|
|
497
|
+
|
|
498
|
+
if (filePath?.startsWith('gs://')) {
|
|
499
|
+
const { uri, bucket, file } = parseGCSUri(filePath);
|
|
500
|
+
writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
|
|
501
|
+
}
|
|
502
|
+
else {
|
|
503
|
+
writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
|
|
504
|
+
if (gzip) {
|
|
505
|
+
const gzipStream = zlib.createGzip();
|
|
506
|
+
gzipStream.pipe(writeStream);
|
|
507
|
+
writeStream = gzipStream;
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
data.forEach(item => {
|
|
511
|
+
writeStream.write(JSON.stringify(item) + '\n');
|
|
512
|
+
});
|
|
513
|
+
writeStream.end();
|
|
514
|
+
writeStream.on('finish', () => {
|
|
515
|
+
resolve(filePath);
|
|
516
|
+
});
|
|
517
|
+
writeStream.on('error', reject);
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
function streamCSV(filePath, data, options = {}) {
|
|
522
|
+
return new Promise((resolve, reject) => {
|
|
523
|
+
let writeStream;
|
|
524
|
+
const { gzip = false } = options;
|
|
525
|
+
|
|
526
|
+
if (filePath?.startsWith('gs://')) {
|
|
527
|
+
const { uri, bucket, file } = parseGCSUri(filePath);
|
|
528
|
+
writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
|
|
529
|
+
}
|
|
530
|
+
else {
|
|
531
|
+
writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
|
|
532
|
+
if (gzip) {
|
|
533
|
+
const gzipStream = zlib.createGzip();
|
|
534
|
+
gzipStream.pipe(writeStream);
|
|
535
|
+
writeStream = gzipStream;
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// Extract all unique keys from the data array
|
|
540
|
+
const columns = getUniqueKeys(data); // Assuming getUniqueKeys properly retrieves all keys
|
|
541
|
+
|
|
542
|
+
// Stream the header
|
|
543
|
+
writeStream.write(columns.join(',') + '\n');
|
|
544
|
+
|
|
545
|
+
// Stream each data row
|
|
546
|
+
data.forEach(item => {
|
|
547
|
+
for (const key in item) {
|
|
548
|
+
// Ensure all nested objects are properly stringified
|
|
549
|
+
if (typeof item[key] === "object") item[key] = JSON.stringify(item[key]);
|
|
550
|
+
}
|
|
551
|
+
const row = columns.map(col => item[col] ? `"${item[col].toString().replace(/"/g, '""')}"` : "").join(',');
|
|
552
|
+
writeStream.write(row + '\n');
|
|
553
|
+
});
|
|
554
|
+
|
|
555
|
+
writeStream.end();
|
|
556
|
+
writeStream.on('finish', () => {
|
|
557
|
+
resolve(filePath);
|
|
558
|
+
});
|
|
559
|
+
writeStream.on('error', reject);
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
async function streamParquet(filePath, data, options = {}) {
|
|
564
|
+
const { gzip = false } = options;
|
|
565
|
+
|
|
566
|
+
// Dynamically import hyparquet-writer
|
|
567
|
+
// @ts-ignore
|
|
568
|
+
const { parquetWriteFile, parquetWriteBuffer } = await import('hyparquet-writer');
|
|
569
|
+
|
|
570
|
+
if (data.length === 0) {
|
|
571
|
+
throw new Error('Cannot write parquet file with empty data');
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
// Extract column names and data from the input array
|
|
575
|
+
const columns = getUniqueKeys(data);
|
|
576
|
+
const columnData = columns.map(columnName => {
|
|
577
|
+
const columnValues = data.map(row => {
|
|
578
|
+
let value = row[columnName];
|
|
579
|
+
|
|
580
|
+
// Handle null/undefined values
|
|
581
|
+
if (value === null || value === undefined) {
|
|
582
|
+
return null;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
// Convert objects to strings
|
|
586
|
+
if (typeof value === 'object') {
|
|
587
|
+
value = JSON.stringify(value);
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
return value;
|
|
591
|
+
});
|
|
592
|
+
|
|
593
|
+
// Determine the type based on the first non-null value
|
|
594
|
+
let type = 'STRING'; // default
|
|
595
|
+
const firstValue = columnValues.find(v => v !== null && v !== undefined);
|
|
596
|
+
|
|
597
|
+
if (firstValue !== undefined) {
|
|
598
|
+
if (typeof firstValue === 'boolean') {
|
|
599
|
+
type = 'BOOLEAN';
|
|
600
|
+
} else if (typeof firstValue === 'number') {
|
|
601
|
+
// For parquet compatibility, convert numbers to appropriate types
|
|
602
|
+
if (Number.isInteger(firstValue)) {
|
|
603
|
+
// Use INT32 for smaller integers, convert to BigInt for INT64 if needed
|
|
604
|
+
if (firstValue >= -2147483648 && firstValue <= 2147483647) {
|
|
605
|
+
type = 'INT32';
|
|
606
|
+
} else {
|
|
607
|
+
type = 'INT64';
|
|
608
|
+
// Convert all values to BigInt for INT64
|
|
609
|
+
for (let i = 0; i < columnValues.length; i++) {
|
|
610
|
+
if (columnValues[i] !== null && columnValues[i] !== undefined) {
|
|
611
|
+
columnValues[i] = BigInt(columnValues[i]);
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
} else {
|
|
616
|
+
type = 'DOUBLE';
|
|
617
|
+
}
|
|
618
|
+
} else if (firstValue instanceof Date) {
|
|
619
|
+
type = 'TIMESTAMP';
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
return {
|
|
624
|
+
name: columnName,
|
|
625
|
+
data: columnValues,
|
|
626
|
+
type: type
|
|
627
|
+
};
|
|
628
|
+
});
|
|
629
|
+
|
|
630
|
+
if (filePath?.startsWith('gs://')) {
|
|
631
|
+
// For GCS, write to buffer first, then upload
|
|
632
|
+
// @ts-ignore
|
|
633
|
+
const arrayBuffer = parquetWriteBuffer({ columnData });
|
|
634
|
+
const { bucket, file } = parseGCSUri(filePath);
|
|
635
|
+
|
|
636
|
+
const writeStream = storage.bucket(bucket).file(file).createWriteStream({
|
|
637
|
+
gzip: gzip || true // Always gzip for GCS
|
|
638
|
+
});
|
|
639
|
+
|
|
640
|
+
return new Promise((resolve, reject) => {
|
|
641
|
+
writeStream.write(Buffer.from(arrayBuffer));
|
|
642
|
+
writeStream.end();
|
|
643
|
+
writeStream.on('finish', () => resolve(filePath));
|
|
644
|
+
writeStream.on('error', reject);
|
|
645
|
+
});
|
|
646
|
+
} else {
|
|
647
|
+
// For local files
|
|
648
|
+
let actualFilePath = filePath;
|
|
649
|
+
if (gzip && !filePath.endsWith('.gz')) {
|
|
650
|
+
actualFilePath = filePath + '.gz';
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
if (gzip) {
|
|
654
|
+
// Write to buffer then gzip to disk
|
|
655
|
+
// @ts-ignore
|
|
656
|
+
const arrayBuffer = parquetWriteBuffer({ columnData });
|
|
657
|
+
const buffer = Buffer.from(arrayBuffer);
|
|
658
|
+
const gzippedBuffer = zlib.gzipSync(buffer);
|
|
659
|
+
|
|
660
|
+
return new Promise((resolve, reject) => {
|
|
661
|
+
fs.writeFile(actualFilePath, gzippedBuffer, (err) => {
|
|
662
|
+
if (err) reject(err);
|
|
663
|
+
else resolve(actualFilePath);
|
|
664
|
+
});
|
|
665
|
+
});
|
|
666
|
+
} else {
|
|
667
|
+
// Direct write to disk
|
|
668
|
+
parquetWriteFile({
|
|
669
|
+
filename: filePath,
|
|
670
|
+
columnData
|
|
671
|
+
});
|
|
672
|
+
return Promise.resolve(filePath);
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
/*
|
|
679
|
+
----
|
|
680
|
+
WEIGHERS
|
|
681
|
+
----
|
|
682
|
+
*/
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
/**
|
|
687
|
+
* a utility function to generate a range of numbers within a given skew
|
|
688
|
+
* Skew = 0.5: The values are more concentrated towards the extremes (both ends of the range) with a noticeable dip in the middle. The distribution appears more "U" shaped. Larger sizes result in smoother distributions but maintain the overall shape.
|
|
689
|
+
*
|
|
690
|
+
* Skew = 1: This represents the default normal distribution without skew. The values are normally distributed around the mean. Larger sizes create a clearer bell-shaped curve.
|
|
691
|
+
*
|
|
692
|
+
* Skew = 2: The values are more concentrated towards the mean, with a steeper drop-off towards the extremes. The distribution appears more peaked, resembling a "sharper" bell curve. Larger sizes enhance the clarity of this peaked distribution.
|
|
693
|
+
*
|
|
694
|
+
* Size represents the size of the pool to choose from; Larger sizes result in smoother distributions but maintain the overall shape.
|
|
695
|
+
* @param {number} min
|
|
696
|
+
* @param {number} max
|
|
697
|
+
* @param {number} skew=1
|
|
698
|
+
* @param {number} size=100
|
|
699
|
+
*/
|
|
700
|
+
function weighNumRange(min, max, skew = 1, size = 50) {
|
|
701
|
+
if (size > 2000) size = 2000;
|
|
702
|
+
const mean = (max + min) / 2;
|
|
703
|
+
const sd = (max - min) / 4;
|
|
704
|
+
const array = [];
|
|
705
|
+
while (array.length < size) {
|
|
706
|
+
// const normalValue = boxMullerRandom();
|
|
707
|
+
const normalValue = optimizedBoxMuller();
|
|
708
|
+
const skewedValue = applySkew(normalValue, skew);
|
|
709
|
+
const mappedValue = mapToRange(skewedValue, mean, sd);
|
|
710
|
+
if (mappedValue >= min && mappedValue <= max) {
|
|
711
|
+
array.push(mappedValue);
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
return array;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/**
|
|
718
|
+
* arbitrarily weigh an array of values to create repeats
|
|
719
|
+
* @param {Array<any>} arr
|
|
720
|
+
*/
|
|
721
|
+
function weighArray(arr) {
|
|
722
|
+
// Calculate the upper bound based on the size of the array with added noise
|
|
723
|
+
const maxCopies = arr.length + integer(1, arr.length);
|
|
724
|
+
|
|
725
|
+
// Create an empty array to store the weighted elements
|
|
726
|
+
const weightedArray = [];
|
|
727
|
+
|
|
728
|
+
// Iterate over the input array and copy each element a random number of times
|
|
729
|
+
arr.forEach(element => {
|
|
730
|
+
let copies = integer(1, maxCopies);
|
|
731
|
+
for (let i = 0; i < copies; i++) {
|
|
732
|
+
weightedArray.push(element);
|
|
733
|
+
}
|
|
734
|
+
});
|
|
735
|
+
|
|
736
|
+
return weightedArray;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
/**
|
|
740
|
+
* Creates a function that generates a weighted array of values.
|
|
741
|
+
*
|
|
742
|
+
* @overload
|
|
743
|
+
* @param {Array<{value: string, weight: number}>} items - An array of weighted objects or an array of strings.
|
|
744
|
+
* @returns {function(): Array<string>} A function that returns a weighted array of values when called.
|
|
745
|
+
*
|
|
746
|
+
* @overload
|
|
747
|
+
* @param {Array<string>} items - An array of strings.
|
|
748
|
+
* @returns {function(): Array<string>} A function that returns a weighted array with automatically assigned random weights to each string.
|
|
749
|
+
*/
|
|
750
|
+
|
|
751
|
+
function weighChoices(items) {
|
|
752
|
+
let weightedItems;
|
|
753
|
+
|
|
754
|
+
// If items are strings, assign unique random weights
|
|
755
|
+
if (items.every(item => typeof item === 'string')) {
|
|
756
|
+
const weights = shuffleArray(range(1, items.length));
|
|
757
|
+
weightedItems = items.map((item, index) => ({
|
|
758
|
+
value: item,
|
|
759
|
+
weight: weights[index]
|
|
760
|
+
}));
|
|
761
|
+
} else {
|
|
762
|
+
weightedItems = items;
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
return function generateWeightedArray() {
|
|
766
|
+
const weightedArray = [];
|
|
767
|
+
|
|
768
|
+
// Add each value to the array the number of times specified by its weight
|
|
769
|
+
weightedItems.forEach(({ value, weight }) => {
|
|
770
|
+
if (!weight) weight = 1;
|
|
771
|
+
for (let i = 0; i < weight; i++) {
|
|
772
|
+
weightedArray.push(value);
|
|
773
|
+
}
|
|
774
|
+
});
|
|
775
|
+
|
|
776
|
+
return weightedArray;
|
|
777
|
+
};
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
/**
|
|
781
|
+
* Creates a function that generates a weighted list of items
|
|
782
|
+
* with a higher likelihood of picking a specified index and clear second and third place indices.
|
|
783
|
+
*
|
|
784
|
+
* @param {Array} items - The list of items to pick from.
|
|
785
|
+
* @param {number} [mostChosenIndex] - The index of the item to be most favored.
|
|
786
|
+
* @returns {function} - A function that returns a weighted list of items.
|
|
787
|
+
*/
|
|
788
|
+
function pickAWinner(items, mostChosenIndex) {
|
|
789
|
+
const chance = getChance();
|
|
790
|
+
|
|
791
|
+
// Ensure mostChosenIndex is within the bounds of the items array
|
|
792
|
+
if (!items) return () => { return ""; };
|
|
793
|
+
if (!items.length) return () => { return ""; };
|
|
794
|
+
if (!mostChosenIndex) mostChosenIndex = chance.integer({ min: 0, max: items.length - 1 });
|
|
795
|
+
if (mostChosenIndex >= items.length) mostChosenIndex = items.length - 1;
|
|
796
|
+
|
|
797
|
+
// Calculate second and third most chosen indices
|
|
798
|
+
const secondMostChosenIndex = (mostChosenIndex + 1) % items.length;
|
|
799
|
+
const thirdMostChosenIndex = (mostChosenIndex + 2) % items.length;
|
|
800
|
+
|
|
801
|
+
// Return a function that generates a weighted list
|
|
802
|
+
return function () {
|
|
803
|
+
const weighted = [];
|
|
804
|
+
for (let i = 0; i < 10; i++) {
|
|
805
|
+
const rand = chance.d10(); // Random number between 1 and 10
|
|
806
|
+
|
|
807
|
+
// 35% chance to favor the most chosen index
|
|
808
|
+
if (chance.bool({ likelihood: 35 })) {
|
|
809
|
+
// 50% chance to slightly alter the index
|
|
810
|
+
if (chance.bool({ likelihood: 50 })) {
|
|
811
|
+
weighted.push(items[mostChosenIndex]);
|
|
812
|
+
} else {
|
|
813
|
+
const addOrSubtract = chance.bool({ likelihood: 50 }) ? -rand : rand;
|
|
814
|
+
let newIndex = mostChosenIndex + addOrSubtract;
|
|
815
|
+
|
|
816
|
+
// Ensure newIndex is within bounds
|
|
817
|
+
if (newIndex < 0) newIndex = 0;
|
|
818
|
+
if (newIndex >= items.length) newIndex = items.length - 1;
|
|
819
|
+
weighted.push(items[newIndex]);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
// 25% chance to favor the second most chosen index
|
|
823
|
+
else if (chance.bool({ likelihood: 25 })) {
|
|
824
|
+
weighted.push(items[secondMostChosenIndex]);
|
|
825
|
+
}
|
|
826
|
+
// 15% chance to favor the third most chosen index
|
|
827
|
+
else if (chance.bool({ likelihood: 15 })) {
|
|
828
|
+
weighted.push(items[thirdMostChosenIndex]);
|
|
829
|
+
}
|
|
830
|
+
// Otherwise, pick a random item from the list
|
|
831
|
+
else {
|
|
832
|
+
weighted.push(chance.pickone(items));
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
return weighted;
|
|
836
|
+
};
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
function quickHash(str, seed = 0) {
|
|
840
|
+
let h1 = 0xdeadbeef ^ seed, h2 = 0x41c6ce57 ^ seed;
|
|
841
|
+
for (let i = 0, ch; i < str.length; i++) {
|
|
842
|
+
ch = str.charCodeAt(i);
|
|
843
|
+
h1 = Math.imul(h1 ^ ch, 2654435761);
|
|
844
|
+
h2 = Math.imul(h2 ^ ch, 1597334677);
|
|
845
|
+
}
|
|
846
|
+
h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
|
|
847
|
+
h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
|
|
848
|
+
h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
|
|
849
|
+
h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
|
|
850
|
+
|
|
851
|
+
return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString();
|
|
852
|
+
};
|
|
853
|
+
|
|
854
|
+
/*
|
|
855
|
+
----
|
|
856
|
+
SHUFFLERS
|
|
857
|
+
----
|
|
858
|
+
*/
|
|
859
|
+
|
|
860
|
+
// Function to shuffle array
|
|
861
|
+
function shuffleArray(array) {
|
|
862
|
+
const chance = getChance();
|
|
863
|
+
for (let i = array.length - 1; i > 0; i--) {
|
|
864
|
+
const j = chance.integer({ min: 0, max: i });
|
|
865
|
+
const temp = array[i];
|
|
866
|
+
array[i] = array[j];
|
|
867
|
+
array[j] = temp;
|
|
868
|
+
}
|
|
869
|
+
return array;
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
function pickRandom(array) {
|
|
873
|
+
if (!array || array.length === 0) return undefined;
|
|
874
|
+
const chance = getChance();
|
|
875
|
+
return chance.pickone(array);
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
function shuffleExceptFirst(array) {
|
|
879
|
+
if (array.length <= 1) return array;
|
|
880
|
+
const restShuffled = shuffleArray(array.slice(1));
|
|
881
|
+
return [array[0], ...restShuffled];
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
function shuffleExceptLast(array) {
|
|
885
|
+
if (array.length <= 1) return array;
|
|
886
|
+
const restShuffled = shuffleArray(array.slice(0, -1));
|
|
887
|
+
return [...restShuffled, array[array.length - 1]];
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
function fixFirstAndLast(array) {
|
|
891
|
+
if (array.length <= 2) return array;
|
|
892
|
+
const middleShuffled = shuffleArray(array.slice(1, -1));
|
|
893
|
+
return [array[0], ...middleShuffled, array[array.length - 1]];
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
function shuffleMiddle(array) {
|
|
897
|
+
if (array.length <= 2) return array;
|
|
898
|
+
const middleShuffled = shuffleArray(array.slice(1, -1));
|
|
899
|
+
return [array[0], ...middleShuffled, array[array.length - 1]];
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
function shuffleOutside(array) {
|
|
903
|
+
if (array.length <= 2) return array;
|
|
904
|
+
const middleFixed = array.slice(1, -1);
|
|
905
|
+
const outsideShuffled = shuffleArray([array[0], array[array.length - 1]]);
|
|
906
|
+
return [outsideShuffled[0], ...middleFixed, outsideShuffled[1]];
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
/**
|
|
910
|
+
* given a funnel, shuffle the events in the sequence with random events
|
|
911
|
+
* @param {EventConfig[]} funnel
|
|
912
|
+
* @param {EventConfig[]} possibles
|
|
913
|
+
*/
|
|
914
|
+
function interruptArray(funnel, possibles, percent = 50) {
|
|
915
|
+
if (!Array.isArray(funnel)) return funnel;
|
|
916
|
+
if (!Array.isArray(possibles)) return funnel;
|
|
917
|
+
if (!funnel.length) return funnel;
|
|
918
|
+
if (!possibles.length) return funnel;
|
|
919
|
+
const ignorePositions = [0, funnel.length - 1];
|
|
920
|
+
const chance = getChance();
|
|
921
|
+
loopSteps: for (const [index, event] of funnel.entries()) {
|
|
922
|
+
if (ignorePositions.includes(index)) continue loopSteps;
|
|
923
|
+
if (chance.bool({ likelihood: percent })) {
|
|
924
|
+
funnel[index] = chance.pickone(possibles);
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
return funnel;
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
/*
|
|
932
|
+
----
|
|
933
|
+
VALIDATORS
|
|
934
|
+
----
|
|
935
|
+
*/
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
/**
|
|
939
|
+
* @param {EventConfig[] | string[]} events
|
|
940
|
+
*/
|
|
941
|
+
function validateEventConfig(events) {
|
|
942
|
+
if (!Array.isArray(events)) throw new Error("events must be an array");
|
|
943
|
+
const cleanEventConfig = [];
|
|
944
|
+
for (const event of events) {
|
|
945
|
+
if (typeof event === "string") {
|
|
946
|
+
/** @type {EventConfig} */
|
|
947
|
+
const eventTemplate = {
|
|
948
|
+
event,
|
|
949
|
+
isFirstEvent: false,
|
|
950
|
+
properties: {},
|
|
951
|
+
weight: integer(1, 5)
|
|
952
|
+
};
|
|
953
|
+
cleanEventConfig.push(eventTemplate);
|
|
954
|
+
}
|
|
955
|
+
if (typeof event === "object") {
|
|
956
|
+
cleanEventConfig.push(event);
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
return cleanEventConfig;
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
function validTime(chosenTime, earliestTime, latestTime) {
|
|
963
|
+
if (!earliestTime) earliestTime = global.FIXED_BEGIN ? global.FIXED_BEGIN : dayjs().subtract(30, 'd').unix(); // 30 days ago
|
|
964
|
+
if (!latestTime) latestTime = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
965
|
+
|
|
966
|
+
if (typeof chosenTime === 'number') {
|
|
967
|
+
if (chosenTime > 0) {
|
|
968
|
+
if (chosenTime > earliestTime) {
|
|
969
|
+
if (chosenTime < (latestTime)) {
|
|
970
|
+
return true;
|
|
971
|
+
}
|
|
972
|
+
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
return false;
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
function validEvent(row) {
|
|
980
|
+
if (!row) return false;
|
|
981
|
+
if (!row.event) return false;
|
|
982
|
+
if (!row.time) return false;
|
|
983
|
+
if (!row.device_id && !row.user_id) return false;
|
|
984
|
+
if (!row.insert_id) return false;
|
|
985
|
+
// if (!row.source) return false;
|
|
986
|
+
if (typeof row.time !== 'string') return false;
|
|
987
|
+
return true;
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
|
|
991
|
+
/*
|
|
992
|
+
----
|
|
993
|
+
META
|
|
994
|
+
----
|
|
995
|
+
*/
|
|
996
|
+
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
/**
|
|
1000
|
+
* @param {Config} config
|
|
1001
|
+
*/
|
|
1002
|
+
function buildFileNames(config) {
|
|
1003
|
+
const { format = "csv", groupKeys = [], lookupTables = [] } = config;
|
|
1004
|
+
let extension = "";
|
|
1005
|
+
extension = format === "csv" ? "csv" : "json";
|
|
1006
|
+
// const current = dayjs.utc().format("MM-DD-HH");
|
|
1007
|
+
let simName = config.name;
|
|
1008
|
+
let writeDir = typeof config.writeToDisk === 'string' ? config.writeToDisk : "./";
|
|
1009
|
+
if (config.writeToDisk) {
|
|
1010
|
+
const dataFolder = path.resolve("./data");
|
|
1011
|
+
if (existsSync(dataFolder)) writeDir = dataFolder;
|
|
1012
|
+
else writeDir = path.resolve("./");
|
|
1013
|
+
}
|
|
1014
|
+
if (typeof writeDir !== "string") throw new Error("writeDir must be a string");
|
|
1015
|
+
if (typeof simName !== "string") throw new Error("simName must be a string");
|
|
1016
|
+
|
|
1017
|
+
const writePaths = {
|
|
1018
|
+
eventFiles: [path.join(writeDir, `${simName}-EVENTS.${extension}`)],
|
|
1019
|
+
userFiles: [path.join(writeDir, `${simName}-USERS.${extension}`)],
|
|
1020
|
+
adSpendFiles: [],
|
|
1021
|
+
scdFiles: [],
|
|
1022
|
+
mirrorFiles: [],
|
|
1023
|
+
groupFiles: [],
|
|
1024
|
+
lookupFiles: [],
|
|
1025
|
+
folder: writeDir,
|
|
1026
|
+
};
|
|
1027
|
+
//add ad spend files
|
|
1028
|
+
if (config?.hasAdSpend) {
|
|
1029
|
+
writePaths.adSpendFiles.push(path.join(writeDir, `${simName}-AD-SPEND.${extension}`));
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
//add SCD files
|
|
1033
|
+
const scdKeys = Object.keys(config?.scdProps || {});
|
|
1034
|
+
for (const key of scdKeys) {
|
|
1035
|
+
writePaths.scdFiles.push(
|
|
1036
|
+
path.join(writeDir, `${simName}-${key}-SCD.${extension}`)
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
//add group files
|
|
1041
|
+
for (const groupPair of groupKeys) {
|
|
1042
|
+
const groupKey = groupPair[0];
|
|
1043
|
+
|
|
1044
|
+
writePaths.groupFiles.push(
|
|
1045
|
+
path.join(writeDir, `${simName}-${groupKey}-GROUP.${extension}`)
|
|
1046
|
+
);
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
//add lookup files
|
|
1050
|
+
for (const lookupTable of lookupTables) {
|
|
1051
|
+
const { key } = lookupTable;
|
|
1052
|
+
writePaths.lookupFiles.push(
|
|
1053
|
+
//lookups are always CSVs
|
|
1054
|
+
path.join(writeDir, `${simName}-${key}-LOOKUP.csv`)
|
|
1055
|
+
);
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
//add mirror files
|
|
1059
|
+
const mirrorProps = config?.mirrorProps || {};
|
|
1060
|
+
if (Object.keys(mirrorProps).length) {
|
|
1061
|
+
writePaths.mirrorFiles.push(
|
|
1062
|
+
path.join(writeDir, `${simName}-MIRROR.${extension}`)
|
|
1063
|
+
);
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
return writePaths;
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
/**
|
|
1070
|
+
* Human-readable byte size
|
|
1071
|
+
* @param {number} bytes
|
|
1072
|
+
* @param {number} dp - decimal places
|
|
1073
|
+
* @param {boolean} si - use SI units
|
|
1074
|
+
* @returns {string}
|
|
1075
|
+
*/
|
|
1076
|
+
function bytesHuman(bytes, dp = 2, si = true) {
|
|
1077
|
+
const thresh = si ? 1000 : 1024;
|
|
1078
|
+
if (Math.abs(bytes) < thresh) {
|
|
1079
|
+
return bytes + ' B';
|
|
1080
|
+
}
|
|
1081
|
+
const units = si ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
|
|
1082
|
+
let u = -1;
|
|
1083
|
+
const r = 10 ** dp;
|
|
1084
|
+
do {
|
|
1085
|
+
bytes /= thresh;
|
|
1086
|
+
++u;
|
|
1087
|
+
} while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1);
|
|
1088
|
+
return bytes.toFixed(dp) + ' ' + units[u];
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
/**
|
|
1092
|
+
* Format milliseconds as HH:MM:SS
|
|
1093
|
+
* @param {number} ms - Milliseconds
|
|
1094
|
+
* @returns {string} Formatted duration string
|
|
1095
|
+
*/
|
|
1096
|
+
function formatDuration(ms) {
|
|
1097
|
+
const seconds = Math.floor(ms / 1000);
|
|
1098
|
+
const hours = Math.floor(seconds / 3600);
|
|
1099
|
+
const minutes = Math.floor((seconds % 3600) / 60);
|
|
1100
|
+
const secs = seconds % 60;
|
|
1101
|
+
return `${String(hours).padStart(2, '0')}:${String(minutes).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
/**
|
|
1105
|
+
* @param {[string, string | number][]} arrayOfArrays
|
|
1106
|
+
*/
|
|
1107
|
+
function progress(arrayOfArrays) {
|
|
1108
|
+
const terminalWidth = process.stdout.columns || 120;
|
|
1109
|
+
|
|
1110
|
+
// Clear the entire line
|
|
1111
|
+
readline.cursorTo(process.stdout, 0);
|
|
1112
|
+
readline.clearLine(process.stdout, 0);
|
|
1113
|
+
|
|
1114
|
+
// Build message with better formatting
|
|
1115
|
+
const items = arrayOfArrays.map(([thing, p]) => {
|
|
1116
|
+
return `${thing}: ${comma(p)}`;
|
|
1117
|
+
});
|
|
1118
|
+
|
|
1119
|
+
const message = items.join(' โ ');
|
|
1120
|
+
|
|
1121
|
+
// Ensure we don't exceed terminal width
|
|
1122
|
+
const finalMessage = message.length > terminalWidth
|
|
1123
|
+
? message.substring(0, terminalWidth - 3) + '...'
|
|
1124
|
+
: message.padEnd(terminalWidth, ' ');
|
|
1125
|
+
|
|
1126
|
+
process.stdout.write(finalMessage);
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
function getUniqueKeys(data) {
|
|
1130
|
+
const keysSet = new Set();
|
|
1131
|
+
data.forEach(item => {
|
|
1132
|
+
Object.keys(item).forEach(key => keysSet.add(key));
|
|
1133
|
+
});
|
|
1134
|
+
return Array.from(keysSet);
|
|
1135
|
+
};
|
|
1136
|
+
|
|
1137
|
+
|
|
1138
|
+
/*
|
|
1139
|
+
----
|
|
1140
|
+
CORE
|
|
1141
|
+
----
|
|
1142
|
+
*/
|
|
1143
|
+
|
|
1144
|
+
//the function which generates $distinct_id + $anonymous_ids, $session_ids, and created, skewing towards the present
|
|
1145
|
+
function generateUser(user_id, opts, amplitude = 1, frequency = 1, skew = 1) {
|
|
1146
|
+
const chance = getChance();
|
|
1147
|
+
const { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds } = opts;
|
|
1148
|
+
// Uniformly distributed `u`, then skew applied
|
|
1149
|
+
let u = Math.pow(chance.random(), skew);
|
|
1150
|
+
|
|
1151
|
+
// Sine function for a smoother curve
|
|
1152
|
+
const sineValue = (Math.sin(u * Math.PI * frequency - Math.PI / 2) * amplitude + 1) / 2;
|
|
1153
|
+
|
|
1154
|
+
// Scale the sineValue to the range of days
|
|
1155
|
+
let daysAgoBorn = Math.round(sineValue * (numDays - 1)) + 1;
|
|
1156
|
+
|
|
1157
|
+
// Clamp values to ensure they are within the desired range
|
|
1158
|
+
daysAgoBorn = Math.min(daysAgoBorn, numDays);
|
|
1159
|
+
const props = person(user_id, daysAgoBorn, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds);
|
|
1160
|
+
|
|
1161
|
+
const user = {
|
|
1162
|
+
distinct_id: user_id,
|
|
1163
|
+
...props,
|
|
1164
|
+
};
|
|
1165
|
+
|
|
1166
|
+
|
|
1167
|
+
return user;
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
let soupHits = 0;
|
|
1171
|
+
/**
|
|
1172
|
+
* build sign waves basically
|
|
1173
|
+
* @param {number} [earliestTime]
|
|
1174
|
+
* @param {number} [latestTime]
|
|
1175
|
+
* @param {number} [peaks=5]
|
|
1176
|
+
*/
|
|
1177
|
+
/**
|
|
1178
|
+
* Generates a timestamp within a time range using clustered normal distributions.
|
|
1179
|
+
* Divides the range into `peaks` chunks, picks one randomly, then samples within it.
|
|
1180
|
+
* Returns unix seconds (not ISO string) for performance โ caller converts once.
|
|
1181
|
+
*/
|
|
1182
|
+
// Default day-of-week weights (0=Sun, 1=Mon, ..., 6=Sat) โ derived from real Mixpanel data
|
|
1183
|
+
const DEFAULT_DOW_WEIGHTS = [0.637, 1.0, 0.999, 0.998, 0.966, 0.802, 0.528];
|
|
1184
|
+
|
|
1185
|
+
// Default hour-of-day weights (0=midnight, ..., 23=11pm UTC) โ derived from real Mixpanel data
|
|
1186
|
+
const DEFAULT_HOD_WEIGHTS = [
|
|
1187
|
+
0.949, 0.992, 0.998, 0.946, 0.895, 0.938, 1.0, 0.997,
|
|
1188
|
+
0.938, 0.894, 0.827, 0.786, 0.726, 0.699, 0.688, 0.643,
|
|
1189
|
+
0.584, 0.574, 0.554, 0.576, 0.604, 0.655, 0.722, 0.816
|
|
1190
|
+
];
|
|
1191
|
+
|
|
1192
|
+
function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0, dayOfWeekWeights = DEFAULT_DOW_WEIGHTS, hourOfDayWeights = DEFAULT_HOD_WEIGHTS, timeShiftSeconds = 0) {
|
|
1193
|
+
if (!earliestTime) earliestTime = global.FIXED_BEGIN ? global.FIXED_BEGIN : dayjs().subtract(30, 'd').unix();
|
|
1194
|
+
if (!latestTime) latestTime = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
1195
|
+
const chance = getChance();
|
|
1196
|
+
let totalRange = latestTime - earliestTime;
|
|
1197
|
+
if (totalRange <= 0 || earliestTime > latestTime) {
|
|
1198
|
+
const temp = latestTime;
|
|
1199
|
+
latestTime = earliestTime;
|
|
1200
|
+
earliestTime = temp;
|
|
1201
|
+
totalRange = latestTime - earliestTime;
|
|
1202
|
+
}
|
|
1203
|
+
const chunkSize = totalRange / peaks;
|
|
1204
|
+
|
|
1205
|
+
// Phase 1: Gaussian chunk sampling (macro trend across the time range)
|
|
1206
|
+
const peakIndex = integer(0, peaks - 1);
|
|
1207
|
+
const chunkStart = earliestTime + peakIndex * chunkSize;
|
|
1208
|
+
const chunkEnd = chunkStart + chunkSize;
|
|
1209
|
+
const chunkMid = (chunkStart + chunkEnd) / 2;
|
|
1210
|
+
const maxDeviation = chunkSize / deviation;
|
|
1211
|
+
const offset = chance.normal({ mean: mean, dev: maxDeviation });
|
|
1212
|
+
const proposedTime = chunkMid + offset;
|
|
1213
|
+
const clampedTime = Math.max(chunkStart, Math.min(chunkEnd, proposedTime));
|
|
1214
|
+
let candidate = Math.max(earliestTime, Math.min(latestTime, clampedTime));
|
|
1215
|
+
|
|
1216
|
+
// Phase 2: DOW accept/reject โ retry if day-of-week doesn't pass weight check
|
|
1217
|
+
if (dayOfWeekWeights) {
|
|
1218
|
+
for (let attempt = 0; attempt < 50; attempt++) {
|
|
1219
|
+
const dow = new Date((candidate + timeShiftSeconds) * 1000).getUTCDay();
|
|
1220
|
+
if (chance.random() < dayOfWeekWeights[dow]) break;
|
|
1221
|
+
// Rejected โ resample from Gaussian chunks
|
|
1222
|
+
const pi = integer(0, peaks - 1);
|
|
1223
|
+
const cs = earliestTime + pi * chunkSize;
|
|
1224
|
+
const ce = cs + chunkSize;
|
|
1225
|
+
const cm = (cs + ce) / 2;
|
|
1226
|
+
const md = chunkSize / deviation;
|
|
1227
|
+
const off = chance.normal({ mean: mean, dev: md });
|
|
1228
|
+
const pt = cm + off;
|
|
1229
|
+
candidate = Math.max(earliestTime, Math.min(latestTime, Math.max(cs, Math.min(ce, pt))));
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
// Phase 3: Redistribute hour-of-day (changes only hour within same day)
|
|
1234
|
+
if (hourOfDayWeights) {
|
|
1235
|
+
const shifted = candidate + timeShiftSeconds;
|
|
1236
|
+
const d = new Date(shifted * 1000);
|
|
1237
|
+
const currentMinute = d.getUTCMinutes();
|
|
1238
|
+
const currentSecond = d.getUTCSeconds();
|
|
1239
|
+
|
|
1240
|
+
const totalHodWeight = hourOfDayWeights.reduce((s, w) => s + w, 0);
|
|
1241
|
+
let roll = chance.random() * totalHodWeight;
|
|
1242
|
+
let newHour = 0;
|
|
1243
|
+
for (let h = 0; h < 24; h++) {
|
|
1244
|
+
roll -= hourOfDayWeights[h];
|
|
1245
|
+
if (roll <= 0) { newHour = h; break; }
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
const dayStartShifted = Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()) / 1000;
|
|
1249
|
+
const newShifted = dayStartShifted + newHour * 3600 + currentMinute * 60 + currentSecond;
|
|
1250
|
+
candidate = newShifted - timeShiftSeconds;
|
|
1251
|
+
candidate = Math.max(earliestTime, Math.min(latestTime, candidate));
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
soupHits++;
|
|
1255
|
+
return candidate;
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
|
|
1259
|
+
/**
|
|
1260
|
+
* @param {string} userId
|
|
1261
|
+
* @param {number} bornDaysAgo=30
|
|
1262
|
+
* @param {boolean} isAnonymous
|
|
1263
|
+
* @param {boolean} hasAvatar
|
|
1264
|
+
* @param {boolean} hasAnonIds
|
|
1265
|
+
* @param {boolean} hasSessionIds
|
|
1266
|
+
* @return {Person}
|
|
1267
|
+
*/
|
|
1268
|
+
function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false, hasAnonIds = false, hasSessionIds = false) {
|
|
1269
|
+
const chance = getChance();
|
|
1270
|
+
//names and photos
|
|
1271
|
+
const l = chance.letter.bind(chance);
|
|
1272
|
+
let gender = chance.pickone(['male', 'female']);
|
|
1273
|
+
if (!gender) gender = "female";
|
|
1274
|
+
let first = chance.first({ gender });
|
|
1275
|
+
let last = chance.last();
|
|
1276
|
+
let name = `${first} ${last}`;
|
|
1277
|
+
let email = `${first[0]}.${last}@${choose(domainPrefix)}.${choose(domainSuffix)}`;
|
|
1278
|
+
let avatarPrefix = `https://randomuser.me/api/portraits`;
|
|
1279
|
+
let randomAvatarNumber = integer(1, 99);
|
|
1280
|
+
let avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
|
|
1281
|
+
let avatar = avatarPrefix + avPath;
|
|
1282
|
+
let created = dayjs().subtract(bornDaysAgo, 'day').format('YYYY-MM-DD');
|
|
1283
|
+
|
|
1284
|
+
|
|
1285
|
+
// const created = date(bornDaysAgo, true)();
|
|
1286
|
+
|
|
1287
|
+
|
|
1288
|
+
/** @type {Person} */
|
|
1289
|
+
const user = {
|
|
1290
|
+
distinct_id: userId,
|
|
1291
|
+
name,
|
|
1292
|
+
email,
|
|
1293
|
+
avatar,
|
|
1294
|
+
created,
|
|
1295
|
+
anonymousIds: [],
|
|
1296
|
+
sessionIds: []
|
|
1297
|
+
};
|
|
1298
|
+
|
|
1299
|
+
if (isAnonymous) {
|
|
1300
|
+
user.name = "Anonymous User";
|
|
1301
|
+
user.email = l() + l() + `*`.repeat(integer(3, 6)) + l() + `@` + l() + `*`.repeat(integer(3, 6)) + l() + `.` + choose(domainSuffix);
|
|
1302
|
+
delete user.avatar;
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
if (!hasAvatar) delete user.avatar;
|
|
1306
|
+
|
|
1307
|
+
//anon Ids
|
|
1308
|
+
if (hasAnonIds) {
|
|
1309
|
+
const clusterSize = integer(2, 10);
|
|
1310
|
+
for (let i = 0; i < clusterSize; i++) {
|
|
1311
|
+
const anonId = uid(42);
|
|
1312
|
+
user.anonymousIds.push(anonId);
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
if (!hasAnonIds) delete user.anonymousIds;
|
|
1317
|
+
|
|
1318
|
+
//session Ids
|
|
1319
|
+
if (hasSessionIds) {
|
|
1320
|
+
const sessionSize = integer(5, 30);
|
|
1321
|
+
for (let i = 0; i < sessionSize; i++) {
|
|
1322
|
+
const sessionId = [uid(5), uid(5), uid(5), uid(5)].join("-");
|
|
1323
|
+
user.sessionIds.push(sessionId);
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
if (!hasSessionIds) delete user.sessionIds;
|
|
1328
|
+
|
|
1329
|
+
return user;
|
|
1330
|
+
};
|
|
1331
|
+
|
|
1332
|
+
|
|
1333
|
+
function wrapFunc(obj, func, recursion = 0, parentKey = null, grandParentKey = null, whitelist = [
|
|
1334
|
+
"events",
|
|
1335
|
+
"superProps",
|
|
1336
|
+
"userProps",
|
|
1337
|
+
"scdProps",
|
|
1338
|
+
"mirrorProps",
|
|
1339
|
+
"groupEvents",
|
|
1340
|
+
"groupProps"
|
|
1341
|
+
]) {
|
|
1342
|
+
if (recursion === 0) {
|
|
1343
|
+
// Only process top-level keys in the whitelist
|
|
1344
|
+
for (const key in obj) {
|
|
1345
|
+
if (whitelist.includes(key)) {
|
|
1346
|
+
obj[key] = wrapFunc(obj[key], func, recursion + 1, key, null, whitelist);
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
} else {
|
|
1350
|
+
if (Array.isArray(obj) && grandParentKey === 'properties') {
|
|
1351
|
+
return func(obj);
|
|
1352
|
+
} else if (typeof obj === 'object' && obj !== null) {
|
|
1353
|
+
for (const key in obj) {
|
|
1354
|
+
if (obj.hasOwnProperty(key)) {
|
|
1355
|
+
obj[key] = wrapFunc(obj[key], func, recursion + 1, key, parentKey, whitelist);
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
return obj;
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
/**
|
|
1364
|
+
* makes a random-sized array of emojis
|
|
1365
|
+
* @param {number} max=10
|
|
1366
|
+
* @param {boolean} array=false
|
|
1367
|
+
*/
|
|
1368
|
+
function generateEmoji(max = 10, array = false) {
|
|
1369
|
+
const chance = getChance();
|
|
1370
|
+
return function () {
|
|
1371
|
+
const emojis = ['๐', '๐', '๐', '๐', '๐', '๐', '๐ก', '๐ฑ', '๐ญ', '๐ด', '๐คข', '๐ค ', '๐คก', '๐ฝ', '๐ป', '๐ฉ', '๐บ', '๐น', '๐พ', '๐ค', '๐ค', '๐ค', '๐ค', '๐ค', '๐ค', '๐', '๐', '๐', '๐', '๐', '๐', '๐ก', '๐ฑ', '๐ญ', '๐ด', '๐คข', '๐ค ', '๐คก', '๐ฝ', '๐ป', '๐ฉ', '๐บ', '๐น', '๐พ', '๐ค', '๐ค', '๐ค', '๐ค', '๐ค', '๐ค', '๐', '๐ฟ', '๐ฆ', '๐ง', '๐จ', '๐ฉ', '๐ด', '๐ต', '๐ถ', '๐ง', '๐ฎ', '๐ท', '๐', '๐ต', '๐ฉโโ๏ธ', '๐จโโ๏ธ', '๐ฉโ๐พ', '๐จโ๐พ', '๐ฉโ๐ณ', '๐จโ๐ณ', '๐ฉโ๐', '๐จโ๐', '๐ฉโ๐ค', '๐จโ๐ค', '๐ฉโ๐ซ', '๐จโ๐ซ', '๐ฉโ๐ญ', '๐จโ๐ญ', '๐ฉโ๐ป', '๐จโ๐ป', '๐ฉโ๐ผ', '๐จโ๐ผ', '๐ฉโ๐ง', '๐จโ๐ง', '๐ฉโ๐ฌ', '๐จโ๐ฌ', '๐ฉโ๐จ', '๐จโ๐จ', '๐ฉโ๐', '๐จโ๐', '๐ฉโโ๏ธ', '๐จโโ๏ธ', '๐ฉโ๐', '๐จโ๐', '๐ฉโโ๏ธ', '๐จโโ๏ธ', '๐คถ', '๐
', '๐ธ', '๐คด', '๐ฐ', '๐คต', '๐ผ', '๐คฐ', '๐', '๐', '๐
', '๐', '๐', '๐คฆ', '๐คท', '๐', '๐', '๐', '๐', '๐ด', '๐', '๐บ', '๐ถ', '๐', '๐คฒ', '๐', '๐', '๐', '๐ค', '๐', '๐', '๐', 'โ', '๐ค', '๐ค', '๐ค', 'โ๏ธ', '๐ค', '๐ค', '๐', '๐', '๐', '๐', '๐', 'โ๏ธ', 'โ', '๐ค', '๐', '๐', '๐', '๐ค', '๐ช', '๐', 'โ๏ธ', '๐คณ', '๐
', '๐', '๐', '๐ฃ', '๐', '๐', '๐ง ', '๐
', '๐', '๐', '๐', '๐ถ', '๐', '๐', '๐', '๐งฃ', '๐งค', '๐งฅ', '๐งฆ', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐ ', '๐ก', '๐ข', '๐', '๐', '๐ฉ', '๐', '๐งข', 'โ', '๐ฟ', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐ข', '๐ฃ', '๐ฏ', '๐', '๐', '๐ผ', '๐ต', '๐ถ', '๐', '๐', '๐', '๐ค', '๐ง', '๐ป', '๐ท', '๐ธ', '๐น', '๐บ', '๐ป', '๐ฅ', '๐ฑ', '๐ฒ', '๐ป', '๐ฅ', '๐จ', '๐ฑ', '๐ฒ', '๐น', '๐', '๐ฝ', '๐พ', '๐ฟ', '๐', '๐ผ', '๐ท', '๐ธ', '๐น', '๐ฅ', '๐ฝ', '๐', '๐', 'โ๏ธ', '๐', '๐ ', '๐บ', '๐ป', '๐', '๐ก', '๐', '๐', '๐ฌ', '๐ญ', '๐ก', '๐ก', '๐ฆ', '๐ฎ', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐ฐ', '๐', '๐', '๐', '๐ท', '๐ฐ', '๐ด', '๐ต', '๐ถ', '๐ท', '๐ธ', '๐ณ', '๐งพ', '๐น', '๐ฑ', '๐ฒ', 'โ๏ธ', '๐ง', '๐จ', '๐ฉ', '๐ค', '๐ฅ', '๐ฆ', '๐ซ', '๐ช', '๐ฌ', '๐ญ', '๐ฎ', '๐ณ', 'โ๏ธ', 'โ๏ธ', '๐', '๐', '๐', '๐', '๐', '๐ผ', '๐', '๐', '๐', '๐
', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', 'โ๏ธ', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐จ', 'โ', 'โ', '๐ ', '๐ก', 'โ๏ธ', '๐ซ', '๐น', '๐ก', '๐ง', '๐ฉ', 'โ๏ธ', '๐', 'โ๏ธ', '๐', 'โ', '๐งฐ', '๐งฒ', 'โ๏ธ', '๐งช', '๐งซ', '๐งฌ', '๐ฌ', '๐ญ', '๐ก', '๐', '๐', '๐', '๐', '๐ช', '๐ฝ', '๐ฟ', '๐', '๐งด', '๐งท', '๐งน', '๐งบ', '๐งป', '๐งผ', '๐งฝ', '๐งฏ', '๐ฌ', 'โฐ๏ธ', 'โฑ๏ธ', '๐ฟ', '๐บ', '๐งฑ', '๐', '๐', '๐', '๐', '๐', '๐', '๐', '๐ฎ', '๐', '๐งง', 'โ๏ธ', '๐ฉ', '๐จ', '๐ง'];
|
|
1372
|
+
let num = integer(1, max);
|
|
1373
|
+
let arr = [];
|
|
1374
|
+
for (let i = 0; i < num; i++) {
|
|
1375
|
+
arr.push(chance.pickone(emojis));
|
|
1376
|
+
}
|
|
1377
|
+
if (array) return arr;
|
|
1378
|
+
if (!array) return arr.join(', ');
|
|
1379
|
+
return "๐คท";
|
|
1380
|
+
};
|
|
1381
|
+
};
|
|
1382
|
+
|
|
1383
|
+
function deepClone(thing, opts) {
|
|
1384
|
+
// Handle primitives first (most common case)
|
|
1385
|
+
if (thing === null || thing === undefined) return thing;
|
|
1386
|
+
|
|
1387
|
+
const type = typeof thing;
|
|
1388
|
+
if (type !== 'object' && type !== 'function') {
|
|
1389
|
+
if (type === 'symbol') {
|
|
1390
|
+
return Symbol(thing.description);
|
|
1391
|
+
}
|
|
1392
|
+
return thing;
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
// Handle arrays (common case)
|
|
1396
|
+
if (Array.isArray(thing)) {
|
|
1397
|
+
const result = new Array(thing.length);
|
|
1398
|
+
for (let i = 0; i < thing.length; i++) {
|
|
1399
|
+
result[i] = deepClone(thing[i], opts);
|
|
1400
|
+
}
|
|
1401
|
+
return result;
|
|
1402
|
+
}
|
|
1403
|
+
|
|
1404
|
+
// Handle other object types
|
|
1405
|
+
if (thing instanceof Date) return new Date(thing.getTime());
|
|
1406
|
+
if (thing instanceof RegExp) return new RegExp(thing.source, thing.flags);
|
|
1407
|
+
if (thing instanceof Function) {
|
|
1408
|
+
return opts && opts.newFns ?
|
|
1409
|
+
new Function('return ' + thing.toString())() :
|
|
1410
|
+
thing;
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
// Handle plain objects
|
|
1414
|
+
if (thing.constructor === Object) {
|
|
1415
|
+
const newObject = {};
|
|
1416
|
+
const keys = Object.keys(thing);
|
|
1417
|
+
for (let i = 0; i < keys.length; i++) {
|
|
1418
|
+
const key = keys[i];
|
|
1419
|
+
newObject[key] = deepClone(thing[key], opts);
|
|
1420
|
+
}
|
|
1421
|
+
return newObject;
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
// Handle other object types
|
|
1425
|
+
try {
|
|
1426
|
+
return new thing.constructor(thing);
|
|
1427
|
+
} catch (e) {
|
|
1428
|
+
// Fallback for objects that can't be constructed this way
|
|
1429
|
+
const newObject = Object.create(Object.getPrototypeOf(thing));
|
|
1430
|
+
const keys = Object.keys(thing);
|
|
1431
|
+
for (let i = 0; i < keys.length; i++) {
|
|
1432
|
+
const key = keys[i];
|
|
1433
|
+
newObject[key] = deepClone(thing[key], opts);
|
|
1434
|
+
}
|
|
1435
|
+
return newObject;
|
|
1436
|
+
}
|
|
1437
|
+
};
|
|
1438
|
+
|
|
1439
|
+
|
|
1440
|
+
export {
|
|
1441
|
+
pick,
|
|
1442
|
+
date,
|
|
1443
|
+
dates,
|
|
1444
|
+
day,
|
|
1445
|
+
choose,
|
|
1446
|
+
pickRandom,
|
|
1447
|
+
exhaust,
|
|
1448
|
+
integer,
|
|
1449
|
+
TimeSoup,
|
|
1450
|
+
companyName,
|
|
1451
|
+
generateEmoji,
|
|
1452
|
+
hasSameKeys,
|
|
1453
|
+
deepClone,
|
|
1454
|
+
initChance,
|
|
1455
|
+
getChance,
|
|
1456
|
+
decimal,
|
|
1457
|
+
validTime,
|
|
1458
|
+
validEvent,
|
|
1459
|
+
|
|
1460
|
+
boxMullerRandom,
|
|
1461
|
+
applySkew,
|
|
1462
|
+
mapToRange,
|
|
1463
|
+
weighNumRange,
|
|
1464
|
+
progress,
|
|
1465
|
+
range,
|
|
1466
|
+
getUniqueKeys,
|
|
1467
|
+
person,
|
|
1468
|
+
pickAWinner,
|
|
1469
|
+
quickHash,
|
|
1470
|
+
weighArray,
|
|
1471
|
+
validateEventConfig,
|
|
1472
|
+
shuffleArray,
|
|
1473
|
+
shuffleExceptFirst,
|
|
1474
|
+
shuffleExceptLast,
|
|
1475
|
+
fixFirstAndLast,
|
|
1476
|
+
shuffleMiddle,
|
|
1477
|
+
shuffleOutside,
|
|
1478
|
+
interruptArray,
|
|
1479
|
+
generateUser,
|
|
1480
|
+
optimizedBoxMuller,
|
|
1481
|
+
buildFileNames,
|
|
1482
|
+
streamJSON,
|
|
1483
|
+
streamCSV,
|
|
1484
|
+
streamParquet,
|
|
1485
|
+
datesBetween,
|
|
1486
|
+
weighChoices,
|
|
1487
|
+
wrapFunc,
|
|
1488
|
+
bytesHuman,
|
|
1489
|
+
formatDuration,
|
|
1490
|
+
};
|