@ak--47/dungeon-master 1.2.3 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +51 -13
- package/dungeons/technical/ad-spend.js +2 -2
- package/dungeons/technical/anonymous-users.js +2 -2
- package/dungeons/technical/array-of-object-lookup.js +2 -4
- package/dungeons/technical/experiments.js +2 -2
- package/dungeons/technical/foobar.js +2 -2
- package/dungeons/technical/group-analytics.js +2 -2
- package/dungeons/technical/mirror-strategies.js +2 -2
- package/dungeons/technical/nested-objects.js +2 -2
- package/dungeons/technical/retention-cadence.js +2 -3
- package/dungeons/technical/sanity.js +2 -2
- package/dungeons/technical/scale-test.js +2 -2
- package/dungeons/technical/scd.js +2 -2
- package/dungeons/technical/simple.js +5 -6
- package/dungeons/technical/simplest-schema.json +5 -0
- package/dungeons/technical/simplest.js +2 -2
- package/dungeons/technical/text-generation.js +3 -3
- package/dungeons/vertical/ai-platform.js +858 -0
- package/dungeons/vertical/community.js +84 -40
- package/dungeons/vertical/crypto.js +830 -0
- package/dungeons/vertical/dating.js +744 -0
- package/dungeons/vertical/devtools.js +175 -69
- package/dungeons/vertical/ecommerce.js +242 -94
- package/dungeons/vertical/education.js +330 -313
- package/dungeons/vertical/fintech.js +442 -313
- package/dungeons/vertical/fitness.js +143 -61
- package/dungeons/vertical/food-delivery.js +327 -353
- package/dungeons/vertical/gaming.js +912 -382
- package/dungeons/vertical/healthcare.js +142 -63
- package/dungeons/vertical/insurance-application.js +170 -76
- package/dungeons/vertical/logistics.js +115 -20
- package/dungeons/vertical/marketplace.js +152 -58
- package/dungeons/vertical/media.js +248 -384
- package/dungeons/vertical/real-estate.js +781 -0
- package/dungeons/vertical/sass.js +255 -266
- package/dungeons/vertical/social.js +264 -206
- package/dungeons/vertical/travel.js +117 -41
- package/index.js +17 -17
- package/lib/core/config-validator.js +159 -31
- package/lib/core/context.js +10 -24
- package/lib/core/storage.js +6 -1
- package/lib/generators/events.js +11 -14
- package/lib/generators/funnels.js +12 -4
- package/lib/generators/mirror.js +3 -2
- package/lib/generators/product-names.js +1 -1
- package/lib/generators/scd.js +2 -1
- package/lib/generators/text.js +1 -1
- package/lib/orchestrators/user-loop.js +81 -47
- package/lib/templates/macro-presets.js +111 -0
- package/lib/templates/soup-presets.js +19 -36
- package/lib/utils/utils.js +71 -39
- package/package.json +8 -2
- package/scripts/smoke-test-all.mjs +162 -0
- package/scripts/verify-runner.mjs +72 -24
- package/types.d.ts +251 -51
- package/dungeons/technical/ad-spend-schema.json +0 -128
- package/dungeons/technical/anonymous-users-schema.json +0 -92
- package/dungeons/technical/array-of-object-lookup-schema.json +0 -191
- package/dungeons/technical/experiments-schema.json +0 -203
- package/dungeons/technical/foobar-schema.json +0 -362
- package/dungeons/technical/group-analytics-schema.json +0 -241
- package/dungeons/technical/mirror-strategies-schema.json +0 -84
- package/dungeons/technical/nested-objects-schema.json +0 -145
- package/dungeons/technical/retention-cadence-schema.json +0 -37
- package/dungeons/technical/sanity-schema.json +0 -185
- package/dungeons/technical/scale-test-schema.json +0 -70
- package/dungeons/technical/scd-schema.json +0 -467
- package/dungeons/technical/simple-schema.json +0 -362
- package/dungeons/technical/text-generation-schema.json +0 -1062
- package/dungeons/user/.gitkeep +0 -0
- package/dungeons/vertical/community-schema.json +0 -579
- package/dungeons/vertical/devtools-schema.json +0 -601
- package/dungeons/vertical/ecommerce-schema.json +0 -604
- package/dungeons/vertical/education-schema.json +0 -5686
- package/dungeons/vertical/fintech-schema.json +0 -630
- package/dungeons/vertical/fitness-schema.json +0 -530
- package/dungeons/vertical/food-delivery-schema.json +0 -36728
- package/dungeons/vertical/gaming-schema.json +0 -438
- package/dungeons/vertical/healthcare-schema.json +0 -549
- package/dungeons/vertical/insurance-application-schema.json +0 -485
- package/dungeons/vertical/logistics-schema.json +0 -574
- package/dungeons/vertical/marketplace-schema.json +0 -533
- package/dungeons/vertical/media-schema.json +0 -4749
- package/dungeons/vertical/rpg-schema.json +0 -2491
- package/dungeons/vertical/rpg.js +0 -976
- package/dungeons/vertical/sass-schema.json +0 -3128
- package/dungeons/vertical/social-schema.json +0 -620
- package/dungeons/vertical/travel-schema.json +0 -580
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
2
|
+
* Soup preset configurations.
|
|
3
|
+
*
|
|
4
|
+
* Soup presets shape the INTRA-WEEK and INTRA-DAY rhythm: how many Gaussian
|
|
5
|
+
* peaks to sample from, how tightly each clusters, and how to weight each
|
|
6
|
+
* day-of-week and hour-of-day during accept/reject sampling.
|
|
7
|
+
*
|
|
8
|
+
* Soup is the orthogonal axis to macro presets (lib/templates/macro-presets.js).
|
|
9
|
+
* Macro shapes the big-picture trend (births, growth, decline). Soup shapes
|
|
10
|
+
* the texture inside that trend (weekly cycle, daily peak, burstiness).
|
|
4
11
|
*
|
|
5
12
|
* Parameters:
|
|
6
13
|
* - peaks(numDays): function returning number of Gaussian clusters
|
|
@@ -8,9 +15,6 @@
|
|
|
8
15
|
* - mean: offset from chunk center (0 = centered)
|
|
9
16
|
* - dayOfWeekWeights: 7-element array [Sun..Sat], max=1.0, null to disable
|
|
10
17
|
* - hourOfDayWeights: 24-element array [0h..23h UTC], max=1.0, null to disable
|
|
11
|
-
*
|
|
12
|
-
* Some presets also suggest bornRecentBias and percentUsersBornInDataset,
|
|
13
|
-
* but those are top-level dungeon config — presets only set them if not already specified.
|
|
14
18
|
*/
|
|
15
19
|
|
|
16
20
|
// Real-world Mixpanel DOW pattern: weekday-heavy, Saturday valley
|
|
@@ -27,11 +31,11 @@ export const REAL_HOD = [
|
|
|
27
31
|
export const FLAT_DOW = [1, 1, 1, 1, 1, 1, 1];
|
|
28
32
|
export const FLAT_HOD = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1];
|
|
29
33
|
|
|
30
|
-
/** @type {Record<string, {peaks: (numDays: number) => number, deviation: number, mean: number, dayOfWeekWeights: number[]|null, hourOfDayWeights: number[]|null
|
|
34
|
+
/** @type {Record<string, {peaks: (numDays: number) => number, deviation: number, mean: number, dayOfWeekWeights: number[]|null, hourOfDayWeights: number[]|null}>} */
|
|
31
35
|
export const SOUP_PRESETS = {
|
|
32
36
|
/**
|
|
33
37
|
* steady — Mature SaaS / Stable Product
|
|
34
|
-
*
|
|
38
|
+
* Tighter clusters, real-world DOW/HOD weights.
|
|
35
39
|
*/
|
|
36
40
|
steady: {
|
|
37
41
|
peaks: (numDays) => Math.max(5, numDays * 2),
|
|
@@ -39,13 +43,11 @@ export const SOUP_PRESETS = {
|
|
|
39
43
|
mean: 0,
|
|
40
44
|
dayOfWeekWeights: REAL_DOW,
|
|
41
45
|
hourOfDayWeights: REAL_HOD,
|
|
42
|
-
bornRecentBias: 0.1,
|
|
43
|
-
percentUsersBornInDataset: 10,
|
|
44
46
|
},
|
|
45
47
|
|
|
46
48
|
/**
|
|
47
49
|
* growth — Growing Startup (DEFAULT)
|
|
48
|
-
*
|
|
50
|
+
* Standard clustering with real-world weekly + daily rhythm.
|
|
49
51
|
*/
|
|
50
52
|
growth: {
|
|
51
53
|
peaks: (numDays) => Math.max(5, numDays * 2),
|
|
@@ -53,13 +55,11 @@ export const SOUP_PRESETS = {
|
|
|
53
55
|
mean: 0,
|
|
54
56
|
dayOfWeekWeights: REAL_DOW,
|
|
55
57
|
hourOfDayWeights: REAL_HOD,
|
|
56
|
-
bornRecentBias: 0.3,
|
|
57
|
-
percentUsersBornInDataset: 15,
|
|
58
58
|
},
|
|
59
59
|
|
|
60
60
|
/**
|
|
61
61
|
* spiky — Event-Driven / Bursty
|
|
62
|
-
*
|
|
62
|
+
* Fewer Gaussian clusters + tight deviation = visible peaks and valleys.
|
|
63
63
|
*/
|
|
64
64
|
spiky: {
|
|
65
65
|
peaks: (numDays) => Math.max(4, Math.ceil(numDays / 15)),
|
|
@@ -67,13 +67,11 @@ export const SOUP_PRESETS = {
|
|
|
67
67
|
mean: 0,
|
|
68
68
|
dayOfWeekWeights: REAL_DOW,
|
|
69
69
|
hourOfDayWeights: REAL_HOD,
|
|
70
|
-
bornRecentBias: 0.3,
|
|
71
|
-
percentUsersBornInDataset: 20,
|
|
72
70
|
},
|
|
73
71
|
|
|
74
72
|
/**
|
|
75
73
|
* seasonal — Strong Cyclical Patterns
|
|
76
|
-
*
|
|
74
|
+
* 4 major waves across the dataset for dramatic macro trends.
|
|
77
75
|
*/
|
|
78
76
|
seasonal: {
|
|
79
77
|
peaks: () => 4,
|
|
@@ -81,13 +79,11 @@ export const SOUP_PRESETS = {
|
|
|
81
79
|
mean: 0,
|
|
82
80
|
dayOfWeekWeights: REAL_DOW,
|
|
83
81
|
hourOfDayWeights: REAL_HOD,
|
|
84
|
-
bornRecentBias: 0.2,
|
|
85
|
-
percentUsersBornInDataset: 25,
|
|
86
82
|
},
|
|
87
83
|
|
|
88
84
|
/**
|
|
89
85
|
* global — Distributed Users Across Timezones
|
|
90
|
-
*
|
|
86
|
+
* Flat hourly + daily distribution. No cyclical patterns.
|
|
91
87
|
*/
|
|
92
88
|
global: {
|
|
93
89
|
peaks: (numDays) => Math.max(5, numDays * 2),
|
|
@@ -95,15 +91,12 @@ export const SOUP_PRESETS = {
|
|
|
95
91
|
mean: 0,
|
|
96
92
|
dayOfWeekWeights: FLAT_DOW,
|
|
97
93
|
hourOfDayWeights: FLAT_HOD,
|
|
98
|
-
bornRecentBias: 0,
|
|
99
|
-
percentUsersBornInDataset: 10,
|
|
100
94
|
},
|
|
101
95
|
|
|
102
96
|
/**
|
|
103
97
|
* churny — High Churn / Declining Product
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
* filters late events to create a true declining shape.
|
|
98
|
+
* Standard intra-week rhythm. Combine with macro: "decline" and an
|
|
99
|
+
* "everything" hook that filters late events to create a true declining shape.
|
|
107
100
|
*/
|
|
108
101
|
churny: {
|
|
109
102
|
peaks: (numDays) => Math.max(5, numDays * 2),
|
|
@@ -111,8 +104,6 @@ export const SOUP_PRESETS = {
|
|
|
111
104
|
mean: 0,
|
|
112
105
|
dayOfWeekWeights: REAL_DOW,
|
|
113
106
|
hourOfDayWeights: REAL_HOD,
|
|
114
|
-
bornRecentBias: 0,
|
|
115
|
-
percentUsersBornInDataset: 5,
|
|
116
107
|
},
|
|
117
108
|
|
|
118
109
|
/**
|
|
@@ -125,8 +116,6 @@ export const SOUP_PRESETS = {
|
|
|
125
116
|
mean: 0,
|
|
126
117
|
dayOfWeekWeights: REAL_DOW,
|
|
127
118
|
hourOfDayWeights: REAL_HOD,
|
|
128
|
-
bornRecentBias: 0.5,
|
|
129
|
-
percentUsersBornInDataset: 40,
|
|
130
119
|
},
|
|
131
120
|
};
|
|
132
121
|
|
|
@@ -137,7 +126,7 @@ export const PRESET_NAMES = Object.keys(SOUP_PRESETS);
|
|
|
137
126
|
* Resolves a soup config — handles string presets, preset+overrides, and raw objects.
|
|
138
127
|
* @param {string | object} soup - Soup config from dungeon
|
|
139
128
|
* @param {number} numDays - Number of days in the dataset
|
|
140
|
-
* @returns {{ soup: object
|
|
129
|
+
* @returns {{ soup: object }}
|
|
141
130
|
*/
|
|
142
131
|
export function resolveSoup(soup, numDays) {
|
|
143
132
|
if (!soup) return { soup: {} };
|
|
@@ -156,8 +145,6 @@ export function resolveSoup(soup, numDays) {
|
|
|
156
145
|
dayOfWeekWeights: preset.dayOfWeekWeights,
|
|
157
146
|
hourOfDayWeights: preset.hourOfDayWeights,
|
|
158
147
|
},
|
|
159
|
-
suggestedBornRecentBias: preset.bornRecentBias,
|
|
160
|
-
suggestedPercentUsersBornInDataset: preset.percentUsersBornInDataset,
|
|
161
148
|
};
|
|
162
149
|
}
|
|
163
150
|
|
|
@@ -176,11 +163,7 @@ export function resolveSoup(soup, numDays) {
|
|
|
176
163
|
};
|
|
177
164
|
// Apply overrides (excluding the 'preset' key itself)
|
|
178
165
|
const { preset: _, ...overrides } = soup;
|
|
179
|
-
return {
|
|
180
|
-
soup: { ...base, ...overrides },
|
|
181
|
-
suggestedBornRecentBias: preset.bornRecentBias,
|
|
182
|
-
suggestedPercentUsersBornInDataset: preset.percentUsersBornInDataset,
|
|
183
|
-
};
|
|
166
|
+
return { soup: { ...base, ...overrides } };
|
|
184
167
|
}
|
|
185
168
|
|
|
186
169
|
// Raw object: { peaks: 10, deviation: 2 } — pass through unchanged
|
package/lib/utils/utils.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import Chance from 'chance';
|
|
3
3
|
import readline from 'readline';
|
|
4
|
-
import { comma
|
|
4
|
+
import { comma } from 'ak-tools';
|
|
5
5
|
import dayjs from 'dayjs';
|
|
6
6
|
import utc from 'dayjs/plugin/utc.js';
|
|
7
7
|
import path from 'path';
|
|
@@ -24,7 +24,27 @@ const { NODE_ENV = "unknown" } = process.env;
|
|
|
24
24
|
let globalChance;
|
|
25
25
|
let chanceInitialized = false;
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
// Module-scoped memoization cache for weighted-array resolvers in `choose()`.
|
|
28
|
+
// Lives for the lifetime of the node process; key is the function source string.
|
|
29
|
+
const weightedArrayCache = new Map();
|
|
30
|
+
|
|
31
|
+
// Reference "now" used by date() and day() factories. Defaults to wall-clock at
|
|
32
|
+
// import time; the orchestrator overrides this via setDatasetNow() once the dataset
|
|
33
|
+
// window is resolved, so date helpers in dungeon configs produce deterministic
|
|
34
|
+
// values relative to the dataset end (not the process start).
|
|
35
|
+
let DATASET_NOW = dayjs.utc();
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Override the reference "now" used by date()/day() factories. Called by the
|
|
39
|
+
* orchestrator with the resolved dataset end so every date helper produces
|
|
40
|
+
* deterministic values relative to the dataset window.
|
|
41
|
+
* @param {number} unixSeconds
|
|
42
|
+
*/
|
|
43
|
+
function setDatasetNow(unixSeconds) {
|
|
44
|
+
if (typeof unixSeconds === 'number' && Number.isFinite(unixSeconds)) {
|
|
45
|
+
DATASET_NOW = dayjs.unix(unixSeconds).utc();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
28
48
|
|
|
29
49
|
|
|
30
50
|
import { Storage as cloudStorage } from '@google-cloud/storage';
|
|
@@ -102,7 +122,7 @@ function pick(items) {
|
|
|
102
122
|
*/
|
|
103
123
|
function date(inTheLast = 30, isPast = true, format = 'YYYY-MM-DD') {
|
|
104
124
|
const chance = getChance();
|
|
105
|
-
const now =
|
|
125
|
+
const now = DATASET_NOW;
|
|
106
126
|
if (Math.abs(inTheLast) > 365 * 10) inTheLast = chance.integer({ min: 1, max: 180 });
|
|
107
127
|
return function () {
|
|
108
128
|
const when = chance.integer({ min: 0, max: Math.abs(inTheLast) });
|
|
@@ -159,8 +179,8 @@ function datesBetween(start, end) {
|
|
|
159
179
|
*/
|
|
160
180
|
function day(start, end) {
|
|
161
181
|
// if (!end) end = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
162
|
-
if (!start) start =
|
|
163
|
-
if (!end) end =
|
|
182
|
+
if (!start) start = DATASET_NOW.subtract(30, 'd').toISOString();
|
|
183
|
+
if (!end) end = DATASET_NOW.toISOString();
|
|
164
184
|
const chance = getChance();
|
|
165
185
|
const format = 'YYYY-MM-DD';
|
|
166
186
|
return function (min, max) {
|
|
@@ -219,20 +239,15 @@ function choose(value) {
|
|
|
219
239
|
while (typeof value === 'function') {
|
|
220
240
|
const funcString = value.toString();
|
|
221
241
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
global.weightedArrayCache = new Map();
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
if (global.weightedArrayCache.has(funcString)) {
|
|
228
|
-
value = global.weightedArrayCache.get(funcString);
|
|
242
|
+
if (weightedArrayCache.has(funcString)) {
|
|
243
|
+
value = weightedArrayCache.get(funcString);
|
|
229
244
|
break;
|
|
230
245
|
}
|
|
231
246
|
|
|
232
247
|
const result = value();
|
|
233
248
|
if (Array.isArray(result) && result.length > 10) {
|
|
234
249
|
// Cache large arrays (likely weighted arrays)
|
|
235
|
-
|
|
250
|
+
weightedArrayCache.set(funcString, result);
|
|
236
251
|
}
|
|
237
252
|
value = result;
|
|
238
253
|
}
|
|
@@ -1143,7 +1158,7 @@ CORE
|
|
|
1143
1158
|
//the function which generates $distinct_id + $anonymous_ids, $session_ids, and created, skewing towards the present
|
|
1144
1159
|
function generateUser(user_id, opts, amplitude = 1, frequency = 1, skew = 1) {
|
|
1145
1160
|
const chance = getChance();
|
|
1146
|
-
const { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds } = opts;
|
|
1161
|
+
const { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix } = opts;
|
|
1147
1162
|
// Uniformly distributed `u`, then skew applied
|
|
1148
1163
|
let u = Math.pow(chance.random(), skew);
|
|
1149
1164
|
|
|
@@ -1155,7 +1170,7 @@ function generateUser(user_id, opts, amplitude = 1, frequency = 1, skew = 1) {
|
|
|
1155
1170
|
|
|
1156
1171
|
// Clamp values to ensure they are within the desired range
|
|
1157
1172
|
daysAgoBorn = Math.min(daysAgoBorn, numDays);
|
|
1158
|
-
const props = person(user_id, daysAgoBorn, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds);
|
|
1173
|
+
const props = person(user_id, daysAgoBorn, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix);
|
|
1159
1174
|
|
|
1160
1175
|
const user = {
|
|
1161
1176
|
distinct_id: user_id,
|
|
@@ -1188,7 +1203,7 @@ const DEFAULT_HOD_WEIGHTS = [
|
|
|
1188
1203
|
0.584, 0.574, 0.554, 0.576, 0.604, 0.655, 0.722, 0.816
|
|
1189
1204
|
];
|
|
1190
1205
|
|
|
1191
|
-
function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0, dayOfWeekWeights = DEFAULT_DOW_WEIGHTS, hourOfDayWeights = DEFAULT_HOD_WEIGHTS
|
|
1206
|
+
function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0, dayOfWeekWeights = DEFAULT_DOW_WEIGHTS, hourOfDayWeights = DEFAULT_HOD_WEIGHTS) {
|
|
1192
1207
|
if (!earliestTime) earliestTime = global.FIXED_BEGIN ? global.FIXED_BEGIN : dayjs().subtract(30, 'd').unix();
|
|
1193
1208
|
if (!latestTime) latestTime = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
1194
1209
|
const chance = getChance();
|
|
@@ -1215,7 +1230,7 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0,
|
|
|
1215
1230
|
// Phase 2: DOW accept/reject — retry if day-of-week doesn't pass weight check
|
|
1216
1231
|
if (dayOfWeekWeights) {
|
|
1217
1232
|
for (let attempt = 0; attempt < 50; attempt++) {
|
|
1218
|
-
const dow = new Date(
|
|
1233
|
+
const dow = new Date(candidate * 1000).getUTCDay();
|
|
1219
1234
|
if (chance.random() < dayOfWeekWeights[dow]) break;
|
|
1220
1235
|
// Rejected — resample from Gaussian chunks
|
|
1221
1236
|
const pi = integer(0, peaks - 1);
|
|
@@ -1231,8 +1246,7 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0,
|
|
|
1231
1246
|
|
|
1232
1247
|
// Phase 3: Redistribute hour-of-day (changes only hour within same day)
|
|
1233
1248
|
if (hourOfDayWeights) {
|
|
1234
|
-
const
|
|
1235
|
-
const d = new Date(shifted * 1000);
|
|
1249
|
+
const d = new Date(candidate * 1000);
|
|
1236
1250
|
const currentMinute = d.getUTCMinutes();
|
|
1237
1251
|
const currentSecond = d.getUTCSeconds();
|
|
1238
1252
|
|
|
@@ -1244,9 +1258,8 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0,
|
|
|
1244
1258
|
if (roll <= 0) { newHour = h; break; }
|
|
1245
1259
|
}
|
|
1246
1260
|
|
|
1247
|
-
const
|
|
1248
|
-
|
|
1249
|
-
candidate = newShifted - timeShiftSeconds;
|
|
1261
|
+
const dayStart = Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()) / 1000;
|
|
1262
|
+
candidate = dayStart + newHour * 3600 + currentMinute * 60 + currentSecond;
|
|
1250
1263
|
candidate = Math.max(earliestTime, Math.min(latestTime, candidate));
|
|
1251
1264
|
}
|
|
1252
1265
|
|
|
@@ -1264,7 +1277,7 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0,
|
|
|
1264
1277
|
* @param {boolean} hasSessionIds
|
|
1265
1278
|
* @return {Person}
|
|
1266
1279
|
*/
|
|
1267
|
-
function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false, hasAnonIds = false, hasSessionIds = false) {
|
|
1280
|
+
function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false, hasAnonIds = false, hasSessionIds = false, datasetEndUnix) {
|
|
1268
1281
|
const chance = getChance();
|
|
1269
1282
|
//names and photos
|
|
1270
1283
|
const l = chance.letter.bind(chance);
|
|
@@ -1278,7 +1291,11 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false
|
|
|
1278
1291
|
let randomAvatarNumber = integer(1, 99);
|
|
1279
1292
|
let avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
|
|
1280
1293
|
let avatar = avatarPrefix + avPath;
|
|
1281
|
-
|
|
1294
|
+
// Birth date: anchor to dataset end (post-resolution) so user.created lives inside
|
|
1295
|
+
// the configured window, not wall-clock time. Falls back to dayjs() only when
|
|
1296
|
+
// person() is called outside the normal generation pipeline (e.g. direct unit tests).
|
|
1297
|
+
const anchor = datasetEndUnix ? dayjs.unix(datasetEndUnix) : dayjs();
|
|
1298
|
+
let created = anchor.subtract(bornDaysAgo, 'day').format('YYYY-MM-DD');
|
|
1282
1299
|
|
|
1283
1300
|
|
|
1284
1301
|
// const created = date(bornDaysAgo, true)();
|
|
@@ -1307,7 +1324,8 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false
|
|
|
1307
1324
|
if (hasAnonIds) {
|
|
1308
1325
|
const clusterSize = integer(2, 10);
|
|
1309
1326
|
for (let i = 0; i < clusterSize; i++) {
|
|
1310
|
-
|
|
1327
|
+
// Use seeded chance, not ak-tools uid() (which uses Math.random).
|
|
1328
|
+
const anonId = chance.string({ length: 42, pool: 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' });
|
|
1311
1329
|
user.anonymousIds.push(anonId);
|
|
1312
1330
|
}
|
|
1313
1331
|
}
|
|
@@ -1429,11 +1447,26 @@ function deepClone(thing, opts) {
|
|
|
1429
1447
|
|
|
1430
1448
|
|
|
1431
1449
|
/**
|
|
1432
|
-
* Generates a session ID in the standard format
|
|
1450
|
+
* Generates a session ID in the standard format. Derives it deterministically
|
|
1451
|
+
* from a seed string (first event time + user_id, typically) via quickHash so
|
|
1452
|
+
* we don't consume from the main seeded RNG stream — that consumption would
|
|
1453
|
+
* cascade into all downstream events whenever the number of sessions changed
|
|
1454
|
+
* (e.g. when sessionTimeout differs).
|
|
1455
|
+
* @param {string} [seedStr] - Stable input string. If absent, falls back to a
|
|
1456
|
+
* chance.string() pull (only used by tests / direct callers without a seed).
|
|
1433
1457
|
* @returns {string} Session ID like "xxxxx-xxxxx-xxxxx-xxxxx"
|
|
1434
1458
|
*/
|
|
1435
|
-
function generateSessionId() {
|
|
1436
|
-
|
|
1459
|
+
function generateSessionId(seedStr) {
|
|
1460
|
+
if (seedStr) {
|
|
1461
|
+
// quickHash returns ~16 hex chars; expand to 4×5 segments via re-hashing.
|
|
1462
|
+
const h1 = quickHash(seedStr);
|
|
1463
|
+
const h2 = quickHash(h1);
|
|
1464
|
+
const all = (h1 + h2).replace(/[^a-zA-Z0-9]/g, '').padEnd(20, '0');
|
|
1465
|
+
return [all.slice(0, 5), all.slice(5, 10), all.slice(10, 15), all.slice(15, 20)].join('-');
|
|
1466
|
+
}
|
|
1467
|
+
const c = getChance();
|
|
1468
|
+
const seg = () => c.string({ length: 5, pool: 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' });
|
|
1469
|
+
return [seg(), seg(), seg(), seg()].join("-");
|
|
1437
1470
|
}
|
|
1438
1471
|
|
|
1439
1472
|
/**
|
|
@@ -1459,7 +1492,7 @@ function bunchIntoSessions(events, timeoutMinutes, soupParams) {
|
|
|
1459
1492
|
|
|
1460
1493
|
const chance = getChance();
|
|
1461
1494
|
const { earliestTime, latestTime, peaks, deviation, mean,
|
|
1462
|
-
dayOfWeekWeights, hourOfDayWeights
|
|
1495
|
+
dayOfWeekWeights, hourOfDayWeights } = soupParams;
|
|
1463
1496
|
|
|
1464
1497
|
// Sort by time first
|
|
1465
1498
|
events.sort((a, b) => a.time < b.time ? -1 : a.time > b.time ? 1 : 0);
|
|
@@ -1472,8 +1505,8 @@ function bunchIntoSessions(events, timeoutMinutes, soupParams) {
|
|
|
1472
1505
|
const anchors = [];
|
|
1473
1506
|
for (let i = 0; i < numSessions; i++) {
|
|
1474
1507
|
const soupTime = TimeSoup(earliestTime, latestTime, peaks, deviation, mean,
|
|
1475
|
-
dayOfWeekWeights, hourOfDayWeights
|
|
1476
|
-
anchors.push(soupTime
|
|
1508
|
+
dayOfWeekWeights, hourOfDayWeights);
|
|
1509
|
+
anchors.push(soupTime);
|
|
1477
1510
|
}
|
|
1478
1511
|
anchors.sort((a, b) => a - b);
|
|
1479
1512
|
|
|
@@ -1485,7 +1518,6 @@ function bunchIntoSessions(events, timeoutMinutes, soupParams) {
|
|
|
1485
1518
|
}
|
|
1486
1519
|
|
|
1487
1520
|
// Retime events within each session
|
|
1488
|
-
let writeIndex = 0;
|
|
1489
1521
|
for (let s = 0; s < numSessions; s++) {
|
|
1490
1522
|
const bucket = sessionBuckets[s];
|
|
1491
1523
|
if (bucket.length === 0) continue;
|
|
@@ -1493,17 +1525,13 @@ function bunchIntoSessions(events, timeoutMinutes, soupParams) {
|
|
|
1493
1525
|
let currentTime = anchors[s];
|
|
1494
1526
|
for (let e = 0; e < bucket.length; e++) {
|
|
1495
1527
|
const ev = bucket[e];
|
|
1496
|
-
const clampedTime = Math.min(currentTime,
|
|
1528
|
+
const clampedTime = Math.min(currentTime, latestTime);
|
|
1497
1529
|
|
|
1498
1530
|
ev.time = dayjs.unix(clampedTime).toISOString();
|
|
1499
1531
|
// Regenerate insert_id to match new time
|
|
1500
1532
|
const distinctId = ev.user_id || ev.device_id || ev.distinct_id || '';
|
|
1501
1533
|
ev.insert_id = quickHash(`${ev.event}-${ev.time}-${distinctId}`);
|
|
1502
1534
|
|
|
1503
|
-
if (currentTime > maxTime) {
|
|
1504
|
-
ev._drop = true;
|
|
1505
|
-
}
|
|
1506
|
-
|
|
1507
1535
|
// Advance time within session: 5-300 seconds (5s to 5min)
|
|
1508
1536
|
currentTime += chance.integer({ min: 5, max: 300 });
|
|
1509
1537
|
}
|
|
@@ -1532,7 +1560,10 @@ function assignSessionIds(events, timeoutMinutes = 30) {
|
|
|
1532
1560
|
const timeoutMs = timeoutMinutes * 60 * 1000;
|
|
1533
1561
|
const maxSessionMs = 24 * 60 * 60 * 1000;
|
|
1534
1562
|
|
|
1535
|
-
|
|
1563
|
+
// Derive a stable seed from the first event so session IDs are deterministic
|
|
1564
|
+
// across runs without consuming the main seeded RNG stream.
|
|
1565
|
+
const userKey = events[0].user_id || events[0].device_id || events[0].distinct_id || '';
|
|
1566
|
+
let currentSessionId = generateSessionId(`${userKey}-${events[0].time}`);
|
|
1536
1567
|
let sessionStartMs = new Date(events[0].time).getTime();
|
|
1537
1568
|
let lastEventMs = sessionStartMs;
|
|
1538
1569
|
|
|
@@ -1542,7 +1573,7 @@ function assignSessionIds(events, timeoutMinutes = 30) {
|
|
|
1542
1573
|
const sessionDuration = eventMs - sessionStartMs;
|
|
1543
1574
|
|
|
1544
1575
|
if (gapFromLast > timeoutMs || sessionDuration > maxSessionMs) {
|
|
1545
|
-
currentSessionId = generateSessionId();
|
|
1576
|
+
currentSessionId = generateSessionId(`${userKey}-${event.time}`);
|
|
1546
1577
|
sessionStartMs = eventMs;
|
|
1547
1578
|
}
|
|
1548
1579
|
|
|
@@ -1606,4 +1637,5 @@ export {
|
|
|
1606
1637
|
generateSessionId,
|
|
1607
1638
|
assignSessionIds,
|
|
1608
1639
|
bunchIntoSessions,
|
|
1640
|
+
setDatasetNow,
|
|
1609
1641
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ak--47/dungeon-master",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.1",
|
|
4
4
|
"description": "generate fancy datasets",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|
|
@@ -19,10 +19,16 @@
|
|
|
19
19
|
"lib/",
|
|
20
20
|
"dungeons/",
|
|
21
21
|
"!dungeons/customers/",
|
|
22
|
+
"!dungeons/user/",
|
|
22
23
|
"scripts/",
|
|
24
|
+
"!scripts/experiments/",
|
|
23
25
|
"package.json",
|
|
24
|
-
"README.md"
|
|
26
|
+
"README.md",
|
|
27
|
+
"CHANGELOG.md"
|
|
25
28
|
],
|
|
29
|
+
"engines": {
|
|
30
|
+
"node": ">=18.0.0"
|
|
31
|
+
},
|
|
26
32
|
"publishConfig": {
|
|
27
33
|
"access": "public"
|
|
28
34
|
},
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Smoke-test runner — runs every dungeon at tiny scale in parallel to verify it
|
|
4
|
+
* loads, generates events, and writes output without crashing.
|
|
5
|
+
*
|
|
6
|
+
* Default scale: 100 users, 1000 events per dungeon. NOT for verification —
|
|
7
|
+
* use scripts/verify-runner.mjs at full fidelity for that.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* node scripts/smoke-test-all.mjs
|
|
11
|
+
* node scripts/smoke-test-all.mjs --dir dungeons/vertical # default
|
|
12
|
+
* node scripts/smoke-test-all.mjs --dir dungeons/technical
|
|
13
|
+
* node scripts/smoke-test-all.mjs --dir dungeons # both subdirs
|
|
14
|
+
* node scripts/smoke-test-all.mjs --concurrency 4 # default: cpu count
|
|
15
|
+
* node scripts/smoke-test-all.mjs --users 500 --events 5000 # override scale
|
|
16
|
+
*
|
|
17
|
+
* Output: per-dungeon PASS/FAIL line + a final summary table. Spawns each
|
|
18
|
+
* dungeon as a child node process so a crash in one doesn't abort the run.
|
|
19
|
+
*/
|
|
20
|
+
import { spawn } from 'child_process';
|
|
21
|
+
import { readdirSync, statSync, rmSync, existsSync, mkdirSync } from 'fs';
|
|
22
|
+
import path from 'path';
|
|
23
|
+
import os from 'os';
|
|
24
|
+
|
|
25
|
+
const args = process.argv.slice(2);
|
|
26
|
+
function arg(name, fallback) {
|
|
27
|
+
const i = args.indexOf(`--${name}`);
|
|
28
|
+
return i === -1 ? fallback : args[i + 1];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..');
|
|
32
|
+
const dirArg = arg('dir', 'dungeons/vertical');
|
|
33
|
+
const numUsers = parseInt(arg('users', '100'), 10);
|
|
34
|
+
const numEvents = parseInt(arg('events', '1000'), 10);
|
|
35
|
+
const concurrency = parseInt(arg('concurrency', String(Math.max(2, os.cpus().length))), 10);
|
|
36
|
+
const keep = args.includes('--keep');
|
|
37
|
+
|
|
38
|
+
function findDungeons(dir) {
|
|
39
|
+
const abs = path.resolve(ROOT, dir);
|
|
40
|
+
if (!existsSync(abs)) return [];
|
|
41
|
+
const stat = statSync(abs);
|
|
42
|
+
if (stat.isFile()) return [abs];
|
|
43
|
+
const out = [];
|
|
44
|
+
for (const entry of readdirSync(abs)) {
|
|
45
|
+
const full = path.join(abs, entry);
|
|
46
|
+
const s = statSync(full);
|
|
47
|
+
if (s.isDirectory()) out.push(...findDungeons(full));
|
|
48
|
+
else if (entry.endsWith('.js')) out.push(full);
|
|
49
|
+
}
|
|
50
|
+
return out;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const dungeons = findDungeons(dirArg).sort();
|
|
54
|
+
if (dungeons.length === 0) {
|
|
55
|
+
console.error(`No dungeons found under ${dirArg}`);
|
|
56
|
+
process.exit(1);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const dataDir = path.join(ROOT, 'data');
|
|
60
|
+
mkdirSync(dataDir, { recursive: true });
|
|
61
|
+
|
|
62
|
+
const RUNNER = `
|
|
63
|
+
import generate from '${path.join(ROOT, 'index.js')}';
|
|
64
|
+
// With node -e, process.argv = [nodePath, ...userArgs] — no script slot.
|
|
65
|
+
const [dungeonPath, name, numUsers, numEvents] = process.argv.slice(1);
|
|
66
|
+
const r = await generate(dungeonPath, {
|
|
67
|
+
numUsers: parseInt(numUsers, 10),
|
|
68
|
+
numEvents: parseInt(numEvents, 10),
|
|
69
|
+
avgEventsPerUserPerDay: undefined, // force numEvents path
|
|
70
|
+
writeToDisk: true,
|
|
71
|
+
name,
|
|
72
|
+
format: 'json',
|
|
73
|
+
verbose: false,
|
|
74
|
+
concurrency: 1,
|
|
75
|
+
token: '',
|
|
76
|
+
serviceAccount: 'fake', serviceSecret: 'fake', projectId: '1',
|
|
77
|
+
});
|
|
78
|
+
console.log(JSON.stringify({ eventCount: r.eventCount, userCount: r.userCount }));
|
|
79
|
+
`;
|
|
80
|
+
|
|
81
|
+
function runOne(dungeonPath) {
|
|
82
|
+
const base = path.basename(dungeonPath, '.js');
|
|
83
|
+
const namePrefix = `smoke-${base}`;
|
|
84
|
+
return new Promise(resolve => {
|
|
85
|
+
const start = Date.now();
|
|
86
|
+
const child = spawn(
|
|
87
|
+
process.execPath,
|
|
88
|
+
['--input-type=module', '-e', RUNNER, dungeonPath, namePrefix, String(numUsers), String(numEvents)],
|
|
89
|
+
{ cwd: ROOT, stdio: ['ignore', 'pipe', 'pipe'] }
|
|
90
|
+
);
|
|
91
|
+
let out = '';
|
|
92
|
+
let err = '';
|
|
93
|
+
child.stdout.on('data', d => { out += d.toString(); });
|
|
94
|
+
child.stderr.on('data', d => { err += d.toString(); });
|
|
95
|
+
child.on('close', code => {
|
|
96
|
+
const ms = Date.now() - start;
|
|
97
|
+
let result;
|
|
98
|
+
try {
|
|
99
|
+
const lastLine = out.trim().split('\n').filter(Boolean).pop() || '{}';
|
|
100
|
+
result = JSON.parse(lastLine);
|
|
101
|
+
} catch {
|
|
102
|
+
result = {};
|
|
103
|
+
}
|
|
104
|
+
const ok = code === 0 && result.eventCount > 0;
|
|
105
|
+
if (!keep) {
|
|
106
|
+
try {
|
|
107
|
+
for (const f of readdirSync(dataDir)) {
|
|
108
|
+
if (f.startsWith(`${namePrefix}-`)) rmSync(path.join(dataDir, f));
|
|
109
|
+
}
|
|
110
|
+
} catch {}
|
|
111
|
+
}
|
|
112
|
+
resolve({
|
|
113
|
+
dungeon: path.relative(ROOT, dungeonPath),
|
|
114
|
+
ok,
|
|
115
|
+
code,
|
|
116
|
+
ms,
|
|
117
|
+
eventCount: result.eventCount || 0,
|
|
118
|
+
userCount: result.userCount || 0,
|
|
119
|
+
err: ok ? '' : (err.trim().split('\n').slice(-3).join(' | ') || `exit ${code}`),
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
console.log(`Smoke test: ${dungeons.length} dungeons, ${numUsers} users / ${numEvents} events each, concurrency=${concurrency}`);
|
|
126
|
+
const queue = [...dungeons];
|
|
127
|
+
const results = [];
|
|
128
|
+
const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
|
|
129
|
+
while (queue.length) {
|
|
130
|
+
const d = queue.shift();
|
|
131
|
+
const res = await runOne(d);
|
|
132
|
+
const tag = res.ok ? 'PASS' : 'FAIL';
|
|
133
|
+
console.log(` [${tag}] ${path.basename(res.dungeon)} — ${res.eventCount} events, ${res.userCount} users, ${(res.ms / 1000).toFixed(2)}s${res.ok ? '' : ` — ${res.err}`}`);
|
|
134
|
+
results.push(res);
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
await Promise.all(workers);
|
|
138
|
+
|
|
139
|
+
results.sort((a, b) => a.dungeon.localeCompare(b.dungeon));
|
|
140
|
+
const fails = results.filter(r => !r.ok);
|
|
141
|
+
|
|
142
|
+
console.log('\n┌─────────────────────────────────────────────────┬───────┬────────┬───────┬──────┐');
|
|
143
|
+
console.log('│ Dungeon │ State │ Events │ Users │ ms │');
|
|
144
|
+
console.log('├─────────────────────────────────────────────────┼───────┼────────┼───────┼──────┤');
|
|
145
|
+
for (const r of results) {
|
|
146
|
+
const name = r.dungeon.padEnd(47).slice(0, 47);
|
|
147
|
+
const state = (r.ok ? 'PASS' : 'FAIL').padEnd(5);
|
|
148
|
+
const ev = String(r.eventCount).padStart(6);
|
|
149
|
+
const u = String(r.userCount).padStart(5);
|
|
150
|
+
const ms = String(r.ms).padStart(4);
|
|
151
|
+
console.log(`│ ${name} │ ${state} │ ${ev} │ ${u} │ ${ms} │`);
|
|
152
|
+
}
|
|
153
|
+
console.log('└─────────────────────────────────────────────────┴───────┴────────┴───────┴──────┘');
|
|
154
|
+
|
|
155
|
+
console.log(`\n${results.length - fails.length}/${results.length} passed`);
|
|
156
|
+
if (fails.length > 0) {
|
|
157
|
+
console.log('\nFailures:');
|
|
158
|
+
for (const f of fails) {
|
|
159
|
+
console.log(` ✗ ${f.dungeon}: ${f.err}`);
|
|
160
|
+
}
|
|
161
|
+
process.exit(1);
|
|
162
|
+
}
|