@ak--47/dungeon-master 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +51 -13
- package/dungeons/technical/ad-spend.js +2 -2
- package/dungeons/technical/anonymous-users.js +2 -2
- package/dungeons/technical/array-of-object-lookup.js +2 -2
- package/dungeons/technical/experiments.js +2 -2
- package/dungeons/technical/foobar.js +2 -2
- package/dungeons/technical/group-analytics.js +2 -2
- package/dungeons/technical/mirror-strategies.js +2 -2
- package/dungeons/technical/nested-objects.js +2 -2
- package/dungeons/technical/retention-cadence.js +2 -3
- package/dungeons/technical/sanity.js +2 -2
- package/dungeons/technical/scale-test.js +2 -2
- package/dungeons/technical/scd.js +2 -2
- package/dungeons/technical/simple.js +2 -2
- package/dungeons/technical/simplest.js +2 -2
- package/dungeons/technical/text-generation.js +2 -2
- package/dungeons/vertical/ai-platform-schema.json +617 -0
- package/dungeons/vertical/ai-platform.js +799 -0
- package/dungeons/vertical/community.js +40 -26
- package/dungeons/vertical/crypto-schema.json +546 -0
- package/dungeons/vertical/crypto.js +721 -0
- package/dungeons/vertical/dating-schema.json +401 -0
- package/dungeons/vertical/dating.js +798 -0
- package/dungeons/vertical/devtools.js +13 -9
- package/dungeons/vertical/ecommerce.js +2 -3
- package/dungeons/vertical/education.js +4 -5
- package/dungeons/vertical/fintech.js +32 -29
- package/dungeons/vertical/fitness.js +2 -3
- package/dungeons/vertical/food-delivery.js +37 -43
- package/dungeons/vertical/gaming-schema.json +2495 -230
- package/dungeons/vertical/gaming.js +771 -388
- package/dungeons/vertical/healthcare.js +2 -3
- package/dungeons/vertical/insurance-application.js +2 -3
- package/dungeons/vertical/logistics.js +20 -14
- package/dungeons/vertical/marketplace.js +22 -14
- package/dungeons/vertical/media.js +39 -30
- package/dungeons/vertical/real-estate-schema.json +527 -0
- package/dungeons/vertical/real-estate.js +774 -0
- package/dungeons/vertical/sass.js +2 -3
- package/dungeons/vertical/social.js +15 -13
- package/dungeons/vertical/travel.js +59 -26
- package/lib/core/config-validator.js +71 -15
- package/lib/core/storage.js +14 -4
- package/lib/orchestrators/user-loop.js +39 -5
- package/lib/templates/macro-presets.js +111 -0
- package/lib/templates/soup-presets.js +19 -36
- package/package.json +8 -2
- package/types.d.ts +219 -42
- package/dungeons/user/.gitkeep +0 -0
- package/dungeons/vertical/rpg-schema.json +0 -2491
- package/dungeons/vertical/rpg.js +0 -976
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
const SEED = "harness-sass";
|
|
3
3
|
const num_days = 100;
|
|
4
4
|
const num_users = 5_000;
|
|
5
|
-
const
|
|
5
|
+
const avg_events_per_user_per_day = 1.2;
|
|
6
6
|
let token = "your-mixpanel-token";
|
|
7
7
|
|
|
8
8
|
// ── env overrides ──
|
|
@@ -258,7 +258,7 @@ const config = {
|
|
|
258
258
|
token,
|
|
259
259
|
seed: SEED,
|
|
260
260
|
numDays: num_days,
|
|
261
|
-
|
|
261
|
+
avgEventsPerUserPerDay: avg_events_per_user_per_day,
|
|
262
262
|
numUsers: num_users,
|
|
263
263
|
hasAnonIds: false,
|
|
264
264
|
hasSessionIds: true,
|
|
@@ -273,7 +273,6 @@ const config = {
|
|
|
273
273
|
hasCampaigns: false,
|
|
274
274
|
isAnonymous: false,
|
|
275
275
|
hasAdSpend: false,
|
|
276
|
-
percentUsersBornInDataset: 50,
|
|
277
276
|
hasAvatar: true,
|
|
278
277
|
concurrency: 1,
|
|
279
278
|
writeToDisk: false,
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
const SEED = "harness-social";
|
|
3
3
|
const num_days = 100;
|
|
4
4
|
const num_users = 5_000;
|
|
5
|
-
const
|
|
5
|
+
const avg_events_per_user_per_day = 1.2;
|
|
6
6
|
let token = "your-mixpanel-token";
|
|
7
7
|
|
|
8
8
|
// ── env overrides ──
|
|
@@ -172,7 +172,7 @@ const config = {
|
|
|
172
172
|
token,
|
|
173
173
|
seed: SEED,
|
|
174
174
|
numDays: num_days,
|
|
175
|
-
|
|
175
|
+
avgEventsPerUserPerDay: avg_events_per_user_per_day,
|
|
176
176
|
numUsers: num_users,
|
|
177
177
|
hasAnonIds: false,
|
|
178
178
|
hasSessionIds: true,
|
|
@@ -187,7 +187,6 @@ const config = {
|
|
|
187
187
|
hasCampaigns: false,
|
|
188
188
|
isAnonymous: false,
|
|
189
189
|
hasAdSpend: false,
|
|
190
|
-
percentUsersBornInDataset: 50,
|
|
191
190
|
hasAvatar: true,
|
|
192
191
|
concurrency: 1,
|
|
193
192
|
writeToDisk: false,
|
|
@@ -521,15 +520,7 @@ const config = {
|
|
|
521
520
|
}
|
|
522
521
|
}
|
|
523
522
|
|
|
524
|
-
// Hook #8: WEEKEND CONTENT SURGE -
|
|
525
|
-
if (record.event === "post created" || record.event === "story created") {
|
|
526
|
-
const dayOfWeek = EVENT_TIME.day(); // 0 = Sunday, 6 = Saturday
|
|
527
|
-
if (dayOfWeek === 0 || dayOfWeek === 6) {
|
|
528
|
-
record.weekend_surge = true;
|
|
529
|
-
} else {
|
|
530
|
-
record.weekend_surge = false;
|
|
531
|
-
}
|
|
532
|
-
}
|
|
523
|
+
// Hook #8: WEEKEND CONTENT SURGE - tagging moved to everything hook (sessionization reassigns times after event hook)
|
|
533
524
|
}
|
|
534
525
|
|
|
535
526
|
// ─── EVERYTHING-LEVEL HOOKS ──────────────────────────────────────
|
|
@@ -721,7 +712,18 @@ const config = {
|
|
|
721
712
|
}
|
|
722
713
|
}
|
|
723
714
|
|
|
724
|
-
// Hook #8: WEEKEND CONTENT SURGE -
|
|
715
|
+
// Hook #8: WEEKEND CONTENT SURGE - tag weekend content AFTER sessionization has finalized times
|
|
716
|
+
userEvents.forEach(event => {
|
|
717
|
+
if (event.event === "post created" || event.event === "story created") {
|
|
718
|
+
const dow = new Date(event.time).getUTCDay(); // 0 = Sunday, 6 = Saturday
|
|
719
|
+
if (dow === 0 || dow === 6) {
|
|
720
|
+
event.weekend_surge = true;
|
|
721
|
+
} else {
|
|
722
|
+
event.weekend_surge = false;
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
});
|
|
726
|
+
// Inject duplicate events for weekend content
|
|
725
727
|
for (let idx = userEvents.length - 1; idx >= 0; idx--) {
|
|
726
728
|
const event = userEvents[idx];
|
|
727
729
|
if (event.weekend_surge && !event.weekend_duplicate) {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
const SEED = "dm4-travel";
|
|
3
3
|
const num_days = 100;
|
|
4
4
|
const num_users = 5_000;
|
|
5
|
-
const
|
|
5
|
+
const avg_events_per_user_per_day = 1.2;
|
|
6
6
|
let token = "your-mixpanel-token";
|
|
7
7
|
|
|
8
8
|
// ── env overrides ──
|
|
@@ -207,7 +207,7 @@ const config = {
|
|
|
207
207
|
token,
|
|
208
208
|
seed: SEED,
|
|
209
209
|
numDays: num_days,
|
|
210
|
-
|
|
210
|
+
avgEventsPerUserPerDay: avg_events_per_user_per_day,
|
|
211
211
|
numUsers: num_users,
|
|
212
212
|
hasAnonIds: false,
|
|
213
213
|
hasSessionIds: true,
|
|
@@ -222,7 +222,6 @@ const config = {
|
|
|
222
222
|
hasCampaigns: false,
|
|
223
223
|
isAnonymous: false,
|
|
224
224
|
hasAdSpend: false,
|
|
225
|
-
percentUsersBornInDataset: 35,
|
|
226
225
|
hasAvatar: true,
|
|
227
226
|
concurrency: 1,
|
|
228
227
|
writeToDisk: false,
|
|
@@ -647,29 +646,13 @@ const config = {
|
|
|
647
646
|
// prop-setting only; no conversionRate changes
|
|
648
647
|
}
|
|
649
648
|
|
|
650
|
-
// ── HOOK 1: WEEKEND LEISURE SURGE
|
|
651
|
-
|
|
652
|
-
if (record.event === "booking completed") {
|
|
653
|
-
const dayOfWeek = dayjs(record.time).day();
|
|
654
|
-
// Friday=5, Saturday=6, Sunday=0
|
|
655
|
-
if (dayOfWeek === 0 || dayOfWeek === 5 || dayOfWeek === 6) {
|
|
656
|
-
record.nightly_rate = Math.floor((record.nightly_rate || 150) * 1.3);
|
|
657
|
-
record.total_cost = Math.floor((record.total_cost || 450) * 1.3);
|
|
658
|
-
}
|
|
659
|
-
}
|
|
649
|
+
// ── HOOK 1: WEEKEND LEISURE SURGE ────────────
|
|
650
|
+
// Moved to everything hook (sessionization reassigns times after event hook)
|
|
660
651
|
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
if (daysUntilEnd > 21) {
|
|
666
|
-
record.booking_window = "advance";
|
|
667
|
-
record.nightly_rate = Math.floor((record.nightly_rate || 150) * 0.8);
|
|
668
|
-
} else if (daysUntilEnd < 3) {
|
|
669
|
-
record.booking_window = "last_minute";
|
|
670
|
-
record.nightly_rate = Math.floor((record.nightly_rate || 150) * 1.4);
|
|
671
|
-
}
|
|
672
|
-
}
|
|
652
|
+
// ── HOOK 2: ADVANCE BOOKING DISCOUNT ─────
|
|
653
|
+
// Moved to everything hook (sessionization reassigns times after event hook)
|
|
654
|
+
if (type === "event") {
|
|
655
|
+
// no-op: DOW-dependent hooks moved to everything hook
|
|
673
656
|
}
|
|
674
657
|
|
|
675
658
|
// ── EVERYTHING HOOKS ─────────────────────────────────
|
|
@@ -689,6 +672,34 @@ const config = {
|
|
|
689
672
|
});
|
|
690
673
|
}
|
|
691
674
|
|
|
675
|
+
// ── HOOK 1: WEEKEND LEISURE SURGE ────────────
|
|
676
|
+
// Apply AFTER sessionization has finalized times
|
|
677
|
+
events.forEach(e => {
|
|
678
|
+
if (e.event === "booking completed") {
|
|
679
|
+
const dayOfWeek = new Date(e.time).getUTCDay();
|
|
680
|
+
// Friday=5, Saturday=6, Sunday=0
|
|
681
|
+
if (dayOfWeek === 0 || dayOfWeek === 5 || dayOfWeek === 6) {
|
|
682
|
+
e.nightly_rate = Math.floor((e.nightly_rate || 150) * 1.3);
|
|
683
|
+
e.total_cost = Math.floor((e.total_cost || 450) * 1.3);
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
// ── HOOK 2: ADVANCE BOOKING DISCOUNT ─────
|
|
689
|
+
events.forEach(e => {
|
|
690
|
+
if (e.event === "booking completed") {
|
|
691
|
+
const eventTime = dayjs(e.time);
|
|
692
|
+
const daysUntilEnd = NOW.diff(eventTime, "days");
|
|
693
|
+
if (daysUntilEnd > 21) {
|
|
694
|
+
e.booking_window = "advance";
|
|
695
|
+
e.nightly_rate = Math.floor((e.nightly_rate || 150) * 0.8);
|
|
696
|
+
} else if (daysUntilEnd < 3) {
|
|
697
|
+
e.booking_window = "last_minute";
|
|
698
|
+
e.nightly_rate = Math.floor((e.nightly_rate || 150) * 1.4);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
});
|
|
702
|
+
|
|
692
703
|
// ─── Bug 2 fix: Repeat destination clustering conversion filtering ───
|
|
693
704
|
// Drop ~25% of "booking completed" events for users who are NOT
|
|
694
705
|
// business_traveler or luxury_seeker to simulate their lower funnel
|
|
@@ -716,7 +727,29 @@ const config = {
|
|
|
716
727
|
}
|
|
717
728
|
|
|
718
729
|
// ── HOOK 4: CANCELLATION BY BOOKING WINDOW ───────
|
|
719
|
-
//
|
|
730
|
+
// Match each cancellation to its nearest preceding booking
|
|
731
|
+
// and copy the booking's booking_window. This replaces the
|
|
732
|
+
// random booking_window that the event config assigns to
|
|
733
|
+
// cancellation events independently.
|
|
734
|
+
const bookingsByTime = events
|
|
735
|
+
.filter(e => e.event === "booking completed")
|
|
736
|
+
.sort((a, b) => new Date(a.time) - new Date(b.time));
|
|
737
|
+
events.forEach(e => {
|
|
738
|
+
if (e.event === "booking cancelled" && bookingsByTime.length > 0) {
|
|
739
|
+
const cancelTime = new Date(e.time).getTime();
|
|
740
|
+
// Find the nearest preceding booking (or the first one)
|
|
741
|
+
let matched = bookingsByTime[0];
|
|
742
|
+
for (let b = bookingsByTime.length - 1; b >= 0; b--) {
|
|
743
|
+
if (new Date(bookingsByTime[b].time).getTime() <= cancelTime) {
|
|
744
|
+
matched = bookingsByTime[b];
|
|
745
|
+
break;
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
e.booking_window = matched.booking_window;
|
|
749
|
+
}
|
|
750
|
+
});
|
|
751
|
+
|
|
752
|
+
// Last-minute bookers rarely cancel — drop 40%.
|
|
720
753
|
for (let i = events.length - 1; i >= 0; i--) {
|
|
721
754
|
if (events[i].event === "booking cancelled" && events[i].booking_window === "last_minute") {
|
|
722
755
|
if (chance.bool({ likelihood: 40 })) {
|
|
@@ -12,6 +12,7 @@ import dayjs from "dayjs";
|
|
|
12
12
|
import { makeName } from "ak-tools";
|
|
13
13
|
import * as u from "../utils/utils.js";
|
|
14
14
|
import { resolveSoup } from "../templates/soup-presets.js";
|
|
15
|
+
import { resolveMacro } from "../templates/macro-presets.js";
|
|
15
16
|
|
|
16
17
|
/** Fixed reference point for time calculations (2024-02-02) */
|
|
17
18
|
const FIXED_NOW = dayjs('2024-02-02').unix();
|
|
@@ -87,9 +88,10 @@ export function validateDungeonConfig(config) {
|
|
|
87
88
|
// Extract configuration with defaults
|
|
88
89
|
let {
|
|
89
90
|
seed,
|
|
90
|
-
numEvents
|
|
91
|
+
numEvents,
|
|
91
92
|
numUsers = 1000,
|
|
92
93
|
numDays = 30,
|
|
94
|
+
avgEventsPerUserPerDay,
|
|
93
95
|
epochStart = 0,
|
|
94
96
|
epochEnd = dayjs().unix(),
|
|
95
97
|
events = [{ event: "foo" }, { event: "bar" }, { event: "baz" }],
|
|
@@ -148,18 +150,24 @@ export function validateDungeonConfig(config) {
|
|
|
148
150
|
);
|
|
149
151
|
}
|
|
150
152
|
|
|
151
|
-
// Auto-enable batch mode for large datasets to prevent OOM
|
|
152
|
-
if (numEvents >= 2_000_000 && config.batchSize === undefined) {
|
|
153
|
-
batchSize = 1_000_000;
|
|
154
|
-
console.warn(`⚠️ Auto-enabling batch mode: numEvents (${numEvents.toLocaleString()}) >= 2M. Using batchSize of ${batchSize.toLocaleString()}.`);
|
|
155
|
-
}
|
|
156
|
-
|
|
157
153
|
// Ensure defaults for deep objects
|
|
158
154
|
if (!config.superProps) config.superProps = superProps;
|
|
159
155
|
if (!config.userProps || Object.keys(config.userProps).length === 0) {
|
|
160
156
|
userProps = { spiritAnimal: chance.animal.bind(chance) };
|
|
161
157
|
}
|
|
162
158
|
|
|
159
|
+
// Guard against zero/negative numUsers up front (would produce NaN budgets).
|
|
160
|
+
if (!Number.isFinite(numUsers) || numUsers <= 0) {
|
|
161
|
+
throw new Error(`numUsers must be a positive number (got ${numUsers})`);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// Reject explicit zero/negative numDays from the caller before falling through
|
|
165
|
+
// to the epoch-derivation branches below (which would otherwise replace 0
|
|
166
|
+
// with a derived value or throw a less-specific message).
|
|
167
|
+
if (config.numDays !== undefined && (!Number.isFinite(config.numDays) || config.numDays <= 0)) {
|
|
168
|
+
throw new Error(`numDays must be a positive number (got ${config.numDays})`);
|
|
169
|
+
}
|
|
170
|
+
|
|
163
171
|
// Setting up "TIME"
|
|
164
172
|
if (epochStart && !numDays) numDays = dayjs.unix(epochEnd).diff(dayjs.unix(epochStart), "day");
|
|
165
173
|
if (!epochStart && numDays) epochStart = dayjs.unix(epochEnd).subtract(numDays, "day").unix();
|
|
@@ -168,15 +176,57 @@ export function validateDungeonConfig(config) {
|
|
|
168
176
|
throw new Error("Either epochStart or numDays must be provided");
|
|
169
177
|
}
|
|
170
178
|
|
|
171
|
-
//
|
|
179
|
+
// Final sanity check (e.g. epochStart/epochEnd produced a non-positive numDays).
|
|
180
|
+
if (!Number.isFinite(numDays) || numDays <= 0) {
|
|
181
|
+
throw new Error(`numDays must be a positive number (got ${numDays})`);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// Resolve event-rate primitive: avgEventsPerUserPerDay is the canonical knob.
|
|
185
|
+
// numEvents is supported as a fallback (legacy + total-volume target). Whichever
|
|
186
|
+
// is set, derive the other so downstream code (batching, progress, mixpanel-sender)
|
|
187
|
+
// can use either.
|
|
188
|
+
if (avgEventsPerUserPerDay !== undefined && numEvents !== undefined) {
|
|
189
|
+
// Both provided: avgEventsPerUserPerDay wins. Recompute numEvents from rate.
|
|
190
|
+
numEvents = Math.round(avgEventsPerUserPerDay * numUsers * numDays);
|
|
191
|
+
} else if (avgEventsPerUserPerDay !== undefined) {
|
|
192
|
+
numEvents = Math.round(avgEventsPerUserPerDay * numUsers * numDays);
|
|
193
|
+
} else if (numEvents !== undefined) {
|
|
194
|
+
avgEventsPerUserPerDay = numEvents / numUsers / numDays;
|
|
195
|
+
} else {
|
|
196
|
+
// Neither set — fall back to legacy default (100K total).
|
|
197
|
+
numEvents = 100_000;
|
|
198
|
+
avgEventsPerUserPerDay = numEvents / numUsers / numDays;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// Auto-enable batch mode for large datasets to prevent OOM.
|
|
202
|
+
// MUST run after rate→numEvents resolution above, otherwise dungeons that set
|
|
203
|
+
// only avgEventsPerUserPerDay would never trigger auto-batch.
|
|
204
|
+
if (numEvents >= 2_000_000 && config.batchSize === undefined) {
|
|
205
|
+
batchSize = 1_000_000;
|
|
206
|
+
console.warn(`⚠️ Auto-enabling batch mode: numEvents (${numEvents.toLocaleString()}) >= 2M. Using batchSize of ${batchSize.toLocaleString()}.`);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Resolve soup presets (intra-week / intra-day shape — must happen after numDays is computed)
|
|
172
210
|
const resolved = resolveSoup(soup, numDays);
|
|
173
211
|
soup = resolved.soup;
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
212
|
+
|
|
213
|
+
// Resolve macro preset (big-picture trend shape across the window).
|
|
214
|
+
// Default is "flat" — see lib/templates/macro-presets.js. Top-level
|
|
215
|
+
// bornRecentBias / percentUsersBornInDataset / preExistingSpread on the
|
|
216
|
+
// dungeon config win over the macro preset's values, so existing dungeons
|
|
217
|
+
// that set these explicitly continue to render the same way.
|
|
218
|
+
// Resolve into local vars (do NOT mutate input config).
|
|
219
|
+
const macroResolved = resolveMacro(config.macro);
|
|
220
|
+
let bornRecentBias = config.bornRecentBias !== undefined ? config.bornRecentBias : macroResolved.bornRecentBias;
|
|
221
|
+
let percentUsersBornInDataset = config.percentUsersBornInDataset !== undefined ? config.percentUsersBornInDataset : macroResolved.percentUsersBornInDataset;
|
|
222
|
+
let preExistingSpread = config.preExistingSpread !== undefined ? config.preExistingSpread : macroResolved.preExistingSpread;
|
|
223
|
+
|
|
224
|
+
// Clamp bornRecentBias to [-1, 1] — values outside this range produce
|
|
225
|
+
// nonsensical exponents (e.g. Math.pow(0, -0.4) = Infinity in user-loop.js).
|
|
226
|
+
if (typeof bornRecentBias === 'number' && Number.isFinite(bornRecentBias)) {
|
|
227
|
+
bornRecentBias = Math.max(-1, Math.min(1, bornRecentBias));
|
|
228
|
+
} else {
|
|
229
|
+
bornRecentBias = 0;
|
|
180
230
|
}
|
|
181
231
|
|
|
182
232
|
// Use provided name if non-empty string, otherwise generate one
|
|
@@ -220,7 +270,7 @@ export function validateDungeonConfig(config) {
|
|
|
220
270
|
}
|
|
221
271
|
|
|
222
272
|
// Validate: if every user is born in dataset, we need either isFirstEvent or isFirstFunnel
|
|
223
|
-
const percentBorn =
|
|
273
|
+
const percentBorn = percentUsersBornInDataset;
|
|
224
274
|
const hasFirstEvent = events.some(e => e.isFirstEvent);
|
|
225
275
|
const hasFirstFunnel = funnels.some(f => f.isFirstFunnel);
|
|
226
276
|
if (percentBorn >= 100 && !hasFirstEvent && !hasFirstFunnel) {
|
|
@@ -354,6 +404,7 @@ export function validateDungeonConfig(config) {
|
|
|
354
404
|
numEvents,
|
|
355
405
|
numUsers,
|
|
356
406
|
numDays,
|
|
407
|
+
avgEventsPerUserPerDay,
|
|
357
408
|
epochStart,
|
|
358
409
|
epochEnd,
|
|
359
410
|
events: validatedEvents,
|
|
@@ -385,6 +436,11 @@ export function validateDungeonConfig(config) {
|
|
|
385
436
|
hasIOSDevices,
|
|
386
437
|
name,
|
|
387
438
|
strictEventCount,
|
|
439
|
+
// Macro trend (resolved from preset + per-dungeon overrides; clamped)
|
|
440
|
+
macro: config.macro,
|
|
441
|
+
bornRecentBias,
|
|
442
|
+
percentUsersBornInDataset,
|
|
443
|
+
preExistingSpread,
|
|
388
444
|
// Advanced features
|
|
389
445
|
personas,
|
|
390
446
|
worldEvents,
|
package/lib/core/storage.js
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
/** @typedef {import('../../types.js').Storage} Storage */
|
|
9
9
|
/** @typedef {import('../../types.js').hookArrayOptions<any>} hookArrayOptions */
|
|
10
10
|
|
|
11
|
-
import { existsSync } from "fs";
|
|
11
|
+
import { existsSync, mkdirSync } from "fs";
|
|
12
12
|
import pLimit from 'p-limit';
|
|
13
13
|
import os from "os";
|
|
14
14
|
import path from "path";
|
|
@@ -60,8 +60,13 @@ export async function createHookArray(arr = [], opts) {
|
|
|
60
60
|
writeDir = path.resolve(os.tmpdir());
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
-
if (typeof config.writeToDisk === "string"
|
|
64
|
-
|
|
63
|
+
if (typeof config.writeToDisk === "string") {
|
|
64
|
+
if (config.writeToDisk.startsWith('gs://')) {
|
|
65
|
+
writeDir = config.writeToDisk;
|
|
66
|
+
} else {
|
|
67
|
+
writeDir = path.resolve(config.writeToDisk);
|
|
68
|
+
if (!existsSync(writeDir)) mkdirSync(writeDir, { recursive: true });
|
|
69
|
+
}
|
|
65
70
|
}
|
|
66
71
|
|
|
67
72
|
function getWritePath() {
|
|
@@ -213,7 +218,7 @@ export async function createHookArray(arr = [], opts) {
|
|
|
213
218
|
while (isWriting) {
|
|
214
219
|
await new Promise(resolve => setTimeout(resolve, 10));
|
|
215
220
|
}
|
|
216
|
-
|
|
221
|
+
|
|
217
222
|
isWriting = true;
|
|
218
223
|
try {
|
|
219
224
|
batch++;
|
|
@@ -221,6 +226,11 @@ export async function createHookArray(arr = [], opts) {
|
|
|
221
226
|
const dataToWrite = [...arr];
|
|
222
227
|
arr.length = 0; // Clear array after copying data
|
|
223
228
|
await FILE_CONN(() => writeToDisk(dataToWrite, { writePath }));
|
|
229
|
+
// Data now lives on disk, not in arr — mirror what transformThenPush
|
|
230
|
+
// does when crossing BATCH_SIZE so the Mixpanel sender knows to read
|
|
231
|
+
// from disk instead of from the (now empty) in-memory array.
|
|
232
|
+
isBatchMode = true;
|
|
233
|
+
runtime.isBatchMode = true;
|
|
224
234
|
} finally {
|
|
225
235
|
isWriting = false;
|
|
226
236
|
}
|
|
@@ -39,9 +39,11 @@ export async function userLoop(context) {
|
|
|
39
39
|
userProps,
|
|
40
40
|
scdProps,
|
|
41
41
|
numDays,
|
|
42
|
+
avgEventsPerUserPerDay,
|
|
42
43
|
percentUsersBornInDataset = 15,
|
|
43
44
|
strictEventCount = false,
|
|
44
|
-
bornRecentBias = 0
|
|
45
|
+
bornRecentBias = 0, // -1..1; positive = births skew toward end of window
|
|
46
|
+
preExistingSpread = 'uniform', // 'pinned' (FIXED_BEGIN ± 1d) | 'uniform' ([FIXED_BEGIN-30d, FIXED_BEGIN])
|
|
45
47
|
personas,
|
|
46
48
|
worldEvents,
|
|
47
49
|
engagementDecay: globalEngagementDecay,
|
|
@@ -54,7 +56,10 @@ export async function userLoop(context) {
|
|
|
54
56
|
} = config;
|
|
55
57
|
|
|
56
58
|
const { eventData, userProfilesData, scdTableData } = storage;
|
|
57
|
-
|
|
59
|
+
// Per-user-per-day rate is the canonical event-volume primitive (config-validator
|
|
60
|
+
// guarantees it is set). Each user's event count is rate × their active days, so
|
|
61
|
+
// born-late users don't compress a full per-user budget into a tiny window.
|
|
62
|
+
const ratePerDay = avgEventsPerUserPerDay ?? (numEvents / numUsers / numDays);
|
|
58
63
|
const startTime = Date.now();
|
|
59
64
|
|
|
60
65
|
// Create batches for parallel processing
|
|
@@ -160,7 +165,18 @@ export async function userLoop(context) {
|
|
|
160
165
|
// Update user.created to match biased timestamp for profile consistency
|
|
161
166
|
user.created = adjustedCreated.toISOString();
|
|
162
167
|
} else {
|
|
163
|
-
|
|
168
|
+
// Pre-existing user: choose where their "first ever event" lives.
|
|
169
|
+
// 'pinned' (legacy): exactly at FIXED_BEGIN — all pre-existing users stack
|
|
170
|
+
// at day 0, which front-loads early-window event density.
|
|
171
|
+
// 'uniform' (default): sample uniformly from [FIXED_BEGIN - 30d, FIXED_BEGIN]
|
|
172
|
+
// so pre-existing users have varied "born before dataset" timestamps.
|
|
173
|
+
// Their TimeSoup-distributed events still mostly land in [FIXED_BEGIN, FIXED_NOW].
|
|
174
|
+
if (preExistingSpread === 'uniform') {
|
|
175
|
+
const offsetDays = chance.floating({ min: 0, max: 30 });
|
|
176
|
+
adjustedCreated = dayjs.unix(context.FIXED_BEGIN).subtract(offsetDays, 'day');
|
|
177
|
+
} else {
|
|
178
|
+
adjustedCreated = dayjs.unix(context.FIXED_BEGIN);
|
|
179
|
+
}
|
|
164
180
|
}
|
|
165
181
|
|
|
166
182
|
// Feature 7: Geographic intelligence — assign sticky location
|
|
@@ -297,9 +313,17 @@ export async function userLoop(context) {
|
|
|
297
313
|
}
|
|
298
314
|
}
|
|
299
315
|
|
|
316
|
+
// User's active days = how much of the dataset window they're alive for.
|
|
317
|
+
// Pre-existing users: full window. Born-in-dataset: from birth to FIXED_NOW.
|
|
318
|
+
// Floor at 1 day so users born on the very last day still emit a few events.
|
|
319
|
+
const userActiveDays = userIsBornInDataset
|
|
320
|
+
? Math.max(1, (context.FIXED_NOW - adjustedCreated.unix()) / 86400)
|
|
321
|
+
: numDays;
|
|
322
|
+
const userEventBudget = ratePerDay * userActiveDays;
|
|
323
|
+
|
|
300
324
|
let numEventsThisUserWillPreform = Math.floor(chance.normal({
|
|
301
|
-
mean:
|
|
302
|
-
dev:
|
|
325
|
+
mean: userEventBudget,
|
|
326
|
+
dev: userEventBudget / u.integer(u.integer(2, 5), u.integer(2, 7))
|
|
303
327
|
}) * 0.714159265359);
|
|
304
328
|
|
|
305
329
|
// Power users and low-activity users logic
|
|
@@ -480,6 +504,16 @@ export async function userLoop(context) {
|
|
|
480
504
|
if (Array.isArray(newEvents)) usersEvents = newEvents;
|
|
481
505
|
}
|
|
482
506
|
|
|
507
|
+
// Defensive guard: drop any events whose timestamp landed past real "now".
|
|
508
|
+
// Mixpanel rejects future-dated events, and hooks that duplicate events with
|
|
509
|
+
// time offsets (weekend surges, viral spreads) can otherwise leak a few past
|
|
510
|
+
// the boundary even after the per-event _drop check in events.js.
|
|
511
|
+
usersEvents = usersEvents.filter(e => {
|
|
512
|
+
if (!e || !e.time) return true;
|
|
513
|
+
const t = typeof e.time === 'string' ? Date.parse(e.time) / 1000 : Number(e.time);
|
|
514
|
+
return Number.isFinite(t) ? t <= context.MAX_TIME : true;
|
|
515
|
+
});
|
|
516
|
+
|
|
483
517
|
// Store all user data
|
|
484
518
|
await userProfilesData.hookPush(profile);
|
|
485
519
|
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Macro preset configurations.
|
|
3
|
+
*
|
|
4
|
+
* Macro presets shape the BIG-PICTURE trend across the dataset window —
|
|
5
|
+
* how user births are distributed in time and what fraction of users are
|
|
6
|
+
* born inside the window vs already existing at its start. These are the
|
|
7
|
+
* knobs that determine whether the chart tilts up, tilts down, or stays flat.
|
|
8
|
+
*
|
|
9
|
+
* Soup presets (lib/templates/soup-presets.js) are the orthogonal axis:
|
|
10
|
+
* they shape intra-week and intra-day rhythm (DOW/HOD weights, peak count,
|
|
11
|
+
* deviation). The two are independent — pick a macro and a soup separately.
|
|
12
|
+
*
|
|
13
|
+
* The default macro is "flat" so a brand-new dungeon produces a steady
|
|
14
|
+
* baseline. Layer hooks on top to inject deliberate trends/spikes.
|
|
15
|
+
*
|
|
16
|
+
* Parameters:
|
|
17
|
+
* - bornRecentBias: -1..1. Negative pushes births toward dataset start,
|
|
18
|
+
* positive toward the end. Power-function shaped in user-loop.js.
|
|
19
|
+
* - percentUsersBornInDataset: 0..100. Fraction of users created inside the
|
|
20
|
+
* window. The rest are pre-existing (first event ≈ FIXED_BEGIN).
|
|
21
|
+
* - preExistingSpread: "pinned" (current behavior — pre-existing users start
|
|
22
|
+
* at FIXED_BEGIN ± 1 day) | "uniform" (sample first event time uniformly
|
|
23
|
+
* across [FIXED_BEGIN - 30d, FIXED_BEGIN] so they don't all stack at day 0).
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
/** @type {Record<string, {bornRecentBias: number, percentUsersBornInDataset: number, preExistingSpread: 'pinned'|'uniform'}>} */
|
|
27
|
+
export const MACRO_PRESETS = {
|
|
28
|
+
/**
|
|
29
|
+
* flat (DEFAULT) — Mature product, no growth narrative.
|
|
30
|
+
* Tail ratio ≈ 1.0. Pure weekly oscillation, no net drift.
|
|
31
|
+
*/
|
|
32
|
+
flat: {
|
|
33
|
+
bornRecentBias: 0,
|
|
34
|
+
percentUsersBornInDataset: 15,
|
|
35
|
+
preExistingSpread: 'uniform',
|
|
36
|
+
},
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* steady — Lightly-growing SaaS.
|
|
40
|
+
* Slight uptrend without any visible spike at the right edge.
|
|
41
|
+
*/
|
|
42
|
+
steady: {
|
|
43
|
+
bornRecentBias: 0.1,
|
|
44
|
+
percentUsersBornInDataset: 10,
|
|
45
|
+
preExistingSpread: 'uniform',
|
|
46
|
+
},
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* growth — Visible uptrend story without the meteoric blow-up.
|
|
50
|
+
* Use when the dataset wants to show clear acquisition over time.
|
|
51
|
+
*/
|
|
52
|
+
growth: {
|
|
53
|
+
bornRecentBias: 0.3,
|
|
54
|
+
percentUsersBornInDataset: 25,
|
|
55
|
+
preExistingSpread: 'pinned',
|
|
56
|
+
},
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* viral — Hockey-stick acquisition.
|
|
60
|
+
* Strong late-window ramp. Pair with persona/feature hooks for the full effect.
|
|
61
|
+
*/
|
|
62
|
+
viral: {
|
|
63
|
+
bornRecentBias: 0.6,
|
|
64
|
+
percentUsersBornInDataset: 50,
|
|
65
|
+
preExistingSpread: 'pinned',
|
|
66
|
+
},
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* decline — Sunsetting product, churning users.
|
|
70
|
+
* Few new users, those that exist are born early. Pair with churn hooks.
|
|
71
|
+
*/
|
|
72
|
+
decline: {
|
|
73
|
+
bornRecentBias: -0.3,
|
|
74
|
+
percentUsersBornInDataset: 5,
|
|
75
|
+
preExistingSpread: 'uniform',
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
/** @type {string[]} */
|
|
80
|
+
export const MACRO_PRESET_NAMES = Object.keys(MACRO_PRESETS);
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Resolve a macro config — accepts string presets, preset+overrides objects, or raw objects.
|
|
84
|
+
* Defaults to "flat" if nothing is provided.
|
|
85
|
+
*
|
|
86
|
+
* @param {string | object | undefined} macro - Macro config from dungeon
|
|
87
|
+
* @returns {{bornRecentBias: number, percentUsersBornInDataset: number, preExistingSpread: 'pinned' | 'uniform'}}
|
|
88
|
+
*/
|
|
89
|
+
export function resolveMacro(macro) {
|
|
90
|
+
if (!macro) return { ...MACRO_PRESETS.flat };
|
|
91
|
+
|
|
92
|
+
if (typeof macro === 'string') {
|
|
93
|
+
const preset = MACRO_PRESETS[macro];
|
|
94
|
+
if (!preset) {
|
|
95
|
+
throw new Error(`Unknown macro preset: "${macro}". Valid presets: ${MACRO_PRESET_NAMES.join(', ')}`);
|
|
96
|
+
}
|
|
97
|
+
return { ...preset };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (typeof macro === 'object' && macro.preset) {
|
|
101
|
+
const preset = MACRO_PRESETS[macro.preset];
|
|
102
|
+
if (!preset) {
|
|
103
|
+
throw new Error(`Unknown macro preset: "${macro.preset}". Valid presets: ${MACRO_PRESET_NAMES.join(', ')}`);
|
|
104
|
+
}
|
|
105
|
+
const { preset: _, ...overrides } = macro;
|
|
106
|
+
return { ...preset, ...overrides };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// Raw object: pass through, filling in flat defaults for missing fields
|
|
110
|
+
return { ...MACRO_PRESETS.flat, ...macro };
|
|
111
|
+
}
|