@ak--47/dungeon-master 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dungeons/technical/array-of-object-lookup.js +0 -2
- package/dungeons/technical/simple.js +3 -4
- package/dungeons/technical/simplest-schema.json +5 -0
- package/dungeons/technical/text-generation.js +1 -1
- package/dungeons/vertical/ai-platform.js +192 -133
- package/dungeons/vertical/community.js +64 -34
- package/dungeons/vertical/crypto.js +333 -224
- package/dungeons/vertical/dating.js +228 -282
- package/dungeons/vertical/devtools.js +164 -62
- package/dungeons/vertical/ecommerce.js +240 -91
- package/dungeons/vertical/education.js +328 -310
- package/dungeons/vertical/fintech.js +437 -311
- package/dungeons/vertical/fitness.js +141 -58
- package/dungeons/vertical/food-delivery.js +298 -318
- package/dungeons/vertical/gaming.js +378 -231
- package/dungeons/vertical/healthcare.js +140 -60
- package/dungeons/vertical/insurance-application.js +168 -73
- package/dungeons/vertical/logistics.js +95 -6
- package/dungeons/vertical/marketplace.js +137 -51
- package/dungeons/vertical/media.js +241 -386
- package/dungeons/vertical/real-estate.js +322 -315
- package/dungeons/vertical/sass.js +253 -263
- package/dungeons/vertical/social.js +262 -206
- package/dungeons/vertical/travel.js +59 -16
- package/index.js +17 -17
- package/lib/core/config-validator.js +93 -21
- package/lib/core/context.js +10 -24
- package/lib/core/storage.js +6 -1
- package/lib/generators/events.js +11 -14
- package/lib/generators/funnels.js +12 -4
- package/lib/generators/mirror.js +3 -2
- package/lib/generators/product-names.js +1 -1
- package/lib/generators/scd.js +2 -1
- package/lib/generators/text.js +1 -1
- package/lib/orchestrators/user-loop.js +47 -47
- package/lib/utils/utils.js +71 -39
- package/package.json +1 -1
- package/scripts/smoke-test-all.mjs +162 -0
- package/scripts/verify-runner.mjs +72 -24
- package/types.d.ts +38 -15
- package/dungeons/technical/ad-spend-schema.json +0 -128
- package/dungeons/technical/anonymous-users-schema.json +0 -92
- package/dungeons/technical/array-of-object-lookup-schema.json +0 -191
- package/dungeons/technical/experiments-schema.json +0 -203
- package/dungeons/technical/foobar-schema.json +0 -362
- package/dungeons/technical/group-analytics-schema.json +0 -241
- package/dungeons/technical/mirror-strategies-schema.json +0 -84
- package/dungeons/technical/nested-objects-schema.json +0 -145
- package/dungeons/technical/retention-cadence-schema.json +0 -37
- package/dungeons/technical/sanity-schema.json +0 -185
- package/dungeons/technical/scale-test-schema.json +0 -70
- package/dungeons/technical/scd-schema.json +0 -467
- package/dungeons/technical/simple-schema.json +0 -362
- package/dungeons/technical/text-generation-schema.json +0 -1062
- package/dungeons/vertical/ai-platform-schema.json +0 -617
- package/dungeons/vertical/community-schema.json +0 -579
- package/dungeons/vertical/crypto-schema.json +0 -546
- package/dungeons/vertical/dating-schema.json +0 -401
- package/dungeons/vertical/devtools-schema.json +0 -601
- package/dungeons/vertical/ecommerce-schema.json +0 -604
- package/dungeons/vertical/education-schema.json +0 -5686
- package/dungeons/vertical/fintech-schema.json +0 -630
- package/dungeons/vertical/fitness-schema.json +0 -530
- package/dungeons/vertical/food-delivery-schema.json +0 -36728
- package/dungeons/vertical/gaming-schema.json +0 -2703
- package/dungeons/vertical/healthcare-schema.json +0 -549
- package/dungeons/vertical/insurance-application-schema.json +0 -485
- package/dungeons/vertical/logistics-schema.json +0 -574
- package/dungeons/vertical/marketplace-schema.json +0 -533
- package/dungeons/vertical/media-schema.json +0 -4749
- package/dungeons/vertical/real-estate-schema.json +0 -527
- package/dungeons/vertical/sass-schema.json +0 -3128
- package/dungeons/vertical/social-schema.json +0 -620
- package/dungeons/vertical/travel-schema.json +0 -580
|
@@ -108,7 +108,7 @@ export async function userLoop(context) {
|
|
|
108
108
|
}
|
|
109
109
|
|
|
110
110
|
const userId = chance.guid();
|
|
111
|
-
const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds });
|
|
111
|
+
const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix: context.FIXED_NOW });
|
|
112
112
|
const { distinct_id, created } = user;
|
|
113
113
|
const userIsBornInDataset = chance.bool({ likelihood: percentUsersBornInDataset });
|
|
114
114
|
|
|
@@ -128,14 +128,13 @@ export async function userLoop(context) {
|
|
|
128
128
|
|
|
129
129
|
if (!userIsBornInDataset) delete user.created;
|
|
130
130
|
|
|
131
|
-
//
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
// Apply recency bias to birth dates for users born in dataset
|
|
131
|
+
// Apply recency bias to birth dates for users born in dataset.
|
|
132
|
+
// `created` (from generateUser → person) is anchored to the dataset end,
|
|
133
|
+
// so it already lives inside [FIXED_BEGIN, FIXED_NOW]. No shift needed.
|
|
135
134
|
// bornRecentBias: 0 = uniform distribution, 1 = heavily biased toward recent
|
|
136
135
|
let adjustedCreated;
|
|
137
136
|
if (userIsBornInDataset) {
|
|
138
|
-
let biasedCreated = dayjs(created)
|
|
137
|
+
let biasedCreated = dayjs(created);
|
|
139
138
|
|
|
140
139
|
if (bornRecentBias !== 0) {
|
|
141
140
|
// Calculate how far into the dataset this user was born (0 = start, 1 = end/recent)
|
|
@@ -282,7 +281,9 @@ export async function userLoop(context) {
|
|
|
282
281
|
user,
|
|
283
282
|
config,
|
|
284
283
|
userIsBornInDataset,
|
|
285
|
-
persona
|
|
284
|
+
persona,
|
|
285
|
+
datasetStart: context.DATASET_START_SECONDS,
|
|
286
|
+
datasetEnd: context.DATASET_END_SECONDS
|
|
286
287
|
});
|
|
287
288
|
}
|
|
288
289
|
|
|
@@ -304,7 +305,9 @@ export async function userLoop(context) {
|
|
|
304
305
|
type: 'user',
|
|
305
306
|
scd: { [key]: scdProps[key] },
|
|
306
307
|
config,
|
|
307
|
-
allSCDs: userSCD
|
|
308
|
+
allSCDs: userSCD,
|
|
309
|
+
datasetStart: context.DATASET_START_SECONDS,
|
|
310
|
+
datasetEnd: context.DATASET_END_SECONDS
|
|
308
311
|
});
|
|
309
312
|
if (Array.isArray(hookResult)) {
|
|
310
313
|
changes = hookResult;
|
|
@@ -373,8 +376,7 @@ export async function userLoop(context) {
|
|
|
373
376
|
const [data, converted] = await makeFunnel(context, firstFunnel, user, firstTime, profile, userSCD, persona, featureCtx);
|
|
374
377
|
userConverted = converted;
|
|
375
378
|
|
|
376
|
-
|
|
377
|
-
userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
|
|
379
|
+
userFirstEventTime = dayjs(data[0].time).unix();
|
|
378
380
|
numEventsPreformed += data.length;
|
|
379
381
|
usersEvents = usersEvents.concat(data);
|
|
380
382
|
} else {
|
|
@@ -423,10 +425,8 @@ export async function userLoop(context) {
|
|
|
423
425
|
// Subscription events (trial, upgrade, cancel) must not be randomly dropped by decay
|
|
424
426
|
const userDecay = persona?.engagementDecay || globalEngagementDecay;
|
|
425
427
|
if (userDecay && userDecay.model !== 'none' && usersEvents.length > 0) {
|
|
426
|
-
// adjustedCreated
|
|
427
|
-
|
|
428
|
-
const adjustedCreatedPresent = adjustedCreated.add(context.TIME_SHIFT_SECONDS, 'seconds');
|
|
429
|
-
usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreatedPresent, context, chance);
|
|
428
|
+
// adjustedCreated and event times now share the same dataset window — no shift.
|
|
429
|
+
usersEvents = applyEngagementDecay(usersEvents, userDecay, adjustedCreated, context, chance);
|
|
430
430
|
}
|
|
431
431
|
|
|
432
432
|
// Feature 5: Subscription lifecycle — inject after decay (exempt from decay filtering)
|
|
@@ -485,9 +485,7 @@ export async function userLoop(context) {
|
|
|
485
485
|
earliestTime: userFirstEventTime,
|
|
486
486
|
latestTime: context.FIXED_NOW,
|
|
487
487
|
peaks: soupPeaks, deviation: soupDev, mean: soupMean,
|
|
488
|
-
dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD
|
|
489
|
-
timeShiftSeconds: context.TIME_SHIFT_SECONDS,
|
|
490
|
-
maxTime: context.MAX_TIME
|
|
488
|
+
dayOfWeekWeights: soupDOW, hourOfDayWeights: soupHOD
|
|
491
489
|
});
|
|
492
490
|
u.assignSessionIds(usersEvents, sessionTimeout);
|
|
493
491
|
}
|
|
@@ -499,19 +497,20 @@ export async function userLoop(context) {
|
|
|
499
497
|
scd: userSCD,
|
|
500
498
|
config,
|
|
501
499
|
userIsBornInDataset,
|
|
502
|
-
persona
|
|
500
|
+
persona,
|
|
501
|
+
datasetStart: context.DATASET_START_SECONDS,
|
|
502
|
+
datasetEnd: context.DATASET_END_SECONDS
|
|
503
503
|
});
|
|
504
504
|
if (Array.isArray(newEvents)) usersEvents = newEvents;
|
|
505
505
|
}
|
|
506
506
|
|
|
507
|
-
// Defensive guard: drop any events whose timestamp landed past
|
|
508
|
-
//
|
|
509
|
-
//
|
|
510
|
-
// the boundary even after the per-event _drop check in events.js.
|
|
507
|
+
// Defensive guard: drop any events whose timestamp landed past the
|
|
508
|
+
// configured dataset end. Hooks that duplicate events with time offsets
|
|
509
|
+
// (weekend surges, viral spreads) can leak a few past the boundary.
|
|
511
510
|
usersEvents = usersEvents.filter(e => {
|
|
512
511
|
if (!e || !e.time) return true;
|
|
513
512
|
const t = typeof e.time === 'string' ? Date.parse(e.time) / 1000 : Number(e.time);
|
|
514
|
-
return Number.isFinite(t) ? t <= context.
|
|
513
|
+
return Number.isFinite(t) ? t <= context.FIXED_NOW : true;
|
|
515
514
|
});
|
|
516
515
|
|
|
517
516
|
// Store all user data
|
|
@@ -628,8 +627,7 @@ function generateSubscriptionEvents(subscription, user, persona, userCreated, co
|
|
|
628
627
|
const paidPlans = plans.filter(p => p.price > 0);
|
|
629
628
|
|
|
630
629
|
const userStartUnix = dayjs(userCreated).unix();
|
|
631
|
-
const endUnix = context.
|
|
632
|
-
const timeShift = context.TIME_SHIFT_SECONDS;
|
|
630
|
+
const endUnix = context.FIXED_NOW;
|
|
633
631
|
|
|
634
632
|
// Persona modifiers for subscription behavior
|
|
635
633
|
const personaChurnMod = persona?.churnRate ? (1 + persona.churnRate) : 1.0;
|
|
@@ -642,14 +640,14 @@ function generateSubscriptionEvents(subscription, user, persona, userCreated, co
|
|
|
642
640
|
const firstPaidPlan = paidPlans[0];
|
|
643
641
|
if (firstPaidPlan && firstPaidPlan.trialDays) {
|
|
644
642
|
const trialStart = currentUnix + chance.integer({ min: 0, max: 86400 });
|
|
645
|
-
if (trialStart
|
|
646
|
-
subEvents.push(makeSubEvent(eventNames.trialStarted, trialStart
|
|
643
|
+
if (trialStart < endUnix) {
|
|
644
|
+
subEvents.push(makeSubEvent(eventNames.trialStarted, trialStart, user, firstPaidPlan.name, 'trial'));
|
|
647
645
|
currentUnix = trialStart + (firstPaidPlan.trialDays * 86400);
|
|
648
646
|
|
|
649
647
|
// Trial to paid conversion
|
|
650
648
|
if (chance.bool({ likelihood: lc.trialToPayRate * personaUpgradeMod * 100 })) {
|
|
651
|
-
if (currentUnix
|
|
652
|
-
subEvents.push(makeSubEvent(eventNames.subscribed, currentUnix
|
|
649
|
+
if (currentUnix < endUnix) {
|
|
650
|
+
subEvents.push(makeSubEvent(eventNames.subscribed, currentUnix, user, firstPaidPlan.name, 'active'));
|
|
653
651
|
currentPlan = firstPaidPlan;
|
|
654
652
|
currentStatus = 'active';
|
|
655
653
|
}
|
|
@@ -661,15 +659,15 @@ function generateSubscriptionEvents(subscription, user, persona, userCreated, co
|
|
|
661
659
|
}
|
|
662
660
|
|
|
663
661
|
// Monthly lifecycle loop
|
|
664
|
-
while (currentUnix
|
|
662
|
+
while (currentUnix < endUnix) {
|
|
665
663
|
currentUnix += monthSeconds + chance.integer({ min: -86400, max: 86400 });
|
|
666
|
-
if (currentUnix
|
|
664
|
+
if (currentUnix >= endUnix) break;
|
|
667
665
|
if (currentStatus === 'cancelled') {
|
|
668
666
|
// Win-back check
|
|
669
667
|
if (chance.bool({ likelihood: lc.winBackRate * 100 })) {
|
|
670
668
|
currentUnix += lc.winBackDelay * 86400;
|
|
671
|
-
if (currentUnix
|
|
672
|
-
subEvents.push(makeSubEvent(eventNames.wonBack, currentUnix
|
|
669
|
+
if (currentUnix >= endUnix) break;
|
|
670
|
+
subEvents.push(makeSubEvent(eventNames.wonBack, currentUnix, user, currentPlan.name, 'active'));
|
|
673
671
|
currentStatus = 'active';
|
|
674
672
|
}
|
|
675
673
|
break;
|
|
@@ -677,12 +675,12 @@ function generateSubscriptionEvents(subscription, user, persona, userCreated, co
|
|
|
677
675
|
|
|
678
676
|
// Payment failure
|
|
679
677
|
if (currentPlan.price > 0 && chance.bool({ likelihood: lc.paymentFailureRate * 100 })) {
|
|
680
|
-
subEvents.push(makeSubEvent(eventNames.paymentFailed, currentUnix
|
|
678
|
+
subEvents.push(makeSubEvent(eventNames.paymentFailed, currentUnix, user, currentPlan.name, 'payment_issue'));
|
|
681
679
|
}
|
|
682
680
|
|
|
683
681
|
// Churn
|
|
684
682
|
if (chance.bool({ likelihood: lc.churnRate * personaChurnMod * 100 })) {
|
|
685
|
-
subEvents.push(makeSubEvent(eventNames.cancelled, currentUnix
|
|
683
|
+
subEvents.push(makeSubEvent(eventNames.cancelled, currentUnix, user, currentPlan.name, 'cancelled'));
|
|
686
684
|
currentStatus = 'cancelled';
|
|
687
685
|
continue;
|
|
688
686
|
}
|
|
@@ -691,7 +689,7 @@ function generateSubscriptionEvents(subscription, user, persona, userCreated, co
|
|
|
691
689
|
const currentPlanIndex = plans.indexOf(currentPlan);
|
|
692
690
|
if (currentPlanIndex < plans.length - 1 && chance.bool({ likelihood: lc.upgradeRate * personaUpgradeMod * 100 })) {
|
|
693
691
|
const newPlan = plans[currentPlanIndex + 1];
|
|
694
|
-
subEvents.push(makeSubEvent(eventNames.upgraded, currentUnix
|
|
692
|
+
subEvents.push(makeSubEvent(eventNames.upgraded, currentUnix, user, newPlan.name, 'active', currentPlan.name));
|
|
695
693
|
currentPlan = newPlan;
|
|
696
694
|
continue;
|
|
697
695
|
}
|
|
@@ -699,14 +697,14 @@ function generateSubscriptionEvents(subscription, user, persona, userCreated, co
|
|
|
699
697
|
// Downgrade
|
|
700
698
|
if (currentPlanIndex > 0 && currentPlan.price > 0 && chance.bool({ likelihood: lc.downgradeRate * 100 })) {
|
|
701
699
|
const newPlan = plans[currentPlanIndex - 1];
|
|
702
|
-
subEvents.push(makeSubEvent(eventNames.downgraded, currentUnix
|
|
700
|
+
subEvents.push(makeSubEvent(eventNames.downgraded, currentUnix, user, newPlan.name, 'active', currentPlan.name));
|
|
703
701
|
currentPlan = newPlan;
|
|
704
702
|
continue;
|
|
705
703
|
}
|
|
706
704
|
|
|
707
705
|
// Renewal
|
|
708
706
|
if (currentPlan.price > 0) {
|
|
709
|
-
subEvents.push(makeSubEvent(eventNames.renewed, currentUnix
|
|
707
|
+
subEvents.push(makeSubEvent(eventNames.renewed, currentUnix, user, currentPlan.name, 'active'));
|
|
710
708
|
}
|
|
711
709
|
}
|
|
712
710
|
|
|
@@ -714,9 +712,12 @@ function generateSubscriptionEvents(subscription, user, persona, userCreated, co
|
|
|
714
712
|
}
|
|
715
713
|
|
|
716
714
|
function makeSubEvent(eventName, unixTime, user, planName, status, previousPlan) {
|
|
715
|
+
// Callers already guard `unixTime < endUnix` before invoking, so no clamp here.
|
|
716
|
+
// Previous code clamped against `dayjs()` (wall-clock now) — leaked wall-clock into
|
|
717
|
+
// timestamps and wasn't even the right upper bound.
|
|
717
718
|
const ev = {
|
|
718
719
|
event: eventName,
|
|
719
|
-
time: dayjs.unix(
|
|
720
|
+
time: dayjs.unix(unixTime).toISOString(),
|
|
720
721
|
user_id: user.distinct_id,
|
|
721
722
|
insert_id: u.quickHash(`${eventName}-${unixTime}-${user.distinct_id}`),
|
|
722
723
|
subscription_plan: planName,
|
|
@@ -742,14 +743,14 @@ async function generateBotUsers(context, dataQuality, storage) {
|
|
|
742
743
|
const botId = `bot_${chance.guid().slice(0, 8)}`;
|
|
743
744
|
const botEvents = [];
|
|
744
745
|
|
|
745
|
-
// Bots generate events at machine-like intervals
|
|
746
|
-
let currentTime = context.FIXED_BEGIN
|
|
747
|
-
const interval = Math.floor(((context.
|
|
746
|
+
// Bots generate events at machine-like intervals across the dataset window
|
|
747
|
+
let currentTime = context.FIXED_BEGIN;
|
|
748
|
+
const interval = Math.floor(((context.FIXED_NOW - currentTime) / botEventsPerUser));
|
|
748
749
|
|
|
749
750
|
for (let e = 0; e < botEventsPerUser; e++) {
|
|
750
751
|
const chosenEvent = botEventTypes[e % botEventTypes.length];
|
|
751
752
|
currentTime += interval + chance.integer({ min: 0, max: 10 });
|
|
752
|
-
if (currentTime > context.
|
|
753
|
+
if (currentTime > context.FIXED_NOW) break;
|
|
753
754
|
botEvents.push({
|
|
754
755
|
event: chosenEvent.event,
|
|
755
756
|
time: dayjs.unix(currentTime).toISOString(),
|
|
@@ -777,20 +778,19 @@ async function generateBotUsers(context, dataQuality, storage) {
|
|
|
777
778
|
*/
|
|
778
779
|
async function generateAnomalyBursts(context, anomalies, storage) {
|
|
779
780
|
const chance = u.getChance();
|
|
780
|
-
const timeShift = context.TIME_SHIFT_SECONDS;
|
|
781
781
|
|
|
782
782
|
for (const a of anomalies) {
|
|
783
783
|
if (a.type !== 'burst' && a.type !== 'coordinated') continue;
|
|
784
784
|
if (!a._startUnix || !a.count) continue;
|
|
785
785
|
|
|
786
786
|
const burstEvents = [];
|
|
787
|
-
const startUnix = a._startUnix
|
|
788
|
-
const endUnix = a._endUnix
|
|
787
|
+
const startUnix = a._startUnix;
|
|
788
|
+
const endUnix = a._endUnix;
|
|
789
789
|
const windowSeconds = endUnix - startUnix;
|
|
790
790
|
|
|
791
791
|
for (let i = 0; i < a.count; i++) {
|
|
792
792
|
const eventTime = startUnix + chance.integer({ min: 0, max: Math.max(1, windowSeconds) });
|
|
793
|
-
if (eventTime > context.
|
|
793
|
+
if (eventTime > context.FIXED_NOW) continue;
|
|
794
794
|
const userId = a.type === 'coordinated'
|
|
795
795
|
? `anomaly_${chance.guid().slice(0, 8)}`
|
|
796
796
|
: `burst_${chance.integer({ min: 1, max: 100 })}`;
|
package/lib/utils/utils.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import Chance from 'chance';
|
|
3
3
|
import readline from 'readline';
|
|
4
|
-
import { comma
|
|
4
|
+
import { comma } from 'ak-tools';
|
|
5
5
|
import dayjs from 'dayjs';
|
|
6
6
|
import utc from 'dayjs/plugin/utc.js';
|
|
7
7
|
import path from 'path';
|
|
@@ -24,7 +24,27 @@ const { NODE_ENV = "unknown" } = process.env;
|
|
|
24
24
|
let globalChance;
|
|
25
25
|
let chanceInitialized = false;
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
// Module-scoped memoization cache for weighted-array resolvers in `choose()`.
|
|
28
|
+
// Lives for the lifetime of the node process; key is the function source string.
|
|
29
|
+
const weightedArrayCache = new Map();
|
|
30
|
+
|
|
31
|
+
// Reference "now" used by date() and day() factories. Defaults to wall-clock at
|
|
32
|
+
// import time; the orchestrator overrides this via setDatasetNow() once the dataset
|
|
33
|
+
// window is resolved, so date helpers in dungeon configs produce deterministic
|
|
34
|
+
// values relative to the dataset end (not the process start).
|
|
35
|
+
let DATASET_NOW = dayjs.utc();
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Override the reference "now" used by date()/day() factories. Called by the
|
|
39
|
+
* orchestrator with the resolved dataset end so every date helper produces
|
|
40
|
+
* deterministic values relative to the dataset window.
|
|
41
|
+
* @param {number} unixSeconds
|
|
42
|
+
*/
|
|
43
|
+
function setDatasetNow(unixSeconds) {
|
|
44
|
+
if (typeof unixSeconds === 'number' && Number.isFinite(unixSeconds)) {
|
|
45
|
+
DATASET_NOW = dayjs.unix(unixSeconds).utc();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
28
48
|
|
|
29
49
|
|
|
30
50
|
import { Storage as cloudStorage } from '@google-cloud/storage';
|
|
@@ -102,7 +122,7 @@ function pick(items) {
|
|
|
102
122
|
*/
|
|
103
123
|
function date(inTheLast = 30, isPast = true, format = 'YYYY-MM-DD') {
|
|
104
124
|
const chance = getChance();
|
|
105
|
-
const now =
|
|
125
|
+
const now = DATASET_NOW;
|
|
106
126
|
if (Math.abs(inTheLast) > 365 * 10) inTheLast = chance.integer({ min: 1, max: 180 });
|
|
107
127
|
return function () {
|
|
108
128
|
const when = chance.integer({ min: 0, max: Math.abs(inTheLast) });
|
|
@@ -159,8 +179,8 @@ function datesBetween(start, end) {
|
|
|
159
179
|
*/
|
|
160
180
|
function day(start, end) {
|
|
161
181
|
// if (!end) end = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
162
|
-
if (!start) start =
|
|
163
|
-
if (!end) end =
|
|
182
|
+
if (!start) start = DATASET_NOW.subtract(30, 'd').toISOString();
|
|
183
|
+
if (!end) end = DATASET_NOW.toISOString();
|
|
164
184
|
const chance = getChance();
|
|
165
185
|
const format = 'YYYY-MM-DD';
|
|
166
186
|
return function (min, max) {
|
|
@@ -219,20 +239,15 @@ function choose(value) {
|
|
|
219
239
|
while (typeof value === 'function') {
|
|
220
240
|
const funcString = value.toString();
|
|
221
241
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
global.weightedArrayCache = new Map();
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
if (global.weightedArrayCache.has(funcString)) {
|
|
228
|
-
value = global.weightedArrayCache.get(funcString);
|
|
242
|
+
if (weightedArrayCache.has(funcString)) {
|
|
243
|
+
value = weightedArrayCache.get(funcString);
|
|
229
244
|
break;
|
|
230
245
|
}
|
|
231
246
|
|
|
232
247
|
const result = value();
|
|
233
248
|
if (Array.isArray(result) && result.length > 10) {
|
|
234
249
|
// Cache large arrays (likely weighted arrays)
|
|
235
|
-
|
|
250
|
+
weightedArrayCache.set(funcString, result);
|
|
236
251
|
}
|
|
237
252
|
value = result;
|
|
238
253
|
}
|
|
@@ -1143,7 +1158,7 @@ CORE
|
|
|
1143
1158
|
//the function which generates $distinct_id + $anonymous_ids, $session_ids, and created, skewing towards the present
|
|
1144
1159
|
function generateUser(user_id, opts, amplitude = 1, frequency = 1, skew = 1) {
|
|
1145
1160
|
const chance = getChance();
|
|
1146
|
-
const { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds } = opts;
|
|
1161
|
+
const { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix } = opts;
|
|
1147
1162
|
// Uniformly distributed `u`, then skew applied
|
|
1148
1163
|
let u = Math.pow(chance.random(), skew);
|
|
1149
1164
|
|
|
@@ -1155,7 +1170,7 @@ function generateUser(user_id, opts, amplitude = 1, frequency = 1, skew = 1) {
|
|
|
1155
1170
|
|
|
1156
1171
|
// Clamp values to ensure they are within the desired range
|
|
1157
1172
|
daysAgoBorn = Math.min(daysAgoBorn, numDays);
|
|
1158
|
-
const props = person(user_id, daysAgoBorn, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds);
|
|
1173
|
+
const props = person(user_id, daysAgoBorn, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds, datasetEndUnix);
|
|
1159
1174
|
|
|
1160
1175
|
const user = {
|
|
1161
1176
|
distinct_id: user_id,
|
|
@@ -1188,7 +1203,7 @@ const DEFAULT_HOD_WEIGHTS = [
|
|
|
1188
1203
|
0.584, 0.574, 0.554, 0.576, 0.604, 0.655, 0.722, 0.816
|
|
1189
1204
|
];
|
|
1190
1205
|
|
|
1191
|
-
function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0, dayOfWeekWeights = DEFAULT_DOW_WEIGHTS, hourOfDayWeights = DEFAULT_HOD_WEIGHTS
|
|
1206
|
+
function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0, dayOfWeekWeights = DEFAULT_DOW_WEIGHTS, hourOfDayWeights = DEFAULT_HOD_WEIGHTS) {
|
|
1192
1207
|
if (!earliestTime) earliestTime = global.FIXED_BEGIN ? global.FIXED_BEGIN : dayjs().subtract(30, 'd').unix();
|
|
1193
1208
|
if (!latestTime) latestTime = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
1194
1209
|
const chance = getChance();
|
|
@@ -1215,7 +1230,7 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0,
|
|
|
1215
1230
|
// Phase 2: DOW accept/reject — retry if day-of-week doesn't pass weight check
|
|
1216
1231
|
if (dayOfWeekWeights) {
|
|
1217
1232
|
for (let attempt = 0; attempt < 50; attempt++) {
|
|
1218
|
-
const dow = new Date(
|
|
1233
|
+
const dow = new Date(candidate * 1000).getUTCDay();
|
|
1219
1234
|
if (chance.random() < dayOfWeekWeights[dow]) break;
|
|
1220
1235
|
// Rejected — resample from Gaussian chunks
|
|
1221
1236
|
const pi = integer(0, peaks - 1);
|
|
@@ -1231,8 +1246,7 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0,
|
|
|
1231
1246
|
|
|
1232
1247
|
// Phase 3: Redistribute hour-of-day (changes only hour within same day)
|
|
1233
1248
|
if (hourOfDayWeights) {
|
|
1234
|
-
const
|
|
1235
|
-
const d = new Date(shifted * 1000);
|
|
1249
|
+
const d = new Date(candidate * 1000);
|
|
1236
1250
|
const currentMinute = d.getUTCMinutes();
|
|
1237
1251
|
const currentSecond = d.getUTCSeconds();
|
|
1238
1252
|
|
|
@@ -1244,9 +1258,8 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0,
|
|
|
1244
1258
|
if (roll <= 0) { newHour = h; break; }
|
|
1245
1259
|
}
|
|
1246
1260
|
|
|
1247
|
-
const
|
|
1248
|
-
|
|
1249
|
-
candidate = newShifted - timeShiftSeconds;
|
|
1261
|
+
const dayStart = Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()) / 1000;
|
|
1262
|
+
candidate = dayStart + newHour * 3600 + currentMinute * 60 + currentSecond;
|
|
1250
1263
|
candidate = Math.max(earliestTime, Math.min(latestTime, candidate));
|
|
1251
1264
|
}
|
|
1252
1265
|
|
|
@@ -1264,7 +1277,7 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0,
|
|
|
1264
1277
|
* @param {boolean} hasSessionIds
|
|
1265
1278
|
* @return {Person}
|
|
1266
1279
|
*/
|
|
1267
|
-
function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false, hasAnonIds = false, hasSessionIds = false) {
|
|
1280
|
+
function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false, hasAnonIds = false, hasSessionIds = false, datasetEndUnix) {
|
|
1268
1281
|
const chance = getChance();
|
|
1269
1282
|
//names and photos
|
|
1270
1283
|
const l = chance.letter.bind(chance);
|
|
@@ -1278,7 +1291,11 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false
|
|
|
1278
1291
|
let randomAvatarNumber = integer(1, 99);
|
|
1279
1292
|
let avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
|
|
1280
1293
|
let avatar = avatarPrefix + avPath;
|
|
1281
|
-
|
|
1294
|
+
// Birth date: anchor to dataset end (post-resolution) so user.created lives inside
|
|
1295
|
+
// the configured window, not wall-clock time. Falls back to dayjs() only when
|
|
1296
|
+
// person() is called outside the normal generation pipeline (e.g. direct unit tests).
|
|
1297
|
+
const anchor = datasetEndUnix ? dayjs.unix(datasetEndUnix) : dayjs();
|
|
1298
|
+
let created = anchor.subtract(bornDaysAgo, 'day').format('YYYY-MM-DD');
|
|
1282
1299
|
|
|
1283
1300
|
|
|
1284
1301
|
// const created = date(bornDaysAgo, true)();
|
|
@@ -1307,7 +1324,8 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false
|
|
|
1307
1324
|
if (hasAnonIds) {
|
|
1308
1325
|
const clusterSize = integer(2, 10);
|
|
1309
1326
|
for (let i = 0; i < clusterSize; i++) {
|
|
1310
|
-
|
|
1327
|
+
// Use seeded chance, not ak-tools uid() (which uses Math.random).
|
|
1328
|
+
const anonId = chance.string({ length: 42, pool: 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' });
|
|
1311
1329
|
user.anonymousIds.push(anonId);
|
|
1312
1330
|
}
|
|
1313
1331
|
}
|
|
@@ -1429,11 +1447,26 @@ function deepClone(thing, opts) {
|
|
|
1429
1447
|
|
|
1430
1448
|
|
|
1431
1449
|
/**
|
|
1432
|
-
* Generates a session ID in the standard format
|
|
1450
|
+
* Generates a session ID in the standard format. Derives it deterministically
|
|
1451
|
+
* from a seed string (first event time + user_id, typically) via quickHash so
|
|
1452
|
+
* we don't consume from the main seeded RNG stream — that consumption would
|
|
1453
|
+
* cascade into all downstream events whenever the number of sessions changed
|
|
1454
|
+
* (e.g. when sessionTimeout differs).
|
|
1455
|
+
* @param {string} [seedStr] - Stable input string. If absent, falls back to a
|
|
1456
|
+
* chance.string() pull (only used by tests / direct callers without a seed).
|
|
1433
1457
|
* @returns {string} Session ID like "xxxxx-xxxxx-xxxxx-xxxxx"
|
|
1434
1458
|
*/
|
|
1435
|
-
function generateSessionId() {
|
|
1436
|
-
|
|
1459
|
+
function generateSessionId(seedStr) {
|
|
1460
|
+
if (seedStr) {
|
|
1461
|
+
// quickHash returns ~16 hex chars; expand to 4×5 segments via re-hashing.
|
|
1462
|
+
const h1 = quickHash(seedStr);
|
|
1463
|
+
const h2 = quickHash(h1);
|
|
1464
|
+
const all = (h1 + h2).replace(/[^a-zA-Z0-9]/g, '').padEnd(20, '0');
|
|
1465
|
+
return [all.slice(0, 5), all.slice(5, 10), all.slice(10, 15), all.slice(15, 20)].join('-');
|
|
1466
|
+
}
|
|
1467
|
+
const c = getChance();
|
|
1468
|
+
const seg = () => c.string({ length: 5, pool: 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' });
|
|
1469
|
+
return [seg(), seg(), seg(), seg()].join("-");
|
|
1437
1470
|
}
|
|
1438
1471
|
|
|
1439
1472
|
/**
|
|
@@ -1459,7 +1492,7 @@ function bunchIntoSessions(events, timeoutMinutes, soupParams) {
|
|
|
1459
1492
|
|
|
1460
1493
|
const chance = getChance();
|
|
1461
1494
|
const { earliestTime, latestTime, peaks, deviation, mean,
|
|
1462
|
-
dayOfWeekWeights, hourOfDayWeights
|
|
1495
|
+
dayOfWeekWeights, hourOfDayWeights } = soupParams;
|
|
1463
1496
|
|
|
1464
1497
|
// Sort by time first
|
|
1465
1498
|
events.sort((a, b) => a.time < b.time ? -1 : a.time > b.time ? 1 : 0);
|
|
@@ -1472,8 +1505,8 @@ function bunchIntoSessions(events, timeoutMinutes, soupParams) {
|
|
|
1472
1505
|
const anchors = [];
|
|
1473
1506
|
for (let i = 0; i < numSessions; i++) {
|
|
1474
1507
|
const soupTime = TimeSoup(earliestTime, latestTime, peaks, deviation, mean,
|
|
1475
|
-
dayOfWeekWeights, hourOfDayWeights
|
|
1476
|
-
anchors.push(soupTime
|
|
1508
|
+
dayOfWeekWeights, hourOfDayWeights);
|
|
1509
|
+
anchors.push(soupTime);
|
|
1477
1510
|
}
|
|
1478
1511
|
anchors.sort((a, b) => a - b);
|
|
1479
1512
|
|
|
@@ -1485,7 +1518,6 @@ function bunchIntoSessions(events, timeoutMinutes, soupParams) {
|
|
|
1485
1518
|
}
|
|
1486
1519
|
|
|
1487
1520
|
// Retime events within each session
|
|
1488
|
-
let writeIndex = 0;
|
|
1489
1521
|
for (let s = 0; s < numSessions; s++) {
|
|
1490
1522
|
const bucket = sessionBuckets[s];
|
|
1491
1523
|
if (bucket.length === 0) continue;
|
|
@@ -1493,17 +1525,13 @@ function bunchIntoSessions(events, timeoutMinutes, soupParams) {
|
|
|
1493
1525
|
let currentTime = anchors[s];
|
|
1494
1526
|
for (let e = 0; e < bucket.length; e++) {
|
|
1495
1527
|
const ev = bucket[e];
|
|
1496
|
-
const clampedTime = Math.min(currentTime,
|
|
1528
|
+
const clampedTime = Math.min(currentTime, latestTime);
|
|
1497
1529
|
|
|
1498
1530
|
ev.time = dayjs.unix(clampedTime).toISOString();
|
|
1499
1531
|
// Regenerate insert_id to match new time
|
|
1500
1532
|
const distinctId = ev.user_id || ev.device_id || ev.distinct_id || '';
|
|
1501
1533
|
ev.insert_id = quickHash(`${ev.event}-${ev.time}-${distinctId}`);
|
|
1502
1534
|
|
|
1503
|
-
if (currentTime > maxTime) {
|
|
1504
|
-
ev._drop = true;
|
|
1505
|
-
}
|
|
1506
|
-
|
|
1507
1535
|
// Advance time within session: 5-300 seconds (5s to 5min)
|
|
1508
1536
|
currentTime += chance.integer({ min: 5, max: 300 });
|
|
1509
1537
|
}
|
|
@@ -1532,7 +1560,10 @@ function assignSessionIds(events, timeoutMinutes = 30) {
|
|
|
1532
1560
|
const timeoutMs = timeoutMinutes * 60 * 1000;
|
|
1533
1561
|
const maxSessionMs = 24 * 60 * 60 * 1000;
|
|
1534
1562
|
|
|
1535
|
-
|
|
1563
|
+
// Derive a stable seed from the first event so session IDs are deterministic
|
|
1564
|
+
// across runs without consuming the main seeded RNG stream.
|
|
1565
|
+
const userKey = events[0].user_id || events[0].device_id || events[0].distinct_id || '';
|
|
1566
|
+
let currentSessionId = generateSessionId(`${userKey}-${events[0].time}`);
|
|
1536
1567
|
let sessionStartMs = new Date(events[0].time).getTime();
|
|
1537
1568
|
let lastEventMs = sessionStartMs;
|
|
1538
1569
|
|
|
@@ -1542,7 +1573,7 @@ function assignSessionIds(events, timeoutMinutes = 30) {
|
|
|
1542
1573
|
const sessionDuration = eventMs - sessionStartMs;
|
|
1543
1574
|
|
|
1544
1575
|
if (gapFromLast > timeoutMs || sessionDuration > maxSessionMs) {
|
|
1545
|
-
currentSessionId = generateSessionId();
|
|
1576
|
+
currentSessionId = generateSessionId(`${userKey}-${event.time}`);
|
|
1546
1577
|
sessionStartMs = eventMs;
|
|
1547
1578
|
}
|
|
1548
1579
|
|
|
@@ -1606,4 +1637,5 @@ export {
|
|
|
1606
1637
|
generateSessionId,
|
|
1607
1638
|
assignSessionIds,
|
|
1608
1639
|
bunchIntoSessions,
|
|
1640
|
+
setDatasetNow,
|
|
1609
1641
|
};
|