@ak--47/dungeon-master 1.4.5 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +158 -0
- package/.claude/skills/create-dungeon/SKILL.md +464 -0
- package/.claude/skills/verify-dungeon/SKILL.md +157 -0
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +161 -0
- package/.claude/skills/verify-dungeon/references/report-format.md +216 -0
- package/.claude/skills/verify-dungeon/references/sql-recipes.md +857 -0
- package/.claude/skills/write-hooks/SKILL.md +468 -0
- package/CHANGELOG.md +182 -0
- package/HOOKS.md +1256 -597
- package/README.md +140 -5
- package/dungeons/technical/ad-spend.js +41 -49
- package/dungeons/technical/anonymous-users.js +38 -36
- package/dungeons/technical/array-of-object-lookup.js +136 -153
- package/dungeons/technical/datagen-v15-verify.js +87 -0
- package/dungeons/technical/experiments.js +42 -40
- package/dungeons/technical/foobar.js +114 -118
- package/dungeons/technical/group-analytics.js +42 -40
- package/dungeons/technical/hook-helpers-verify.js +69 -50
- package/dungeons/technical/identity-model-verify.js +22 -12
- package/dungeons/technical/mirror-strategies.js +37 -39
- package/dungeons/technical/nested-objects.js +119 -118
- package/dungeons/technical/pattern-aggregate-by-bin.js +21 -8
- package/dungeons/technical/pattern-attributed-by-source.js +23 -9
- package/dungeons/technical/pattern-frequency-by-frequency.js +21 -8
- package/dungeons/technical/pattern-funnel-frequency.js +30 -15
- package/dungeons/technical/pattern-ttc-by-segment.js +21 -8
- package/dungeons/technical/retention-cadence.js +115 -112
- package/dungeons/technical/sanity.js +86 -80
- package/dungeons/technical/scale-test.js +34 -38
- package/dungeons/technical/scd.js +111 -128
- package/dungeons/technical/simple.js +134 -141
- package/dungeons/technical/simplest.js +111 -65
- package/dungeons/technical/text-generation.js +110 -146
- package/dungeons/vertical/ai-platform.js +300 -333
- package/dungeons/vertical/community.js +290 -255
- package/dungeons/vertical/crypto.js +400 -391
- package/dungeons/vertical/dating.js +421 -375
- package/dungeons/vertical/devtools.js +346 -298
- package/dungeons/vertical/ecommerce.js +322 -394
- package/dungeons/vertical/education.js +380 -325
- package/dungeons/vertical/fintech.js +371 -325
- package/dungeons/vertical/fitness.js +345 -291
- package/dungeons/vertical/food-delivery.js +352 -307
- package/dungeons/vertical/gaming.js +490 -444
- package/dungeons/vertical/healthcare.js +311 -262
- package/dungeons/vertical/insurance-application.js +437 -409
- package/dungeons/vertical/logistics.js +278 -252
- package/dungeons/vertical/marketplace.js +340 -323
- package/dungeons/vertical/media.js +390 -335
- package/dungeons/vertical/real-estate.js +402 -347
- package/dungeons/vertical/sass.js +331 -333
- package/dungeons/vertical/social.js +377 -316
- package/dungeons/vertical/travel.js +302 -295
- package/index.js +64 -7
- package/lib/core/config-validator.js +378 -17
- package/lib/core/dungeon-loader.js +2 -5
- package/lib/generators/events.js +12 -13
- package/lib/generators/funnels.js +76 -2
- package/lib/hook-helpers/index.js +1 -0
- package/lib/hook-helpers/inject.js +95 -0
- package/lib/orchestrators/mixpanel-sender.js +7 -0
- package/lib/orchestrators/user-loop.js +598 -48
- package/lib/templates/defaults.js +59 -59
- package/lib/templates/macro-presets.js +53 -11
- package/lib/utils/dataset-context.js +103 -0
- package/lib/utils/retention-curve.js +140 -0
- package/lib/utils/utils.js +157 -109
- package/lib/verify/counting.js +360 -0
- package/lib/verify/emulate-breakdown.js +531 -108
- package/lib/verify/funnel-engine.js +539 -0
- package/lib/verify/identity.js +78 -0
- package/lib/verify/index.js +20 -0
- package/lib/verify/schema-validator.js +3 -1
- package/lib/verify/verify-dungeon.js +58 -0
- package/package.json +14 -3
- package/scripts/run-dungeon.mjs +12 -1
- package/types.d.ts +353 -4
- package/scripts/smoke-test-all.mjs +0 -162
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v1.5.1: AsyncLocalStorage-scoped dataset-window context.
|
|
3
|
+
*
|
|
4
|
+
* Replaces the legacy module-scoped `DATASET_NOW` / `DATASET_BEGIN` mutable
|
|
5
|
+
* state in `lib/utils/utils.js`. Each `generate()` call wraps the pipeline in
|
|
6
|
+
* `runWithDataset(begin, now, fn)`, and the factory thunks (`date`, `day`,
|
|
7
|
+
* `dateRange`, `TimeSoup`, `validTime`) read from the ALS store via
|
|
8
|
+
* `getDatasetNow()` / `getDatasetBegin()` instead of module globals.
|
|
9
|
+
*
|
|
10
|
+
* This makes concurrent in-process `generate()` calls safe with respect to
|
|
11
|
+
* the dataset-window dimension. RNG scoping is a SEPARATE concurrency hole
|
|
12
|
+
* — see TODOs.md "Considered but punted" for the follow-up plan.
|
|
13
|
+
*
|
|
14
|
+
* Trip-ups (see plans/archived/globals-killplan-1.5.1/kill-globals.md §7):
|
|
15
|
+
* - ALS does NOT propagate across child_process.fork / worker_threads.
|
|
16
|
+
* - Tests that import dungeon configs at top-level evaluate thunks at
|
|
17
|
+
* import time — those thunks fall back to wall-clock if invoked outside
|
|
18
|
+
* a `runWithDataset` scope.
|
|
19
|
+
* - Callbacks scheduled via setImmediate / setTimeout / native event
|
|
20
|
+
* emitters MAY lose context. Audit at the orchestrator level.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
24
|
+
import dayjs from 'dayjs';
|
|
25
|
+
import utc from 'dayjs/plugin/utc.js';
|
|
26
|
+
dayjs.extend(utc);
|
|
27
|
+
|
|
28
|
+
/** @type {AsyncLocalStorage<{ begin: number, now: number }>} */
|
|
29
|
+
const datasetALS = new AsyncLocalStorage();
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Legacy fallback — written by the deprecated `setDatasetNow` / `setDatasetBegin`
|
|
33
|
+
* shims in `lib/utils/utils.js`. Used ONLY when no ALS scope is active. This
|
|
34
|
+
* keeps existing tests that call the setters directly (instead of wrapping in
|
|
35
|
+
* `runWithDataset`) functional through v1.5.1. v1.6 may remove the shims.
|
|
36
|
+
*
|
|
37
|
+
* @type {{ begin: number|null, now: number|null }}
|
|
38
|
+
*/
|
|
39
|
+
const legacyFallback = { begin: null, now: null };
|
|
40
|
+
|
|
41
|
+
/** @internal — used by `setDatasetNow` / `setDatasetBegin` shims. */
|
|
42
|
+
export function _setLegacyDatasetNow(unixSeconds) {
|
|
43
|
+
if (typeof unixSeconds === 'number' && Number.isFinite(unixSeconds)) {
|
|
44
|
+
legacyFallback.now = unixSeconds;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** @internal — used by `setDatasetNow` / `setDatasetBegin` shims. */
|
|
49
|
+
export function _setLegacyDatasetBegin(unixSeconds) {
|
|
50
|
+
if (typeof unixSeconds === 'number' && Number.isFinite(unixSeconds)) {
|
|
51
|
+
legacyFallback.begin = unixSeconds;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Run `fn` inside a dataset-window scope. Factory thunks invoked anywhere in
|
|
57
|
+
* the async call chain (sync or via `await`) will read the scoped window
|
|
58
|
+
* instead of the module fallback.
|
|
59
|
+
*
|
|
60
|
+
* @template T
|
|
61
|
+
* @param {number} datasetBeginUnix
|
|
62
|
+
* @param {number} datasetNowUnix
|
|
63
|
+
* @param {() => T} fn
|
|
64
|
+
* @returns {T}
|
|
65
|
+
*/
|
|
66
|
+
export function runWithDataset(datasetBeginUnix, datasetNowUnix, fn) {
|
|
67
|
+
return datasetALS.run({ begin: datasetBeginUnix, now: datasetNowUnix }, fn);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Read the scoped dataset-now value. Resolution order:
|
|
72
|
+
* 1. ALS store (active `runWithDataset` scope) — production pipeline path.
|
|
73
|
+
* 2. Legacy fallback (set via deprecated `setDatasetNow` shim) — for tests.
|
|
74
|
+
* 3. Wall-clock `dayjs.utc()` — last-resort default.
|
|
75
|
+
*
|
|
76
|
+
* @returns {dayjs.Dayjs}
|
|
77
|
+
*/
|
|
78
|
+
export function getDatasetNow() {
|
|
79
|
+
const store = datasetALS.getStore();
|
|
80
|
+
if (store) return dayjs.unix(store.now).utc();
|
|
81
|
+
if (legacyFallback.now !== null) return dayjs.unix(legacyFallback.now).utc();
|
|
82
|
+
return dayjs.utc();
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Read the scoped dataset-begin value. Same resolution as `getDatasetNow`,
|
|
87
|
+
* falling back to wall-clock minus 30 days.
|
|
88
|
+
*
|
|
89
|
+
* @returns {dayjs.Dayjs}
|
|
90
|
+
*/
|
|
91
|
+
export function getDatasetBegin() {
|
|
92
|
+
const store = datasetALS.getStore();
|
|
93
|
+
if (store) return dayjs.unix(store.begin).utc();
|
|
94
|
+
if (legacyFallback.begin !== null) return dayjs.unix(legacyFallback.begin).utc();
|
|
95
|
+
return dayjs.utc().subtract(30, 'day');
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* @returns {boolean} true when called inside a runWithDataset scope.
|
|
100
|
+
*/
|
|
101
|
+
export function hasDatasetScope() {
|
|
102
|
+
return datasetALS.getStore() !== undefined;
|
|
103
|
+
}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v1.5.1: retention-curve weight interpolation.
|
|
3
|
+
*
|
|
4
|
+
* A `retentionCurve` config knob defines a target retention shape via anchor
|
|
5
|
+
* points (e.g. `day1: 0.40, day7: 0.20, day30: 0.08`). This module turns those
|
|
6
|
+
* anchors into a function `(dayOffset) → weight` used by `buildActiveDayPlan`
|
|
7
|
+
* in `lib/orchestrators/user-loop.js` to bias active-day selection.
|
|
8
|
+
*
|
|
9
|
+
* Interpolation modes:
|
|
10
|
+
* - 'logarithmic' (default) — log-linear between anchors. Matches typical
|
|
11
|
+
* real-world retention decay (large drop early, gentle tail).
|
|
12
|
+
* - 'linear' — straight-line interpolation. Simpler shape; rarely matches
|
|
13
|
+
* real data but useful for testing.
|
|
14
|
+
*
|
|
15
|
+
* Anchor semantics:
|
|
16
|
+
* - dayN keys (`day1`, `day7`, `day30`, etc.) → user's relative fraction
|
|
17
|
+
* active on day N from their birth. Values in [0, 1].
|
|
18
|
+
* - Day 0 (birth day) is implicitly 1.0 (every user is active on their birth
|
|
19
|
+
* day — that's when their first event fires). Not a configurable anchor.
|
|
20
|
+
* - Days beyond the largest anchor extrapolate from the last segment's slope.
|
|
21
|
+
*
|
|
22
|
+
* The curve is NOT normalized into a probability distribution — values
|
|
23
|
+
* directly weight the day-selection sampler. A user with curve sum ≈ 5 will
|
|
24
|
+
* average ~5 active days regardless of the dataset window size.
|
|
25
|
+
*
|
|
26
|
+
* Mixpanel parity note: this models the GENERATOR side. The
|
|
27
|
+
* `emulateBreakdown({ type: 'retention' })` verifier measures per-event
|
|
28
|
+
* retention using Mixpanel's bucketing rule (return_ms - birth_ms / DAY_MS).
|
|
29
|
+
* Generator controls DAYS active; verifier reads EVENTS. Round-trip
|
|
30
|
+
* verification (test-retention-curve.test.js) shows the per-day curve drives
|
|
31
|
+
* the per-event retention shape to within ±5%.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* @typedef {Object} RetentionCurveConfig
|
|
36
|
+
* @property {('logarithmic'|'linear')} [type='logarithmic']
|
|
37
|
+
* @property {number} [day1]
|
|
38
|
+
* @property {number} [day3]
|
|
39
|
+
* @property {number} [day7]
|
|
40
|
+
* @property {number} [day14]
|
|
41
|
+
* @property {number} [day30]
|
|
42
|
+
* @property {number} [day60]
|
|
43
|
+
* @property {number} [day90]
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Extract `dayN` anchors from a curve config, sorted by N ascending.
|
|
48
|
+
* Always includes (0, 1.0) — the birth-day anchor.
|
|
49
|
+
*
|
|
50
|
+
* @param {RetentionCurveConfig} curve
|
|
51
|
+
* @returns {Array<{ day: number, weight: number }>}
|
|
52
|
+
*/
|
|
53
|
+
export function extractAnchors(curve) {
|
|
54
|
+
if (!curve || typeof curve !== 'object') return [{ day: 0, weight: 1 }];
|
|
55
|
+
const anchors = [{ day: 0, weight: 1 }];
|
|
56
|
+
for (const key of Object.keys(curve)) {
|
|
57
|
+
const m = /^day(\d+)$/.exec(key);
|
|
58
|
+
if (!m) continue;
|
|
59
|
+
const day = Number(m[1]);
|
|
60
|
+
const weight = Number(curve[key]);
|
|
61
|
+
if (!Number.isFinite(day) || day < 1) continue;
|
|
62
|
+
if (!Number.isFinite(weight) || weight < 0) continue;
|
|
63
|
+
anchors.push({ day, weight });
|
|
64
|
+
}
|
|
65
|
+
anchors.sort((a, b) => a.day - b.day);
|
|
66
|
+
return anchors;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Build a weight function `(dayOffset) → weight` from a curve config. Defaults
|
|
71
|
+
* to logarithmic interpolation; pass `type: 'linear'` for straight-line.
|
|
72
|
+
*
|
|
73
|
+
* @param {RetentionCurveConfig} curve
|
|
74
|
+
* @returns {(dayOffset: number) => number}
|
|
75
|
+
*/
|
|
76
|
+
export function buildCurveWeightFn(curve) {
|
|
77
|
+
const anchors = extractAnchors(curve);
|
|
78
|
+
const mode = (curve && curve.type === 'linear') ? 'linear' : 'logarithmic';
|
|
79
|
+
|
|
80
|
+
if (anchors.length === 1) {
|
|
81
|
+
// Only day-0 anchor — return constant 1.0 (effectively legacy uniform).
|
|
82
|
+
return () => 1;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return function weightForDay(dayOffset) {
|
|
86
|
+
if (!Number.isFinite(dayOffset) || dayOffset < 0) return 0;
|
|
87
|
+
if (dayOffset <= anchors[0].day) return anchors[0].weight;
|
|
88
|
+
if (dayOffset >= anchors[anchors.length - 1].day) {
|
|
89
|
+
// Extrapolate from the last segment.
|
|
90
|
+
if (anchors.length === 1) return anchors[0].weight;
|
|
91
|
+
const last = anchors[anchors.length - 1];
|
|
92
|
+
const prev = anchors[anchors.length - 2];
|
|
93
|
+
return Math.max(0, interpolate(prev, last, dayOffset, mode));
|
|
94
|
+
}
|
|
95
|
+
// Find the bracketing anchors and interpolate.
|
|
96
|
+
for (let i = 0; i < anchors.length - 1; i++) {
|
|
97
|
+
if (dayOffset >= anchors[i].day && dayOffset <= anchors[i + 1].day) {
|
|
98
|
+
return interpolate(anchors[i], anchors[i + 1], dayOffset, mode);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return 0;
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Log-linear or linear interpolation between two anchors `a` and `b`.
|
|
107
|
+
*
|
|
108
|
+
* @param {{ day: number, weight: number }} a
|
|
109
|
+
* @param {{ day: number, weight: number }} b
|
|
110
|
+
* @param {number} x
|
|
111
|
+
* @param {'logarithmic'|'linear'} mode
|
|
112
|
+
* @returns {number}
|
|
113
|
+
*/
|
|
114
|
+
function interpolate(a, b, x, mode) {
|
|
115
|
+
if (a.day === b.day) return (a.weight + b.weight) / 2;
|
|
116
|
+
if (mode === 'logarithmic' && a.weight > 0 && b.weight > 0 && a.day > 0 && b.day > 0 && x > 0) {
|
|
117
|
+
// y = y_a * (y_b / y_a) ^ ((log(x) - log(x_a)) / (log(x_b) - log(x_a)))
|
|
118
|
+
const t = (Math.log(x) - Math.log(a.day)) / (Math.log(b.day) - Math.log(a.day));
|
|
119
|
+
return a.weight * Math.pow(b.weight / a.weight, t);
|
|
120
|
+
}
|
|
121
|
+
// Linear fallback (or when log doesn't apply because day=0 / weight=0).
|
|
122
|
+
const t = (x - a.day) / (b.day - a.day);
|
|
123
|
+
return a.weight + t * (b.weight - a.weight);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Convenience: derive the expected active-day count from a curve over a span
|
|
128
|
+
* of `maxDayOffset` days. Sum of weights across days [0, maxDayOffset].
|
|
129
|
+
*
|
|
130
|
+
* @param {RetentionCurveConfig} curve
|
|
131
|
+
* @param {number} maxDayOffset - Number of days to sum over (exclusive upper).
|
|
132
|
+
* @returns {number}
|
|
133
|
+
*/
|
|
134
|
+
export function expectedActiveDays(curve, maxDayOffset) {
|
|
135
|
+
const fn = buildCurveWeightFn(curve);
|
|
136
|
+
let sum = 0;
|
|
137
|
+
const span = Math.max(0, Math.floor(maxDayOffset));
|
|
138
|
+
for (let d = 0; d < span; d++) sum += fn(d);
|
|
139
|
+
return sum;
|
|
140
|
+
}
|
package/lib/utils/utils.js
CHANGED
|
@@ -24,29 +24,56 @@ const { NODE_ENV = "unknown" } = process.env;
|
|
|
24
24
|
let globalChance;
|
|
25
25
|
let chanceInitialized = false;
|
|
26
26
|
|
|
27
|
+
// v1.5.1: Optional second Chance instance dedicated to user-id generation.
|
|
28
|
+
// When `Dungeon.userSeed` is set, the orchestrator calls `initUserChance(userSeed)`
|
|
29
|
+
// to bind this — `chance.guid()` for `distinct_id` (user-loop.js) then uses it,
|
|
30
|
+
// allowing two runs with the SAME `userSeed` but DIFFERENT `seed` to produce the
|
|
31
|
+
// SAME user pool with DIFFERENT events. Used by sharded/cloud-run dungeons that
|
|
32
|
+
// need cross-shard user identity (e.g., dungeons/user/kodiak).
|
|
33
|
+
// Falls back to `globalChance` when never initialized — so all existing dungeons
|
|
34
|
+
// behave unchanged.
|
|
35
|
+
let globalUserChance;
|
|
36
|
+
let userChanceInitialized = false;
|
|
37
|
+
|
|
27
38
|
// Module-scoped memoization cache for weighted-array resolvers in `choose()`.
|
|
28
39
|
// Lives for the lifetime of the node process; key is the function source string.
|
|
29
40
|
const weightedArrayCache = new Map();
|
|
30
41
|
|
|
31
|
-
//
|
|
32
|
-
//
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
|
|
36
|
-
|
|
42
|
+
// v1.5.1: dataset-window state moved to AsyncLocalStorage scope
|
|
43
|
+
// (`lib/utils/dataset-context.js`). Each `generate()` call wraps the pipeline
|
|
44
|
+
// in `runWithDataset(begin, now, fn)`, and factory thunks read via the
|
|
45
|
+
// `getDatasetNow()` / `getDatasetBegin()` getters below. Concurrent in-process
|
|
46
|
+
// `generate()` calls no longer clobber each other's window.
|
|
47
|
+
//
|
|
48
|
+
// `setDatasetNow` / `setDatasetBegin` remain as DEPRECATED shims that write to
|
|
49
|
+
// a legacy fallback in dataset-context.js — kept so existing tests that call
|
|
50
|
+
// the setters directly (instead of wrapping in `runWithDataset`) continue
|
|
51
|
+
// working through v1.5.1. The getters check ALS first, then the legacy
|
|
52
|
+
// fallback, then wall-clock.
|
|
53
|
+
import {
|
|
54
|
+
getDatasetNow as _alsGetDatasetNow,
|
|
55
|
+
getDatasetBegin as _alsGetDatasetBegin,
|
|
56
|
+
_setLegacyDatasetNow,
|
|
57
|
+
_setLegacyDatasetBegin,
|
|
58
|
+
} from './dataset-context.js';
|
|
37
59
|
|
|
38
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* @deprecated v1.5.1 — use `runWithDataset(begin, now, fn)` from
|
|
62
|
+
* `lib/utils/dataset-context.js` to scope the dataset window per
|
|
63
|
+
* `generate()` call. The setter is kept as a back-compat shim that writes
|
|
64
|
+
* to a legacy fallback store; ALS scopes always win when active.
|
|
65
|
+
* @param {number} unixSeconds
|
|
66
|
+
*/
|
|
39
67
|
function setDatasetNow(unixSeconds) {
|
|
40
|
-
|
|
41
|
-
DATASET_NOW = dayjs.unix(unixSeconds).utc();
|
|
42
|
-
}
|
|
68
|
+
_setLegacyDatasetNow(unixSeconds);
|
|
43
69
|
}
|
|
44
70
|
|
|
45
|
-
/**
|
|
71
|
+
/**
|
|
72
|
+
* @deprecated v1.5.1 — see `setDatasetNow` for migration.
|
|
73
|
+
* @param {number} unixSeconds
|
|
74
|
+
*/
|
|
46
75
|
function setDatasetBegin(unixSeconds) {
|
|
47
|
-
|
|
48
|
-
DATASET_BEGIN = dayjs.unix(unixSeconds).utc();
|
|
49
|
-
}
|
|
76
|
+
_setLegacyDatasetBegin(unixSeconds);
|
|
50
77
|
}
|
|
51
78
|
|
|
52
79
|
|
|
@@ -57,9 +84,50 @@ class ListValue extends Array {
|
|
|
57
84
|
}
|
|
58
85
|
}
|
|
59
86
|
|
|
60
|
-
import { Storage as cloudStorage } from '@google-cloud/storage';
|
|
87
|
+
import { Storage as cloudStorage, IdempotencyStrategy } from '@google-cloud/storage';
|
|
61
88
|
const projectId = 'YOUR_PROJECT_ID';
|
|
62
|
-
|
|
89
|
+
// v1.5.1: aggressive retry config. Default SDK gives up after ~3 retries —
|
|
90
|
+
// not enough at large parallel scale. The kodiak Cloud Run Job (1000 concurrent
|
|
91
|
+
// tasks all uploading to GCS) saw "Retry limit exceeded" on ECONNRESET errors,
|
|
92
|
+
// killing entire tasks (each costing ~$0.14 of wasted compute). These settings
|
|
93
|
+
// retry up to 10 times with exponential back-off over a 10-min total budget,
|
|
94
|
+
// covering ECONNRESET / ETIMEDOUT / EAI_AGAIN / 5xx responses.
|
|
95
|
+
//
|
|
96
|
+
// `IdempotencyStrategy.RetryConditional` (the SDK default) only retries when
|
|
97
|
+
// the SDK can verify the operation didn't apply. Resumable upload chunk PUTs
|
|
98
|
+
// are idempotent at the session/offset level, so they retry safely. We tried
|
|
99
|
+
// `RetryAlways` once but it caused "offset is lower than bytes written" by
|
|
100
|
+
// re-sending already-committed chunks — the conditional strategy is correct.
|
|
101
|
+
const storage = new cloudStorage({
|
|
102
|
+
projectId,
|
|
103
|
+
retryOptions: {
|
|
104
|
+
autoRetry: true,
|
|
105
|
+
maxRetries: 10,
|
|
106
|
+
retryDelayMultiplier: 2,
|
|
107
|
+
totalTimeout: 600, // 10-min total budget per request
|
|
108
|
+
maxRetryDelay: 64, // cap each individual back-off at 64 sec
|
|
109
|
+
idempotencyStrategy: IdempotencyStrategy.RetryConditional,
|
|
110
|
+
},
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
// v1.5.1: GCS write options for ALL createWriteStream calls below.
|
|
114
|
+
//
|
|
115
|
+
// `resumable: true` is the SDK default and Google's recommendation for objects
|
|
116
|
+
// >8 MB. Our gzip batches land at ~80 MB compressed; we previously forced
|
|
117
|
+
// `resumable: false` to dodge a different bug (RetryAlways re-sending committed
|
|
118
|
+
// chunks → "offset is lower than bytes written"). Forcing multipart uploads
|
|
119
|
+
// made every batch one fragile 80 MB POST: any TCP hiccup mid-upload killed the
|
|
120
|
+
// whole transfer and bubbled up as ECONNRESET. With `RetryConditional` (above)
|
|
121
|
+
// the offset bug is gone — resumable is back to its proper default.
|
|
122
|
+
//
|
|
123
|
+
// `chunkSize: 8 MiB` enables true chunked resumable upload: the SDK PUTs each
|
|
124
|
+
// 8 MB slice via the resumable session URL; ECONNRESET kills only the in-flight
|
|
125
|
+
// chunk, the SDK queries the session for the committed offset and resumes.
|
|
126
|
+
// 8 MiB is a multiple of GCS's 256 KiB chunk-alignment requirement (32 × 256 KiB).
|
|
127
|
+
const GCS_WRITE_OPTS = {
|
|
128
|
+
resumable: true,
|
|
129
|
+
chunkSize: 8 * 1024 * 1024,
|
|
130
|
+
};
|
|
63
131
|
|
|
64
132
|
|
|
65
133
|
/*
|
|
@@ -96,6 +164,40 @@ function getChance() {
|
|
|
96
164
|
return globalChance;
|
|
97
165
|
}
|
|
98
166
|
|
|
167
|
+
/**
|
|
168
|
+
* Initialize the user-id RNG with a separate seed. Called by the orchestrator
|
|
169
|
+
* when `Dungeon.userSeed` is set (or env `USER_SEED`).
|
|
170
|
+
* @param {string} seed
|
|
171
|
+
* @returns {Chance}
|
|
172
|
+
*/
|
|
173
|
+
function initUserChance(seed) {
|
|
174
|
+
if (!seed && process.env.USER_SEED) seed = process.env.USER_SEED;
|
|
175
|
+
globalUserChance = new Chance(seed);
|
|
176
|
+
userChanceInitialized = true;
|
|
177
|
+
return globalUserChance;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Clear any previously-initialized user RNG so `getUserChance()` falls back to
|
|
182
|
+
* `getChance()` again. Called by the orchestrator at the start of every run
|
|
183
|
+
* when `Dungeon.userSeed` is NOT set, so state from a prior in-process run
|
|
184
|
+
* (e.g., a previous vitest spec) doesn't leak into the next.
|
|
185
|
+
*/
|
|
186
|
+
function resetUserChance() {
|
|
187
|
+
globalUserChance = undefined;
|
|
188
|
+
userChanceInitialized = false;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Returns the user-id RNG. Falls back to the event RNG (`getChance()`) when
|
|
193
|
+
* `userSeed` was not configured, so existing dungeons stay byte-identical.
|
|
194
|
+
* @returns {Chance}
|
|
195
|
+
*/
|
|
196
|
+
function getUserChance() {
|
|
197
|
+
if (!userChanceInitialized) return getChance();
|
|
198
|
+
return globalUserChance;
|
|
199
|
+
}
|
|
200
|
+
|
|
99
201
|
/*
|
|
100
202
|
----
|
|
101
203
|
PICKERS
|
|
@@ -132,9 +234,12 @@ function pick(items) {
|
|
|
132
234
|
*/
|
|
133
235
|
function date(inTheLast = 30, isPast = true, format = 'YYYY-MM-DD') {
|
|
134
236
|
const chance = getChance();
|
|
135
|
-
const now = DATASET_NOW;
|
|
136
237
|
if (Math.abs(inTheLast) > 365 * 10) inTheLast = chance.integer({ min: 1, max: 180 });
|
|
137
238
|
return function () {
|
|
239
|
+
// v1.5.1: read DATASET_NOW per-invocation (inside the thunk) so the value
|
|
240
|
+
// reflects the ALS scope active at event-generation time, not the wall-clock
|
|
241
|
+
// captured when the dungeon module was loaded.
|
|
242
|
+
const now = _alsGetDatasetNow();
|
|
138
243
|
const when = chance.integer({ min: 0, max: Math.abs(inTheLast) });
|
|
139
244
|
let then;
|
|
140
245
|
if (isPast) {
|
|
@@ -188,20 +293,22 @@ function datesBetween(start, end) {
|
|
|
188
293
|
* @param {any} end
|
|
189
294
|
*/
|
|
190
295
|
function day(start, end) {
|
|
191
|
-
// if (!end) end = global.FIXED_NOW ? global.FIXED_NOW : dayjs().unix();
|
|
192
|
-
if (!start) start = DATASET_NOW.subtract(30, 'd').toISOString();
|
|
193
|
-
if (!end) end = DATASET_NOW.toISOString();
|
|
194
296
|
const chance = getChance();
|
|
195
297
|
const format = 'YYYY-MM-DD';
|
|
298
|
+
const startArg = start;
|
|
299
|
+
const endArg = end;
|
|
196
300
|
return function (min, max) {
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
const
|
|
301
|
+
// v1.5.1: read DATASET_NOW per-invocation so missing args resolve against
|
|
302
|
+
// the active ALS scope, not the wall-clock at factory-call time.
|
|
303
|
+
const now = _alsGetDatasetNow();
|
|
304
|
+
const sResolved = startArg != null ? dayjs(startArg) : dayjs(now.subtract(30, 'd').toISOString());
|
|
305
|
+
const eResolved = endArg != null ? dayjs(endArg) : dayjs(now.toISOString());
|
|
306
|
+
const diff = eResolved.diff(sResolved, 'day');
|
|
200
307
|
const delta = chance.integer({ min: min, max: diff });
|
|
201
|
-
const day =
|
|
308
|
+
const day = sResolved.add(delta, 'day');
|
|
202
309
|
return {
|
|
203
|
-
start:
|
|
204
|
-
end:
|
|
310
|
+
start: sResolved.format(format),
|
|
311
|
+
end: eResolved.format(format),
|
|
205
312
|
day: day.format(format)
|
|
206
313
|
};
|
|
207
314
|
};
|
|
@@ -217,8 +324,8 @@ function day(start, end) {
|
|
|
217
324
|
function dateRange(start, end, format = 'YYYY-MM-DDTHH:mm:ss') {
|
|
218
325
|
return function () {
|
|
219
326
|
const chance = getChance();
|
|
220
|
-
const s = start != null ? dayjs(typeof start === 'number' ? start * 1000 : start).utc() :
|
|
221
|
-
const e = end != null ? dayjs(typeof end === 'number' ? end * 1000 : end).utc() :
|
|
327
|
+
const s = start != null ? dayjs(typeof start === 'number' ? start * 1000 : start).utc() : _alsGetDatasetBegin();
|
|
328
|
+
const e = end != null ? dayjs(typeof end === 'number' ? end * 1000 : end).utc() : _alsGetDatasetNow();
|
|
222
329
|
const diffSec = e.diff(s, 'second');
|
|
223
330
|
const offsetSec = chance.integer({ min: 0, max: Math.max(0, diffSec) });
|
|
224
331
|
const result = s.add(offsetSec, 'second');
|
|
@@ -579,7 +686,7 @@ function streamJSON(filePath, data, options = {}) {
|
|
|
579
686
|
|
|
580
687
|
if (filePath?.startsWith('gs://')) {
|
|
581
688
|
const { uri, bucket, file } = parseGCSUri(filePath);
|
|
582
|
-
const gcsStream = storage.bucket(bucket).file(file).createWriteStream();
|
|
689
|
+
const gcsStream = storage.bucket(bucket).file(file).createWriteStream(GCS_WRITE_OPTS);
|
|
583
690
|
gcsStream.on('finish', () => resolve(filePath));
|
|
584
691
|
gcsStream.on('error', reject);
|
|
585
692
|
if (gzip) {
|
|
@@ -616,7 +723,7 @@ function streamCSV(filePath, data, options = {}) {
|
|
|
616
723
|
|
|
617
724
|
if (filePath?.startsWith('gs://')) {
|
|
618
725
|
const { uri, bucket, file } = parseGCSUri(filePath);
|
|
619
|
-
const gcsStream = storage.bucket(bucket).file(file).createWriteStream();
|
|
726
|
+
const gcsStream = storage.bucket(bucket).file(file).createWriteStream(GCS_WRITE_OPTS);
|
|
620
727
|
gcsStream.on('finish', () => resolve(filePath));
|
|
621
728
|
gcsStream.on('error', reject);
|
|
622
729
|
if (gzip) {
|
|
@@ -731,7 +838,7 @@ async function streamParquet(filePath, data, options = {}) {
|
|
|
731
838
|
// @ts-ignore
|
|
732
839
|
const arrayBuffer = parquetWriteBuffer({ columnData });
|
|
733
840
|
const { bucket, file } = parseGCSUri(filePath);
|
|
734
|
-
const gcsStream = storage.bucket(bucket).file(file).createWriteStream();
|
|
841
|
+
const gcsStream = storage.bucket(bucket).file(file).createWriteStream(GCS_WRITE_OPTS);
|
|
735
842
|
|
|
736
843
|
return new Promise((resolve, reject) => {
|
|
737
844
|
gcsStream.on('finish', () => resolve(filePath));
|
|
@@ -1063,8 +1170,10 @@ function validateEventConfig(events) {
|
|
|
1063
1170
|
}
|
|
1064
1171
|
|
|
1065
1172
|
function validTime(chosenTime, earliestTime, latestTime) {
|
|
1066
|
-
|
|
1067
|
-
|
|
1173
|
+
// Fallback: ALS-scoped DATASET_BEGIN/NOW (v1.5.1: was module-scoped, now
|
|
1174
|
+
// resolved per `generate()` call via `runWithDataset` — see dataset-context.js).
|
|
1175
|
+
if (!earliestTime) earliestTime = _alsGetDatasetBegin().unix();
|
|
1176
|
+
if (!latestTime) latestTime = _alsGetDatasetNow().unix();
|
|
1068
1177
|
|
|
1069
1178
|
if (typeof chosenTime === 'number') {
|
|
1070
1179
|
if (chosenTime > 0) {
|
|
@@ -1286,8 +1395,10 @@ const DEFAULT_HOD_WEIGHTS = [
|
|
|
1286
1395
|
];
|
|
1287
1396
|
|
|
1288
1397
|
function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0, dayOfWeekWeights = DEFAULT_DOW_WEIGHTS, hourOfDayWeights = DEFAULT_HOD_WEIGHTS) {
|
|
1289
|
-
|
|
1290
|
-
|
|
1398
|
+
// Fallback: ALS-scoped DATASET_BEGIN/NOW (v1.5.1: was module-scoped, now
|
|
1399
|
+
// resolved per `generate()` call via `runWithDataset` — see dataset-context.js).
|
|
1400
|
+
if (!earliestTime) earliestTime = _alsGetDatasetBegin().unix();
|
|
1401
|
+
if (!latestTime) latestTime = _alsGetDatasetNow().unix();
|
|
1291
1402
|
const chance = getChance();
|
|
1292
1403
|
let totalRange = latestTime - earliestTime;
|
|
1293
1404
|
if (totalRange <= 0 || earliestTime > latestTime) {
|
|
@@ -1571,78 +1682,6 @@ function generateSessionId(seedStr) {
|
|
|
1571
1682
|
return [seg(), seg(), seg(), seg()].join("-");
|
|
1572
1683
|
}
|
|
1573
1684
|
|
|
1574
|
-
/**
|
|
1575
|
-
* Redistributes events into temporal clusters (sessions).
|
|
1576
|
-
*
|
|
1577
|
-
* Algorithm:
|
|
1578
|
-
* 1. Sort events by time
|
|
1579
|
-
* 2. Determine number of sessions (total events / avg events per session)
|
|
1580
|
-
* 3. Generate session anchor times using TimeSoup
|
|
1581
|
-
* 4. Assign events round-robin to sessions
|
|
1582
|
-
* 5. Within each session, retime events with tight spacing (5-300s apart)
|
|
1583
|
-
* 6. Regenerate insert_ids for retimed events
|
|
1584
|
-
* 7. Re-sort by time
|
|
1585
|
-
*
|
|
1586
|
-
* Mutates events in place. Does NOT assign session_id (call assignSessionIds after).
|
|
1587
|
-
*
|
|
1588
|
-
* @param {Object[]} events - Array of event objects with .time (ISO string)
|
|
1589
|
-
* @param {number} timeoutMinutes - Session timeout in minutes (used to determine intra-session spacing)
|
|
1590
|
-
* @param {Object} soupParams - Parameters for TimeSoup anchor generation
|
|
1591
|
-
*/
|
|
1592
|
-
function bunchIntoSessions(events, timeoutMinutes, soupParams) {
|
|
1593
|
-
if (events.length < 2) return;
|
|
1594
|
-
|
|
1595
|
-
const chance = getChance();
|
|
1596
|
-
const { earliestTime, latestTime, peaks, deviation, mean,
|
|
1597
|
-
dayOfWeekWeights, hourOfDayWeights } = soupParams;
|
|
1598
|
-
|
|
1599
|
-
// Sort by time first
|
|
1600
|
-
events.sort((a, b) => a.time < b.time ? -1 : a.time > b.time ? 1 : 0);
|
|
1601
|
-
|
|
1602
|
-
// Determine number of sessions: target 3-8 events per session
|
|
1603
|
-
const eventsPerSession = chance.integer({ min: 3, max: 8 });
|
|
1604
|
-
const numSessions = Math.max(1, Math.ceil(events.length / eventsPerSession));
|
|
1605
|
-
|
|
1606
|
-
// Generate session anchor times using TimeSoup
|
|
1607
|
-
const anchors = [];
|
|
1608
|
-
for (let i = 0; i < numSessions; i++) {
|
|
1609
|
-
const soupTime = TimeSoup(earliestTime, latestTime, peaks, deviation, mean,
|
|
1610
|
-
dayOfWeekWeights, hourOfDayWeights);
|
|
1611
|
-
anchors.push(soupTime);
|
|
1612
|
-
}
|
|
1613
|
-
anchors.sort((a, b) => a - b);
|
|
1614
|
-
|
|
1615
|
-
// Distribute events across sessions round-robin (preserving original order → temporal order)
|
|
1616
|
-
const sessionBuckets = anchors.map(() => []);
|
|
1617
|
-
for (let i = 0; i < events.length; i++) {
|
|
1618
|
-
const bucketIndex = Math.min(i % numSessions, numSessions - 1);
|
|
1619
|
-
sessionBuckets[bucketIndex].push(events[i]);
|
|
1620
|
-
}
|
|
1621
|
-
|
|
1622
|
-
// Retime events within each session
|
|
1623
|
-
for (let s = 0; s < numSessions; s++) {
|
|
1624
|
-
const bucket = sessionBuckets[s];
|
|
1625
|
-
if (bucket.length === 0) continue;
|
|
1626
|
-
|
|
1627
|
-
let currentTime = anchors[s];
|
|
1628
|
-
for (let e = 0; e < bucket.length; e++) {
|
|
1629
|
-
const ev = bucket[e];
|
|
1630
|
-
const clampedTime = Math.min(currentTime, latestTime);
|
|
1631
|
-
|
|
1632
|
-
ev.time = dayjs.unix(clampedTime).toISOString();
|
|
1633
|
-
// Regenerate insert_id to match new time
|
|
1634
|
-
const distinctId = ev.user_id || ev.device_id || ev.distinct_id || '';
|
|
1635
|
-
ev.insert_id = quickHash(`${ev.event}-${ev.time}-${distinctId}`);
|
|
1636
|
-
|
|
1637
|
-
// Advance time within session: 5-300 seconds (5s to 5min)
|
|
1638
|
-
currentTime += chance.integer({ min: 5, max: 300 });
|
|
1639
|
-
}
|
|
1640
|
-
}
|
|
1641
|
-
|
|
1642
|
-
// Re-sort by time
|
|
1643
|
-
events.sort((a, b) => a.time < b.time ? -1 : a.time > b.time ? 1 : 0);
|
|
1644
|
-
}
|
|
1645
|
-
|
|
1646
1685
|
/**
|
|
1647
1686
|
* Assigns session IDs to a chronologically sorted array of events.
|
|
1648
1687
|
* A new session starts when:
|
|
@@ -1661,6 +1700,7 @@ function assignSessionIds(events, timeoutMinutes = 30) {
|
|
|
1661
1700
|
|
|
1662
1701
|
const timeoutMs = timeoutMinutes * 60 * 1000;
|
|
1663
1702
|
const maxSessionMs = 24 * 60 * 60 * 1000;
|
|
1703
|
+
const DAY_MS = 86400 * 1000;
|
|
1664
1704
|
|
|
1665
1705
|
// Derive a stable seed from the first event so session IDs are deterministic
|
|
1666
1706
|
// across runs without consuming the main seeded RNG stream.
|
|
@@ -1668,15 +1708,21 @@ function assignSessionIds(events, timeoutMinutes = 30) {
|
|
|
1668
1708
|
let currentSessionId = generateSessionId(`${userKey}-${events[0].time}`);
|
|
1669
1709
|
let sessionStartMs = new Date(events[0].time).getTime();
|
|
1670
1710
|
let lastEventMs = sessionStartMs;
|
|
1711
|
+
let sessionDayIdx = Math.floor(sessionStartMs / DAY_MS);
|
|
1671
1712
|
|
|
1672
1713
|
for (const event of events) {
|
|
1673
1714
|
const eventMs = new Date(event.time).getTime();
|
|
1674
1715
|
const gapFromLast = eventMs - lastEventMs;
|
|
1675
1716
|
const sessionDuration = eventMs - sessionStartMs;
|
|
1717
|
+
const dayIdx = Math.floor(eventMs / DAY_MS);
|
|
1676
1718
|
|
|
1677
|
-
|
|
1719
|
+
// Mixpanel session_query.cpp:828-830, 911 — sessions terminate on a
|
|
1720
|
+
// day-index change in qtz (UTC here). Three reset triggers: timeout
|
|
1721
|
+
// gap, max-session cap, OR day boundary crossed.
|
|
1722
|
+
if (gapFromLast > timeoutMs || sessionDuration > maxSessionMs || dayIdx !== sessionDayIdx) {
|
|
1678
1723
|
currentSessionId = generateSessionId(`${userKey}-${event.time}`);
|
|
1679
1724
|
sessionStartMs = eventMs;
|
|
1725
|
+
sessionDayIdx = dayIdx;
|
|
1680
1726
|
}
|
|
1681
1727
|
|
|
1682
1728
|
event.session_id = currentSessionId;
|
|
@@ -1711,6 +1757,9 @@ export {
|
|
|
1711
1757
|
deepClone,
|
|
1712
1758
|
initChance,
|
|
1713
1759
|
getChance,
|
|
1760
|
+
initUserChance,
|
|
1761
|
+
getUserChance,
|
|
1762
|
+
resetUserChance,
|
|
1714
1763
|
decimal,
|
|
1715
1764
|
validTime,
|
|
1716
1765
|
validEvent,
|
|
@@ -1747,7 +1796,6 @@ export {
|
|
|
1747
1796
|
formatDuration,
|
|
1748
1797
|
generateSessionId,
|
|
1749
1798
|
assignSessionIds,
|
|
1750
|
-
bunchIntoSessions,
|
|
1751
1799
|
setDatasetNow,
|
|
1752
1800
|
setDatasetBegin,
|
|
1753
1801
|
dateRange,
|