@ak--47/dungeon-master 1.4.5 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/.claude/skills/analyze-soup/SKILL.md +158 -0
  2. package/.claude/skills/create-dungeon/SKILL.md +464 -0
  3. package/.claude/skills/verify-dungeon/SKILL.md +157 -0
  4. package/.claude/skills/verify-dungeon/references/counting-semantics.md +161 -0
  5. package/.claude/skills/verify-dungeon/references/report-format.md +216 -0
  6. package/.claude/skills/verify-dungeon/references/sql-recipes.md +857 -0
  7. package/.claude/skills/write-hooks/SKILL.md +468 -0
  8. package/CHANGELOG.md +182 -0
  9. package/HOOKS.md +1256 -597
  10. package/README.md +140 -5
  11. package/dungeons/technical/ad-spend.js +41 -49
  12. package/dungeons/technical/anonymous-users.js +38 -36
  13. package/dungeons/technical/array-of-object-lookup.js +136 -153
  14. package/dungeons/technical/datagen-v15-verify.js +87 -0
  15. package/dungeons/technical/experiments.js +42 -40
  16. package/dungeons/technical/foobar.js +114 -118
  17. package/dungeons/technical/group-analytics.js +42 -40
  18. package/dungeons/technical/hook-helpers-verify.js +69 -50
  19. package/dungeons/technical/identity-model-verify.js +22 -12
  20. package/dungeons/technical/mirror-strategies.js +37 -39
  21. package/dungeons/technical/nested-objects.js +119 -118
  22. package/dungeons/technical/pattern-aggregate-by-bin.js +21 -8
  23. package/dungeons/technical/pattern-attributed-by-source.js +23 -9
  24. package/dungeons/technical/pattern-frequency-by-frequency.js +21 -8
  25. package/dungeons/technical/pattern-funnel-frequency.js +30 -15
  26. package/dungeons/technical/pattern-ttc-by-segment.js +21 -8
  27. package/dungeons/technical/retention-cadence.js +115 -112
  28. package/dungeons/technical/sanity.js +86 -80
  29. package/dungeons/technical/scale-test.js +34 -38
  30. package/dungeons/technical/scd.js +111 -128
  31. package/dungeons/technical/simple.js +134 -141
  32. package/dungeons/technical/simplest.js +111 -65
  33. package/dungeons/technical/text-generation.js +110 -146
  34. package/dungeons/vertical/ai-platform.js +300 -333
  35. package/dungeons/vertical/community.js +290 -255
  36. package/dungeons/vertical/crypto.js +400 -391
  37. package/dungeons/vertical/dating.js +421 -375
  38. package/dungeons/vertical/devtools.js +346 -298
  39. package/dungeons/vertical/ecommerce.js +322 -394
  40. package/dungeons/vertical/education.js +380 -325
  41. package/dungeons/vertical/fintech.js +371 -325
  42. package/dungeons/vertical/fitness.js +345 -291
  43. package/dungeons/vertical/food-delivery.js +352 -307
  44. package/dungeons/vertical/gaming.js +490 -444
  45. package/dungeons/vertical/healthcare.js +311 -262
  46. package/dungeons/vertical/insurance-application.js +437 -409
  47. package/dungeons/vertical/logistics.js +278 -252
  48. package/dungeons/vertical/marketplace.js +340 -323
  49. package/dungeons/vertical/media.js +390 -335
  50. package/dungeons/vertical/real-estate.js +402 -347
  51. package/dungeons/vertical/sass.js +331 -333
  52. package/dungeons/vertical/social.js +377 -316
  53. package/dungeons/vertical/travel.js +302 -295
  54. package/index.js +64 -7
  55. package/lib/core/config-validator.js +378 -17
  56. package/lib/core/dungeon-loader.js +2 -5
  57. package/lib/generators/events.js +12 -13
  58. package/lib/generators/funnels.js +76 -2
  59. package/lib/hook-helpers/index.js +1 -0
  60. package/lib/hook-helpers/inject.js +95 -0
  61. package/lib/orchestrators/mixpanel-sender.js +7 -0
  62. package/lib/orchestrators/user-loop.js +598 -48
  63. package/lib/templates/defaults.js +59 -59
  64. package/lib/templates/macro-presets.js +53 -11
  65. package/lib/utils/dataset-context.js +103 -0
  66. package/lib/utils/retention-curve.js +140 -0
  67. package/lib/utils/utils.js +157 -109
  68. package/lib/verify/counting.js +360 -0
  69. package/lib/verify/emulate-breakdown.js +531 -108
  70. package/lib/verify/funnel-engine.js +539 -0
  71. package/lib/verify/identity.js +78 -0
  72. package/lib/verify/index.js +20 -0
  73. package/lib/verify/schema-validator.js +3 -1
  74. package/lib/verify/verify-dungeon.js +58 -0
  75. package/package.json +14 -3
  76. package/scripts/run-dungeon.mjs +12 -1
  77. package/types.d.ts +353 -4
  78. package/scripts/smoke-test-all.mjs +0 -162
package/index.js CHANGED
@@ -25,7 +25,8 @@ import { makeMirror } from './lib/generators/mirror.js';
25
25
  import { makeGroupProfile, makeProfile } from './lib/generators/profiles.js';
26
26
 
27
27
  // Utilities
28
- import { initChance, setDatasetNow, setDatasetBegin, deleteFile } from './lib/utils/utils.js';
28
+ import { initChance, initUserChance, resetUserChance, setDatasetNow, setDatasetBegin, deleteFile } from './lib/utils/utils.js';
29
+ import { runWithDataset } from './lib/utils/dataset-context.js';
29
30
 
30
31
  // External dependencies
31
32
  import dayjs from "dayjs";
@@ -130,6 +131,15 @@ async function runDungeon(config) {
130
131
  if (config.seed) {
131
132
  initChance(config.seed);
132
133
  }
134
+ // v1.5.1: optional separate user-id RNG. Same userSeed across runs
135
+ // produces the same user pool, regardless of `seed`. When unset, reset
136
+ // any leftover state from a prior in-process run so getUserChance()
137
+ // falls back to the event chance (= pre-v1.5.1 behavior).
138
+ if (config.userSeed) {
139
+ initUserChance(config.userSeed);
140
+ } else {
141
+ resetUserChance();
142
+ }
133
143
 
134
144
  // Step 1: Validate and enrich configuration (resolves dataset window)
135
145
  validatedConfig = validateDungeonConfig(config);
@@ -140,11 +150,20 @@ async function runDungeon(config) {
140
150
  const fixedBegin = /** @type {number} */ (validatedConfig.datasetStart);
141
151
 
142
152
  // Anchor the wall-clock-free reference used by date()/day() helpers in
143
- // dungeon configs. Without this, those factories produce values relative
144
- // to process-start, which leaks wall-clock time into the output.
153
+ // dungeon configs. v1.5.1: ALS-scoped via `runWithDataset` below; the
154
+ // legacy `setDatasetNow` / `setDatasetBegin` setters still fire as a
155
+ // back-compat fallback for tests that haven't migrated to the new API.
145
156
  setDatasetNow(fixedNow);
146
157
  setDatasetBegin(fixedBegin);
147
158
 
159
+ // v1.5.1: wrap the entire pipeline in an ALS scope so factory thunks
160
+ // (`date`, `day`, `dateRange`, `TimeSoup`, `validTime`) inside dungeon
161
+ // configs resolve the dataset window per-`generate()`-call instead of
162
+ // reading clobberable module state. Concurrent in-process `generate()`
163
+ // calls now run safely with distinct windows. See
164
+ // `lib/utils/dataset-context.js`.
165
+ return await runWithDataset(fixedBegin, fixedNow, async () => {
166
+
148
167
  // Step 2: Create context with validated config (pass time constants explicitly)
149
168
  const context = createContext(validatedConfig, null, { fixedNow, fixedBegin });
150
169
 
@@ -222,10 +241,25 @@ async function runDungeon(config) {
222
241
  // Now happens AFTER disk flush so batch files are available for import
223
242
  let importResults;
224
243
  if (validatedConfig.token) {
225
- context.reportProgress({ phase: "step", step: "import", status: "start" });
226
- const _t12 = Date.now();
227
- importResults = await sendToMixpanel(context);
228
- context.reportProgress({ phase: "step", step: "import", status: "complete", duration: Date.now() - _t12 });
244
+ // Defensive guard: real Mixpanel project tokens are 32-char hex.
245
+ // Placeholder strings ("your-mixpanel-token", "test-token", "hello token!", etc.)
246
+ // trigger mixpanel-import's infinite retry loop and hang the entire
247
+ // process historically a brutal source of silent test timeouts.
248
+ // If the token doesn't look like a real project token, warn loudly and skip the send.
249
+ // Set MP_BYPASS_TOKEN_CHECK=1 to override (e.g., legitimate non-standard tokens).
250
+ const looksReal = /^[0-9a-f]{32}$/i.test(String(validatedConfig.token).trim());
251
+ if (looksReal || process.env.MP_BYPASS_TOKEN_CHECK === '1') {
252
+ context.reportProgress({ phase: "step", step: "import", status: "start" });
253
+ const _t12 = Date.now();
254
+ importResults = await sendToMixpanel(context);
255
+ context.reportProgress({ phase: "step", step: "import", status: "complete", duration: Date.now() - _t12 });
256
+ } else if (validatedConfig.verbose) {
257
+ console.warn(
258
+ `⚠️ Skipping Mixpanel import: token "${validatedConfig.token}" does not look like a real Mixpanel project token ` +
259
+ `(expected 32-char hex). Set a real token or pass empty string to skip. ` +
260
+ `Override with MP_BYPASS_TOKEN_CHECK=1 if your token is legitimately non-standard.`
261
+ );
262
+ }
229
263
  }
230
264
 
231
265
  // Step 13: Compile results
@@ -236,6 +270,12 @@ async function runDungeon(config) {
236
270
 
237
271
  const progressSummary = context.getProgressSummary();
238
272
 
273
+ // v1.5.1: count of profiles that would be pushed to Mixpanel (non-`_drop`).
274
+ // Anonymous non-converters get `_drop: true` stamped in user-loop.js so
275
+ // mixpanel-sender skips them. `userProfilesData` still holds the full
276
+ // population for downstream tools.
277
+ const profilesPushed = countProfilesPushed(storage.userProfilesData);
278
+
239
279
  return {
240
280
  ...extractedData,
241
281
  importResults,
@@ -244,9 +284,12 @@ async function runDungeon(config) {
244
284
  operations: context.getOperations(),
245
285
  eventCount: context.getStoredEventCount(),
246
286
  userCount: context.getUserCount(),
287
+ profilesPushed,
247
288
  ...(progressSummary.updates > 0 || progressSummary.errors > 0 ? { progress: progressSummary } : {})
248
289
  };
249
290
 
291
+ }); // end runWithDataset
292
+
250
293
  } catch (error) {
251
294
  logger.error({ err: error }, `Error: ${error.message}`);
252
295
  throw error;
@@ -520,6 +563,20 @@ function extractFileInfo(storage) {
520
563
  return collectWrittenFiles(storage);
521
564
  }
522
565
 
566
+ /**
567
+ * Count profiles that would be pushed to Mixpanel (anonymous non-converters carry
568
+ * `_drop: true` and are skipped by mixpanel-sender). v1.5.1.
569
+ * @param {any} profilesContainer
570
+ * @returns {number}
571
+ */
572
+ function countProfilesPushed(profilesContainer) {
573
+ if (!profilesContainer) return 0;
574
+ const arr = Array.isArray(profilesContainer) ? profilesContainer : Array.from(profilesContainer);
575
+ let n = 0;
576
+ for (const p of arr) if (p && !p._drop) n++;
577
+ return n;
578
+ }
579
+
523
580
  /**
524
581
  * Extract data from storage containers, preserving array structure for groups/lookups/SCDs
525
582
  * @param {import('./types').Storage} storage - Storage object
@@ -9,6 +9,8 @@
9
9
  /** @typedef {import('../../types.js').Funnel} Funnel */
10
10
 
11
11
  import dayjs from "dayjs";
12
+ import utc from "dayjs/plugin/utc.js";
13
+ dayjs.extend(utc);
12
14
  import { makeName } from "ak-tools";
13
15
  import * as u from "../utils/utils.js";
14
16
  import { resolveSoup } from "../templates/soup-presets.js";
@@ -42,8 +44,8 @@ function resolveDatasetWindow(datasetStart, datasetEnd, userNumDays, verbose = f
42
44
  }
43
45
 
44
46
  if (hasStart && hasEnd) {
45
- const startUnix = parseToUnix(datasetStart, 'datasetStart');
46
- const endUnix = parseToUnix(datasetEnd, 'datasetEnd');
47
+ const startUnix = parseToUnix(datasetStart, 'datasetStart', false);
48
+ const endUnix = parseToUnix(datasetEnd, 'datasetEnd', true);
47
49
  if (endUnix <= startUnix) {
48
50
  throw new Error(`datasetEnd (${datasetEnd}) must be after datasetStart (${datasetStart}).`);
49
51
  }
@@ -69,11 +71,22 @@ function resolveDatasetWindow(datasetStart, datasetEnd, userNumDays, verbose = f
69
71
  /**
70
72
  * Parse a value (ISO string, unix seconds, dayjs-parseable) into unix seconds.
71
73
  * Throws if the value can't be parsed into a valid date.
74
+ *
75
+ * Bare-date convention (`YYYY-MM-DD` with no time component): pinned to UTC and
76
+ * interpreted as the user's intuitive intent — start of UTC day for the start
77
+ * boundary, end of UTC day for the end boundary. Without this rule, `dayjs()`
78
+ * would parse bare dates in the local timezone (cross-machine non-determinism)
79
+ * and treat both as start-of-day (datasetEnd would truncate ~16 hours).
80
+ *
81
+ * Full ISO strings with explicit time (`2026-05-10T23:59:59Z`,
82
+ * `2026-05-10T15:30:00-08:00`, etc.) are trusted as-is and parsed in UTC.
83
+ *
72
84
  * @param {*} value
73
85
  * @param {string} fieldName
86
+ * @param {boolean} [isEnd=false] - true treats bare-date strings as end-of-UTC-day
74
87
  * @returns {number}
75
88
  */
76
- function parseToUnix(value, fieldName) {
89
+ function parseToUnix(value, fieldName, isEnd = false) {
77
90
  // Treat numbers as unix seconds (or unix milliseconds if too large)
78
91
  if (typeof value === 'number') {
79
92
  if (!Number.isFinite(value) || value <= 0) {
@@ -82,7 +95,16 @@ function parseToUnix(value, fieldName) {
82
95
  // Heuristic: > 10^12 means milliseconds, otherwise seconds
83
96
  return value > 1e12 ? Math.floor(value / 1000) : Math.floor(value);
84
97
  }
85
- const parsed = dayjs(value);
98
+ // Bare YYYY-MM-DD → UTC start/end of day (intuitive convention)
99
+ if (typeof value === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(value)) {
100
+ const d = dayjs.utc(value);
101
+ if (!d.isValid()) {
102
+ throw new Error(`${fieldName} could not be parsed as a date (got ${JSON.stringify(value)}).`);
103
+ }
104
+ return (isEnd ? d.endOf('day') : d.startOf('day')).unix();
105
+ }
106
+ // Full ISO string (with time component) or other dayjs-parseable input — UTC parse
107
+ const parsed = dayjs.utc(value);
86
108
  if (!parsed.isValid()) {
87
109
  throw new Error(`${fieldName} could not be parsed as a date (got ${JSON.stringify(value)}).`);
88
110
  }
@@ -152,6 +174,63 @@ function inferFunnels(events) {
152
174
  */
153
175
  const KILLED_CONFIG_KEYS = ['subscription', 'attribution', 'geo', 'features', 'anomalies'];
154
176
 
177
+ // v1.5.1 (TODO #8): config restructure. Map sub-object → list of keys that can
178
+ // live in that sub-object. Sub-object value wins ONLY when the top-level key
179
+ // isn't set; if both are set, the top-level value wins and a verbose warning
180
+ // fires (one per offending key). Old top-level keys remain functional through
181
+ // v1.5.1 — migration is gradual.
182
+ const CONFIG_SUBOBJECTS = {
183
+ credentials: ['token', 'region', 'serviceAccount', 'serviceSecret', 'projectId'],
184
+ switches: ['hasLocation', 'hasCampaigns', 'hasAdSpend', 'hasSessionIds', 'hasAvatar', 'hasIOSDevices', 'hasAndroidDevices', 'hasDesktopDevices', 'hasBrowser', 'isAnonymous', 'alsoInferFunnels', 'hasAttributionFlags'],
185
+ identity: ['avgDevicePerUser', 'sessionTimeout'],
186
+ };
187
+
188
+ /**
189
+ * v1.5.1 (TODO #8): merge `credentials` / `switches` / `identity` sub-objects
190
+ * into top-level keys. Returns a NEW config object — does not mutate input.
191
+ *
192
+ * Resolution semantics:
193
+ * - sub-object key set, top-level NOT set → use sub-object value
194
+ * - sub-object key set, top-level set → top-level wins + warn (verbose)
195
+ * - sub-object missing, top-level set → use top-level (back-compat)
196
+ *
197
+ * Special case: `identity.hasAnonIds` → deprecated. If present, maps to
198
+ * `avgDevicePerUser: 1` and emits a verbose warning. Top-level `hasAnonIds`
199
+ * preserved (legacy alias) and untouched here.
200
+ *
201
+ * @param {Partial<Dungeon>} config
202
+ * @returns {Partial<Dungeon>}
203
+ */
204
+ function mergeConfigSubObjects(config) {
205
+ if (!config || typeof config !== 'object') return config;
206
+ const verbose = config.verbose === true;
207
+ const out = { ...config };
208
+ for (const [subKey, fields] of Object.entries(CONFIG_SUBOBJECTS)) {
209
+ const sub = out[subKey];
210
+ if (!sub || typeof sub !== 'object') continue;
211
+ for (const field of fields) {
212
+ if (sub[field] === undefined) continue;
213
+ if (out[field] === undefined) {
214
+ out[field] = sub[field];
215
+ } else if (verbose) {
216
+ console.warn(
217
+ `⚠️ config.${field} and config.${subKey}.${field} both set; ` +
218
+ `top-level wins. Drop config.${subKey}.${field} or remove the top-level value.`
219
+ );
220
+ }
221
+ }
222
+ }
223
+ // identity.hasAnonIds: deprecated alias.
224
+ if (out.identity && typeof out.identity === 'object' && out.identity.hasAnonIds !== undefined) {
225
+ if (verbose) console.warn(
226
+ `⚠️ identity.hasAnonIds is deprecated. Use identity.avgDevicePerUser: 1 instead. ` +
227
+ `Falling back to avgDevicePerUser=1 for v1.5.1.`
228
+ );
229
+ if (out.avgDevicePerUser === undefined) out.avgDevicePerUser = 1;
230
+ }
231
+ return out;
232
+ }
233
+
155
234
  /**
156
235
  * Strip killed config keys in place, log one deprecation warning per dungeon.
157
236
  * @param {Partial<Dungeon>} config
@@ -168,6 +247,85 @@ function stripKilledConfigKeys(config) {
168
247
  }
169
248
  }
170
249
 
250
+ /**
251
+ * v1.5: validate / default `Funnel.conversionWindowDays` in place.
252
+ *
253
+ * - Missing field + `timeToConvert/24 < 30` → set to 30 (Mixpanel UI default)
254
+ * - Missing field + `timeToConvert/24 >= 30` → auto-bump to `min(180, ceil(ttc * 1.5))` + warn
255
+ * - Field set > 180 → throw (Mixpanel hard cap)
256
+ *
257
+ * Reference: `backend/arb/reader/funnels/conversion_window.cpp`.
258
+ *
259
+ * @param {import('../../types.js').Funnel[]} funnels
260
+ */
261
+ function validateConversionWindow(funnels, verbose = false) {
262
+ const DEFAULT_WINDOW_DAYS = 30;
263
+ const MAX_WINDOW_DAYS = 180;
264
+ for (const f of funnels) {
265
+ if (!f) continue;
266
+ const ttcHours = Number.isFinite(f.timeToConvert) ? f.timeToConvert : 24;
267
+ const ttcDays = ttcHours / 24;
268
+ if (f.conversionWindowDays === undefined || f.conversionWindowDays === null) {
269
+ if (ttcDays >= DEFAULT_WINDOW_DAYS) {
270
+ f.conversionWindowDays = Math.min(MAX_WINDOW_DAYS, Math.ceil(ttcDays * 1.5));
271
+ if (verbose) console.warn(
272
+ `⚠️ Funnel "${f.name || (f.sequence && f.sequence.join(' > '))}": ` +
273
+ `timeToConvert (${ttcDays.toFixed(1)}d) exceeds default 30d conversion window. ` +
274
+ `Auto-set conversionWindowDays=${f.conversionWindowDays}. Set explicitly to silence.`
275
+ );
276
+ } else {
277
+ f.conversionWindowDays = DEFAULT_WINDOW_DAYS;
278
+ }
279
+ } else {
280
+ if (!Number.isFinite(f.conversionWindowDays) || f.conversionWindowDays <= 0) {
281
+ throw new Error(
282
+ `Funnel "${f.name || (f.sequence && f.sequence.join(' > '))}": ` +
283
+ `conversionWindowDays must be a positive finite number (got ${f.conversionWindowDays})`
284
+ );
285
+ }
286
+ if (f.conversionWindowDays > MAX_WINDOW_DAYS) {
287
+ throw new Error(
288
+ `Funnel "${f.name || (f.sequence && f.sequence.join(' > '))}": ` +
289
+ `conversionWindowDays cannot exceed ${MAX_WINDOW_DAYS} (Mixpanel hard cap)`
290
+ );
291
+ }
292
+ }
293
+ }
294
+ }
295
+
296
+ /**
297
+ * v1.5.0: validate `Funnel.exclusionEvents` in place.
298
+ * - Each entry must reference an event in `events[]` (schema-first guarantee).
299
+ * - Warn (don't throw) when an exclusion event is also a step in the funnel — the
300
+ * verifier will treat its presence as a terminator, but the same name appearing as
301
+ * a step is ambiguous.
302
+ *
303
+ * @param {import('../../types.js').Funnel[]} funnels
304
+ * @param {Array<{event?: string}>} events
305
+ */
306
+ function validateExclusionEvents(funnels, events, verbose = false) {
307
+ if (!Array.isArray(funnels) || !Array.isArray(events)) return;
308
+ const eventNames = new Set(events.map(e => e && e.event).filter(Boolean));
309
+ for (const f of funnels) {
310
+ if (!f || !Array.isArray(f.exclusionEvents) || !f.exclusionEvents.length) continue;
311
+ for (const name of f.exclusionEvents) {
312
+ if (!eventNames.has(name)) {
313
+ throw new Error(
314
+ `Funnel "${f.name || (f.sequence && f.sequence.join(' > '))}": ` +
315
+ `exclusionEvents entry "${name}" is not declared in events[]. ` +
316
+ `Add it as an event (schema-first) before referencing it as an exclusion.`
317
+ );
318
+ }
319
+ if (verbose && Array.isArray(f.sequence) && f.sequence.includes(name)) {
320
+ console.warn(
321
+ `⚠️ Funnel "${f.name || (f.sequence && f.sequence.join(' > '))}": ` +
322
+ `exclusion event "${name}" is also a funnel step — semantics are ambiguous.`
323
+ );
324
+ }
325
+ }
326
+ }
327
+ }
328
+
171
329
  /**
172
330
  * Validate `Funnel.attempts` config in place. Coerces missing/invalid bounds so callers
173
331
  * downstream don't have to re-defend. Throws on logically invalid configs (max < min).
@@ -253,6 +411,13 @@ function resolveDevicesPerUser(config) {
253
411
  export function validateDungeonConfig(config) {
254
412
  const chance = u.getChance();
255
413
 
414
+ // v1.5.1 (TODO #8): merge `switches` / `identity` / `credentials`
415
+ // sub-objects into top-level keys. New shape is preferred; old top-level
416
+ // keys still honored for back-compat with a verbose-gated warning. The
417
+ // rest of the validator reads from flat keys, so this is a transparent
418
+ // normalization step.
419
+ config = mergeConfigSubObjects(config);
420
+
256
421
  // Phase 1 — strip killed config keys before anything else reads them.
257
422
  stripKilledConfigKeys(config);
258
423
 
@@ -373,6 +538,31 @@ export function validateDungeonConfig(config) {
373
538
  avgEventsPerUserPerDay = numEvents / numUsers / numDays;
374
539
  }
375
540
 
541
+ // ── v1.5 Active-day primitive validation ──
542
+ // `avgActiveDaysPerUser` is a CONCENTRATOR — total event count is preserved
543
+ // (`avgEventsPerUserPerDay × numDays`), but events cluster onto fewer days.
544
+ // The implied per-active-day rate inflates: warn when it exceeds 50.
545
+ // v1.5.1: each macro preset (except `flat`) ships a sensible default —
546
+ // applied only when the dungeon doesn't set the field explicitly.
547
+ const macroResolvedEarly = resolveMacro(config.macro);
548
+ const avgActiveDaysPerUser = (config.avgActiveDaysPerUser !== undefined && config.avgActiveDaysPerUser !== null)
549
+ ? config.avgActiveDaysPerUser
550
+ : macroResolvedEarly.avgActiveDaysPerUser;
551
+ if (avgActiveDaysPerUser !== undefined && avgActiveDaysPerUser !== null) {
552
+ if (!Number.isFinite(avgActiveDaysPerUser) || avgActiveDaysPerUser <= 0) {
553
+ throw new Error(`avgActiveDaysPerUser must be a positive finite number (got ${avgActiveDaysPerUser})`);
554
+ }
555
+ const impliedRatePerActiveDay = (avgEventsPerUserPerDay * numDays) / avgActiveDaysPerUser;
556
+ if (verbose && impliedRatePerActiveDay > 50) {
557
+ console.warn(
558
+ `⚠️ avgActiveDaysPerUser=${avgActiveDaysPerUser} concentrates ` +
559
+ `${Math.round(avgEventsPerUserPerDay * numDays).toLocaleString()} events into ${avgActiveDaysPerUser} day(s) ` +
560
+ `→ ${impliedRatePerActiveDay.toFixed(0)} events per active day. ` +
561
+ `If you want fewer total events, lower avgEventsPerUserPerDay.`
562
+ );
563
+ }
564
+ }
565
+
376
566
  // Auto-enable batch mode for large datasets to prevent OOM.
377
567
  // MUST run after rate→numEvents resolution above, otherwise dungeons that set
378
568
  // only avgEventsPerUserPerDay would never trigger auto-batch.
@@ -390,20 +580,133 @@ export function validateDungeonConfig(config) {
390
580
  // bornRecentBias / percentUsersBornInDataset / preExistingSpread on the
391
581
  // dungeon config win over the macro preset's values, so existing dungeons
392
582
  // that set these explicitly continue to render the same way.
393
- // Resolve into local vars (do NOT mutate input config).
394
- const macroResolved = resolveMacro(config.macro);
583
+ // Resolve into local vars (do NOT mutate input config). v1.5.1: reuse the
584
+ // early-resolved macro from the avgActiveDaysPerUser merge above.
585
+ const macroResolved = macroResolvedEarly;
395
586
  let bornRecentBias = config.bornRecentBias !== undefined ? config.bornRecentBias : macroResolved.bornRecentBias;
396
587
  let percentUsersBornInDataset = config.percentUsersBornInDataset !== undefined ? config.percentUsersBornInDataset : macroResolved.percentUsersBornInDataset;
397
588
  let preExistingSpread = config.preExistingSpread !== undefined ? config.preExistingSpread : macroResolved.preExistingSpread;
398
589
 
399
- // Clamp bornRecentBias to [-1, 1] values outside this range produce
400
- // nonsensical exponents (e.g. Math.pow(0, -0.4) = Infinity in user-loop.js).
401
- if (typeof bornRecentBias === 'number' && Number.isFinite(bornRecentBias)) {
402
- bornRecentBias = Math.max(-1, Math.min(1, bornRecentBias));
403
- } else {
590
+ // ── v1.5 Engine-validation strict clamps ──────────────────────────────
591
+ // Pathological knob combinations produce nosedive / right-edge explosion
592
+ // patterns that no engine fix can rescue. We clamp them at validation time
593
+ // with a clear warning so dungeon authors fix the config rather than ship
594
+ // a broken-looking dataset. See `plans/ENGINE-VALIDATION/FIX.md` for the
595
+ // sweep evidence behind each rule.
596
+ // User-explicit detection. Fires when the value comes from EITHER top-level
597
+ // dungeon config OR macro-object override (e.g., `macro: { preset: 'growth',
598
+ // bornRecentBias: 0.5 }`). Both paths represent user intent to override; only
599
+ // raw preset names (e.g., `macro: 'growth'`) are exempt — their preset values
600
+ // are designed to be safe.
601
+ const macroAsObj = (config.macro && typeof config.macro === 'object' && !Array.isArray(config.macro))
602
+ ? /** @type {{preset?: string, percentUsersBornInDataset?: number, bornRecentBias?: number}} */ (config.macro)
603
+ : null;
604
+ const userBornExplicit =
605
+ (config.percentUsersBornInDataset !== undefined && config.percentUsersBornInDataset !== null)
606
+ || (macroAsObj !== null && macroAsObj.percentUsersBornInDataset !== undefined && macroAsObj.percentUsersBornInDataset !== null);
607
+ const userBiasExplicit =
608
+ (config.bornRecentBias !== undefined && config.bornRecentBias !== null)
609
+ || (macroAsObj !== null && macroAsObj.bornRecentBias !== undefined && macroAsObj.bornRecentBias !== null);
610
+
611
+ // Coerce non-finite to 0 first (sanity)
612
+ if (typeof bornRecentBias !== 'number' || !Number.isFinite(bornRecentBias)) {
404
613
  bornRecentBias = 0;
405
614
  }
406
615
 
616
+ // Clamp 1: hard absolute bounds on born% (data sanity)
617
+ if (typeof percentUsersBornInDataset !== 'number' || !Number.isFinite(percentUsersBornInDataset)) {
618
+ percentUsersBornInDataset = 0;
619
+ }
620
+ if (percentUsersBornInDataset > 100) {
621
+ if (verbose) console.warn(`⚠️ percentUsersBornInDataset=${percentUsersBornInDataset} clamped to 100. Values above 100 are not meaningful.`);
622
+ percentUsersBornInDataset = 100;
623
+ }
624
+ if (percentUsersBornInDataset < 0) {
625
+ if (verbose) console.warn(`⚠️ percentUsersBornInDataset=${percentUsersBornInDataset} clamped to 0. Negative values are not meaningful.`);
626
+ percentUsersBornInDataset = 0;
627
+ }
628
+
629
+ // Clamp 2: per-macro born compatibility — fires when the user explicitly opts
630
+ // into a named macro AND explicitly sets born%. Macro = contract: "produce
631
+ // the shape this preset describes". Born% over the cap breaks that contract
632
+ // (cumulative-acquisition right-edge explosion). Caps match each preset's
633
+ // default `percentUsersBornInDataset` to preserve the macro's characteristic
634
+ // shape. Users who need higher born% should switch macros (flat→growth,
635
+ // growth→viral). When no macro is set, the clamp does NOT fire — legacy
636
+ // dungeons that set percentUsersBornInDataset directly without picking a
637
+ // macro keep their existing behavior. Tuned empirically against the
638
+ // engine-validation sweep matrix (research/engine-sweep-pass*.json).
639
+ const MACRO_BORN_CAP = { flat: 12, steady: 12, growth: 30, viral: 55, decline: 5 };
640
+ const macroExplicit = config.macro !== undefined && config.macro !== null;
641
+ const macroKey = (typeof config.macro === 'string')
642
+ ? config.macro
643
+ : (config.macro && config.macro.preset) ? config.macro.preset : 'flat';
644
+ if (userBornExplicit && macroExplicit && MACRO_BORN_CAP[macroKey] !== undefined) {
645
+ const cap = MACRO_BORN_CAP[macroKey];
646
+ if (percentUsersBornInDataset > cap) {
647
+ if (verbose) console.warn(
648
+ `⚠️ macro="${macroKey}" + percentUsersBornInDataset=${percentUsersBornInDataset} ` +
649
+ `clamped to ${cap}. High born% with macro="${macroKey}" produces cumulative-acquisition ` +
650
+ `right-edge explosion. Use macro="growth" or "viral" for genuinely high-born configs. ` +
651
+ `To suppress, fix the config.`
652
+ );
653
+ percentUsersBornInDataset = cap;
654
+ }
655
+ }
656
+
657
+ // Clamp 3: bornRecentBias safe range. Plan PROMPT.md "[-0.5, 0.5]". Only fires
658
+ // on user-set values — viral preset (0.6) is allowed by design.
659
+ if (userBiasExplicit) {
660
+ if (bornRecentBias > 0.5) {
661
+ if (verbose) console.warn(`⚠️ bornRecentBias=${bornRecentBias} clamped to 0.5. Above 0.5 produces unusable right-skew. To suppress, fix the config.`);
662
+ bornRecentBias = 0.5;
663
+ }
664
+ if (bornRecentBias < -0.5) {
665
+ if (verbose) console.warn(`⚠️ bornRecentBias=${bornRecentBias} clamped to -0.5. Below -0.5 produces unusable left-skew. To suppress, fix the config.`);
666
+ bornRecentBias = -0.5;
667
+ }
668
+ }
669
+
670
+ // Clamp 4: bias × born compound check (only on explicit user values).
671
+ // Plan PROMPT.md: "born > 80 + bias > 0.4 → clamp bias to 0.3".
672
+ if ((userBornExplicit || userBiasExplicit) && percentUsersBornInDataset > 60 && bornRecentBias > 0.4) {
673
+ if (verbose) console.warn(
674
+ `⚠️ percentUsersBornInDataset=${percentUsersBornInDataset} + bornRecentBias=${bornRecentBias} ` +
675
+ `compounds to right-edge explosion. Clamping bornRecentBias to 0.3. To suppress, fix the config.`
676
+ );
677
+ bornRecentBias = 0.3;
678
+ }
679
+
680
+ // Final sanity: bias must be in [-1, 1] (Math.pow guards)
681
+ bornRecentBias = Math.max(-1, Math.min(1, bornRecentBias));
682
+
683
+ // Clamp 5: avgEventsPerUserPerDay safe range. Above 50 produces unrealistic
684
+ // load + memory cost. Plan PROMPT.md: clamp to 50.
685
+ if (Number.isFinite(avgEventsPerUserPerDay) && avgEventsPerUserPerDay > 50) {
686
+ if (verbose) console.warn(`⚠️ avgEventsPerUserPerDay=${avgEventsPerUserPerDay} clamped to 50. Above 50 is unrealistic load + memory cost. To suppress, fix the config.`);
687
+ avgEventsPerUserPerDay = 50;
688
+ numEvents = Math.round(avgEventsPerUserPerDay * numUsers * numDays);
689
+ }
690
+
691
+ // Clamp 6: avgActiveDaysPerUser cap at numDays/2. Above defeats the
692
+ // concentrator purpose. Reassign config so the user-loop sees the clamped value.
693
+ let avgActiveDaysClamped = avgActiveDaysPerUser;
694
+ if (avgActiveDaysPerUser !== undefined && avgActiveDaysPerUser !== null && avgActiveDaysPerUser > numDays * 0.5) {
695
+ const cap = Math.max(1, Math.floor(numDays * 0.5));
696
+ if (verbose) console.warn(`⚠️ avgActiveDaysPerUser=${avgActiveDaysPerUser} > numDays/2 (${numDays}/2); clamped to ${cap}. Above 50% defeats the concentrator purpose. To suppress, fix the config.`);
697
+ avgActiveDaysClamped = cap;
698
+ }
699
+
700
+ // Clamp 7: numDays minimum. Below 14 makes the strict-bar 14-day window
701
+ // meaningless. We WARN but do NOT clamp here because the dataset window has
702
+ // already been resolved upstream — clamping numDays alone would desync the
703
+ // engine. Pre-validator numDays bound is preferred (validator throws on
704
+ // numDays <= 0 already at line ~422). Just warn for visibility.
705
+ if (verbose && numDays < 14) {
706
+ console.warn(`⚠️ numDays=${numDays} < 14. Strict-bar engine-validation metrics use 14-day windows; results are noisy below 14 days. Consider increasing numDays.`);
707
+ }
708
+ // ──────────────────────────────────────────────────────────────────────
709
+
407
710
  // Use provided name if non-empty string, otherwise generate one
408
711
  if (!name || name === "") {
409
712
  name = makeName();
@@ -461,6 +764,26 @@ export function validateDungeonConfig(config) {
461
764
  funnels = [...funnels, ...inferredFunnels];
462
765
  }
463
766
 
767
+ // v1.5: auto-promote funnel-step events to `isStrictEvent: true`. Run BEFORE the
768
+ // catch-all funnel below so the catch-all only sweeps non-strict events. Without
769
+ // this, the greedy single-pass funnel engine consumes standalone instances as
770
+ // funnel step matches — corrupting both the standalone count AND the funnel TTC.
771
+ // Explicit `isStrictEvent: false` opts out (advanced; preserves mixed semantics).
772
+ // Skip `$experiment_started` since it's prepended by experiments, not user-declared.
773
+ const userDeclaredFunnelSteps = new Set(funnels.flatMap(f => Array.isArray(f.sequence) ? f.sequence : []));
774
+ userDeclaredFunnelSteps.delete('$experiment_started');
775
+ for (const ev of events) {
776
+ if (!ev || typeof ev.event !== 'string') continue;
777
+ if (!userDeclaredFunnelSteps.has(ev.event)) continue;
778
+ if (ev.isStrictEvent === false) continue; // explicit opt-out
779
+ if (ev.isStrictEvent === true) continue; // already set
780
+ ev.isStrictEvent = true;
781
+ if (verbose) console.warn(
782
+ `⚠️ Auto-promoted "${ev.event}" to isStrictEvent: true (appears as a funnel step). ` +
783
+ `Set isStrictEvent: false to opt out and allow standalone instances.`
784
+ );
785
+ }
786
+
464
787
  // Create funnel for events not in other funnels
465
788
  const eventContainedInFunnels = Array.from(funnels.reduce((acc, f) => {
466
789
  const events = f.sequence;
@@ -494,7 +817,15 @@ export function validateDungeonConfig(config) {
494
817
  sequence,
495
818
  conversionRate: 50,
496
819
  order: 'random',
497
- timeToConvert: 24 * 14,
820
+ // v1.5 engine bunchiness fix: shortened catch-all ttc from 14d → 1d.
821
+ // The user-loop fix constrains funnel step1's TimeSoup `latestTime` to
822
+ // `FIXED_NOW - ttc` to prevent spillover-and-_drop. With ttc=14d, this
823
+ // created a 14-day "no event zone" at the right edge of every dataset
824
+ // — flattening growth/viral macros into near-baseline shapes. ttc=1d
825
+ // gives the catch-all a 1-day right-edge zone, restoring magnitude
826
+ // distinction across macro presets while keeping spillover near zero.
827
+ // Explicit user-defined funnels keep their declared `timeToConvert`.
828
+ timeToConvert: 24,
498
829
  requireRepeats: false,
499
830
  });
500
831
  }
@@ -510,7 +841,7 @@ export function validateDungeonConfig(config) {
510
841
 
511
842
 
512
843
 
513
- // Event validation
844
+ // Event validation
514
845
  const validatedEvents = u.validateEventConfig(events);
515
846
 
516
847
  // ── Validate and resolve advanced features ──
@@ -542,6 +873,12 @@ export function validateDungeonConfig(config) {
542
873
  // Phase 1: validate Funnel.attempts on every funnel (additive — most have none).
543
874
  validateAttempts(funnels);
544
875
 
876
+ // v1.5: default + auto-bump Funnel.conversionWindowDays.
877
+ validateConversionWindow(funnels, verbose);
878
+
879
+ // v1.5.0: validate Funnel.exclusionEvents — entries must exist in events[].
880
+ validateExclusionEvents(funnels, validatedEvents, verbose);
881
+
545
882
  // Normalize experiment configs: true → default 3-variant, object → validated.
546
883
  normalizeExperiments(funnels, datasetEndUnix);
547
884
 
@@ -565,6 +902,16 @@ export function validateDungeonConfig(config) {
565
902
  // Precompute whether any event has isAttributionEvent for UTM stamping logic.
566
903
  const hasAttributionFlags = validatedEvents.some(e => e.isAttributionEvent);
567
904
 
905
+ // v1.5: Touchpoint cap. Default 10 (Mixpanel TOUCHPOINTS_LIMIT). Setting Infinity
906
+ // disables the cap (every eligible event gets stamped). Negative or zero disables
907
+ // stamping entirely (treat as "don't apply touchpoint cap pass").
908
+ let maxTouchpointsPerUser = config.maxTouchpointsPerUser;
909
+ if (maxTouchpointsPerUser === undefined || maxTouchpointsPerUser === null) {
910
+ maxTouchpointsPerUser = 10;
911
+ } else if (maxTouchpointsPerUser !== Infinity && (!Number.isFinite(maxTouchpointsPerUser) || maxTouchpointsPerUser < 0)) {
912
+ throw new Error(`maxTouchpointsPerUser must be a non-negative finite number or Infinity (got ${maxTouchpointsPerUser})`);
913
+ }
914
+
568
915
  // Build final config object
569
916
  const validatedConfig = {
570
917
  ...config,
@@ -616,6 +963,20 @@ export function validateDungeonConfig(config) {
616
963
  bornRecentBias,
617
964
  percentUsersBornInDataset,
618
965
  preExistingSpread,
966
+ // v1.5 distinct-day primitive (concentrator). undefined = legacy behavior.
967
+ avgActiveDaysPerUser: avgActiveDaysClamped !== undefined && avgActiveDaysClamped !== null
968
+ ? avgActiveDaysClamped
969
+ : undefined,
970
+ // v1.5.1 retention curve (`day1`/`day7`/`day30` etc. anchor weights). When
971
+ // set, `buildActiveDayPlan` biases day selection by the curve and the
972
+ // effective `avgActiveDaysPerUser` is derived from the curve's sum.
973
+ retentionCurve: (typeof config.retentionCurve === 'object' && config.retentionCurve !== null)
974
+ ? config.retentionCurve
975
+ : undefined,
976
+ // v1.5 attribution touchpoint cap (Mixpanel TOUCHPOINTS_LIMIT = 10).
977
+ maxTouchpointsPerUser,
978
+ // v1.5 auto-sort after everything hook. Default true. Opt out with explicit `false`.
979
+ autoSortAfterEverything: config.autoSortAfterEverything !== false,
619
980
  // Advanced features (kept after 1.4)
620
981
  personas,
621
982
  worldEvents,
@@ -669,7 +1030,7 @@ function transformSCDPropsWithoutCredentials(config) {
669
1030
  }
670
1031
 
671
1032
  // UI job without credentials - convert SCD props to regular props
672
- if (config.verbose !== false) console.log('\u26a0\ufe0f Service account credentials missing - converting SCD properties to static properties');
1033
+ if (config.verbose === true) console.log('\u26a0\ufe0f Service account credentials missing - converting SCD properties to static properties');
673
1034
 
674
1035
  // Ensure userProps and groupProps exist
675
1036
  if (!config.userProps) config.userProps = {};
@@ -688,20 +1049,20 @@ function transformSCDPropsWithoutCredentials(config) {
688
1049
  if (type === "user") {
689
1050
  // Add to userProps
690
1051
  config.userProps[propKey] = values;
691
- if (config.verbose !== false) console.log(` \u2713 Converted user SCD property: ${propKey}`);
1052
+ if (config.verbose === true) console.log(` \u2713 Converted user SCD property: ${propKey}`);
692
1053
  } else {
693
1054
  // Add to groupProps for the specific group type
694
1055
  if (!config.groupProps[type]) {
695
1056
  config.groupProps[type] = {};
696
1057
  }
697
1058
  config.groupProps[type][propKey] = values;
698
- if (config.verbose !== false) console.log(` \u2713 Converted group SCD property: ${propKey} (${type})`);
1059
+ if (config.verbose === true) console.log(` \u2713 Converted group SCD property: ${propKey} (${type})`);
699
1060
  }
700
1061
  }
701
1062
 
702
1063
  // Clear out scdProps since we've converted everything
703
1064
  config.scdProps = {};
704
- if (config.verbose !== false) console.log('\u2713 SCD properties converted to static properties\n');
1065
+ if (config.verbose === true) console.log('\u2713 SCD properties converted to static properties\n');
705
1066
  }
706
1067
 
707
1068
  // ── Advanced Feature Validation Functions ──