@ak--47/dungeon-master 1.6.0 → 1.6.2

package/lib/utils/utils.js

@@ -36,9 +36,35 @@ let globalUserChance;
 let userChanceInitialized = false;
 
 // Module-scoped memoization cache for weighted-array resolvers in `choose()`.
-// Lives for the lifetime of the node process; key is the function source string.
+// Key is the function source string; cleared per run by resetValueCaches().
+// Functions tagged `noCache = true` (e.g. pickAWinner closures, whose source
+// strings are identical across instances) bypass this cache entirely.
 const weightedArrayCache = new Map();
 
+// v1.6.1: per-run winner memo for unweighted string arrays. Two layers:
+// a WeakMap keyed by array IDENTITY holding the full {idx, weights} entry
+// (property arrays are stable references across a run, so the steady state is
+// a single WeakMap hit per event — no join, no weight rebuild), plus a Map
+// keyed by array CONTENTS so distinct array instances with equal contents
+// share one winner. Guarantees ONE stable winner per array per run so
+// aggregate distributions stay visibly skewed instead of cancelling to
+// uniform. Cleared by resetValueCaches() at run start / initChance so winners
+// never leak across runs or seeds.
+let winnerEntryCache = new WeakMap();
+const winnerCache = new Map();
+
+/**
+ * Clear the per-run value-resolution caches (winner memo + weighted-array
+ * resolver cache). Called by initChance() and at the top of every runDungeon
+ * so in-process back-to-back runs (different seeds, different users) never
+ * inherit a prior run's winners.
+ */
+function resetValueCaches() {
+	winnerEntryCache = new WeakMap();
+	winnerCache.clear();
+	weightedArrayCache.clear();
+}
+
 // v1.5.1: dataset-window state moved to AsyncLocalStorage scope
 // (`lib/utils/dataset-context.js`). Each `generate()` call wraps the pipeline
 // in `runWithDataset(begin, now, fn)`, and factory thunks read via the
@@ -146,6 +172,7 @@ function initChance(seed) {
 	if (!seed && process.env.SEED) seed = process.env.SEED;
 	globalChance = new Chance(seed);
 	chanceInitialized = true;
+	resetValueCaches();
 	return globalChance;
 }
 
@@ -387,10 +414,10 @@ function choose(value) {
 			// check to make sure that each element in the array only occurs once...
 			const uniqueItems = new Set(value);
 			if (uniqueItems.size === value.length) {
-				// Array has no duplicates, use pickAWinner
-				const quickList = pickAWinner(value, 0)();
-				const theChosenOne = chance.pickone(quickList);
-				return theChosenOne;
+				// Array has no duplicates → power-law draw with ONE stable
+				// seed-deterministic winner per array per run (v1.6.1)
+				const entry = getWinnerEntry(/** @type {string[]} */ (value));
+				return chance.weighted(value, entry.weights);
 			}
 
 		}
@@ -408,8 +435,17 @@ function choose(value) {
 	}
 
 	try {
-		// Keep resolving the value if it's a function (with caching)
+		// Keep resolving the value if it's a function (with caching).
+		// Functions tagged noCache (pickAWinner closures) skip the source-string
+		// cache: their toString() is identical across instances, so caching by
+		// source would hand one property's expansion to every other property.
 		while (typeof value === 'function') {
+			if (/** @type {any} */ (value).noCache === true) {
+				const result = value();
+				if (result instanceof ListValue) return result;
+				value = result;
+				continue;
+			}
 			const funcString = value.toString();
 
 			if (weightedArrayCache.has(funcString)) {
@@ -452,11 +488,6 @@ function choose(value) {
 			}
 		}
 
-		// ["","",""] should pick-a-winner
-		if (Array.isArray(value) && typeof value[0] === "string") {
-			value = pickAWinner(value)();
-		}
-
 		// [0,1,2] should pick one
 		if (Array.isArray(value) && typeof value[0] === "number") {
 			return chance.pickone(value);
@@ -988,63 +1019,122 @@ function weighChoices(items) {
 	};
 }
 
+/**
+ * Resolve the stable per-run winner index for an array of items. Memoized in
+ * `winnerCache` keyed by array contents (raw joined string — no hashing, so
+ * no collision risk), so every event drawing from the same array favors the
+ * SAME winner for the whole run. Winner is rolled from the seeded chance →
+ * same seed = same winner; resetValueCaches() clears the memo between runs.
+ *
+ * @param {Array} items - The list of items to pick a winner from.
+ * @returns {number} - The stable winner index for this run.
+ */
+function getStableWinnerIndex(items) {
+	const key = items.join('\u0000');
+	if (winnerCache.has(key)) return winnerCache.get(key);
+	const chance = getChance();
+	const winner = chance.integer({ min: 0, max: items.length - 1 });
+	winnerCache.set(key, winner);
+	return winner;
+}
+
+/**
+ * Full winner entry ({idx, weights}) for an array, memoized by array identity
+ * in a WeakMap. Steady state for the choose() hot path is one WeakMap hit per
+ * event — no join, no weight rebuild. Distinct array instances with equal
+ * contents converge on the same winner via getStableWinnerIndex.
+ *
+ * @param {string[]} items
+ * @returns {{idx: number, weights: number[]}}
+ */
+function getWinnerEntry(items) {
+	let entry = winnerEntryCache.get(items);
+	if (entry) return entry;
+	const idx = getStableWinnerIndex(items);
+	entry = { idx, weights: winnerWeights(items.length, idx) };
+	winnerEntryCache.set(items, entry);
+	return entry;
+}
+
+/**
+ * Build a power-law weight vector aligned to item indices: winner ~45%,
+ * second ~25%, third ~15%, remainder split over the tail with geometric decay.
+ * Rank order follows the rotational convention: second = (winner+1) % n,
+ * third = (winner+2) % n. This is THE place to tune the curve.
+ *
+ * @param {number} n - Number of items.
+ * @param {number} winnerIndex - Index of the winning item.
+ * @returns {number[]} - Weights (unnormalized; chance.weighted normalizes).
+ */
+function winnerWeights(n, winnerIndex) {
+	if (n <= 0) return [];
+	if (n === 1) return [1];
+	const HEAD = [45, 25, 15];
+	const TAIL_BUDGET = 15;
+	const TAIL_DECAY = 0.7;
+	const tailCount = Math.max(0, n - HEAD.length);
+	let tailWeights = [];
+	if (tailCount > 0) {
+		const raw = [];
+		let w = 1;
+		for (let k = 0; k < tailCount; k++) { raw.push(w); w *= TAIL_DECAY; }
+		const sum = raw.reduce((a, b) => a + b, 0);
+		tailWeights = raw.map(r => (r / sum) * TAIL_BUDGET);
+	}
+	const weights = new Array(n).fill(0);
+	for (let rank = 0; rank < n; rank++) {
+		const idx = (winnerIndex + rank) % n;
+		weights[idx] = rank < HEAD.length ? HEAD[rank] : tailWeights[rank - HEAD.length];
+	}
+	return weights;
+}
+
 /**
  * Creates a function that generates a weighted list of items
  * with a higher likelihood of picking a specified index and clear second and third place indices.
- * 
+ *
+ * v1.6.1: the returned closure yields a DETERMINISTIC weighted expansion
+ * (winner ~45%, second ~25%, third ~15%, geometric tail) built once — no
+ * per-call sampling variance. When no index is passed, the winner is the
+ * stable per-run memoized winner for the array (see getStableWinnerIndex),
+ * so direct dungeon-file usage is stable and seed-deterministic too.
+ *
  * @param {Array} items - The list of items to pick from.
  * @param {number} [mostChosenIndex] - The index of the item to be most favored.
  * @returns {function} - A function that returns a weighted list of items.
  */
 function pickAWinner(items, mostChosenIndex) {
-	const chance = getChance();
-
-	// Ensure mostChosenIndex is within the bounds of the items array
 	if (!items) return () => { return ""; };
 	if (!items.length) return () => { return ""; };
-	if (!mostChosenIndex) mostChosenIndex = chance.integer({ min: 0, max: items.length - 1 });
-	if (mostChosenIndex >= items.length) mostChosenIndex = items.length - 1;
-
-	// Calculate second and third most chosen indices
-	const secondMostChosenIndex = (mostChosenIndex + 1) % items.length;
-	const thirdMostChosenIndex = (mostChosenIndex + 2) % items.length;
+	if (!Number.isFinite(mostChosenIndex)) {
+		// undefined/null/NaN → stable per-run memoized winner
+		mostChosenIndex = getStableWinnerIndex(items);
+	} else {
+		mostChosenIndex = Math.floor(mostChosenIndex);
+		if (mostChosenIndex >= items.length) mostChosenIndex = items.length - 1;
+		if (mostChosenIndex < 0) mostChosenIndex = 0;
+	}
 
-	// Return a function that generates a weighted list
-	return function () {
-		const weighted = [];
-		for (let i = 0; i < 10; i++) {
-			const rand = chance.d10(); // Random number between 1 and 10
-
-			// 35% chance to favor the most chosen index
-			if (chance.bool({ likelihood: 35 })) {
-				// 50% chance to slightly alter the index
-				if (chance.bool({ likelihood: 50 })) {
-					weighted.push(items[mostChosenIndex]);
-				} else {
-					const addOrSubtract = chance.bool({ likelihood: 50 }) ? -rand : rand;
-					let newIndex = mostChosenIndex + addOrSubtract;
+	const weights = winnerWeights(items.length, mostChosenIndex);
+	const total = weights.reduce((a, b) => a + b, 0);
+	// scale slots with n so large arrays keep the ~45% winner share instead of
+	// having Math.max(1, ...) floors dilute it
+	const SLOTS = Math.max(20, items.length * 4);
+	const expansion = [];
+	items.forEach((item, i) => {
+		// every item keeps at least one slot so no value becomes unreachable
+		const count = Math.max(1, Math.round((weights[i] / total) * SLOTS));
+		for (let j = 0; j < count; j++) expansion.push(item);
+	});
 
-					// Ensure newIndex is within bounds
-					if (newIndex < 0) newIndex = 0;
-					if (newIndex >= items.length) newIndex = items.length - 1;
-					weighted.push(items[newIndex]);
-				}
-			}
-			// 25% chance to favor the second most chosen index
-			else if (chance.bool({ likelihood: 25 })) {
-				weighted.push(items[secondMostChosenIndex]);
-			}
-			// 15% chance to favor the third most chosen index
-			else if (chance.bool({ likelihood: 15 })) {
-				weighted.push(items[thirdMostChosenIndex]);
-			}
-			// Otherwise, pick a random item from the list
-			else {
-				weighted.push(chance.pickone(items));
-			}
-		}
-		return weighted;
+	const resolver = function () {
+		return expansion;
 	};
+	// all pickAWinner closures share one source string; without this tag,
+	// choose()'s weightedArrayCache would cache the first closure's expansion
+	// under that shared key and serve it for every other pickAWinner property
+	resolver.noCache = true;
+	return resolver;
 }
 
 function quickHash(str, seed = 0) {
@@ -1774,6 +1864,7 @@ export {
 	getUniqueKeys,
 	person,
 	pickAWinner,
+	resetValueCaches,
 	quickHash,
 	weighArray,
 	validateEventConfig,

package/lib/verify/index.js

@@ -11,6 +11,12 @@
 
 export { emulateBreakdown } from './emulate-breakdown.js';
 export { verifyDungeon } from './verify-dungeon.js';
+// v1.6.2: exposed on the verify surface so a standalone verify script — one that
+// reads shards off disk instead of running the dungeon — can resolve funnel
+// defaults (`conversionWindowDays`, `order`) before handing funnels to
+// `evaluateStories` / `applyFunnelDefaults`. Use the RETURN value; it does not
+// enrich the config you pass in.
+export { validateDungeonConfig } from '../core/config-validator.js';
 export { deriveExpectedSchema, validateSchema } from './schema-validator.js';
 export {
 	evaluateFunnel,

package/lib/verify/verify-dungeon.js

@@ -59,9 +59,10 @@ function findMatchingFunnel(funnels, steps) {
  * args object — the input is not mutated. Extracted from `verifyDungeon` in
  * v1.6 so the story runner (P3.3) reuses the exact same threading.
  * @param {Object} breakdownArgs - Args destined for `emulateBreakdown`.
- * @param {Array<Object>} funnels - VALIDATED dungeon funnels (`validateDungeonConfig`
- *   mutates funnels in place, so post-run `config.funnels` carries resolved
- *   `conversionWindowDays` / `order`).
+ * @param {Array<Object>} funnels - VALIDATED dungeon funnels — i.e. the funnels off
+ *   `validateDungeonConfig`'s RETURN value (post-1.6.2 it no longer enriches the
+ *   caller's object), which carry resolved `conversionWindowDays` / `order`. A run
+ *   result exposes them as `result.validatedConfig.funnels`.
  * @param {Array<Object>} [profiles] - User profiles, threaded into `timeToConvert`.
  * @returns {Object}
  */
@@ -113,20 +114,44 @@ export function applyFunnelDefaults(breakdownArgs, funnels, profiles) {
 /**
  * @param {Object} config - Dungeon config (or path; passed straight to DUNGEON_MASTER).
  * @param {VerifyCheck[]} checks
- * @returns {Promise<{ pass: boolean, results: Array<{ name: string, pass: boolean, detail?: string, rows?: Array<Object> }>, schemaReport: Object }>}
+ * @param {Object} [overrides] - v1.6.2: merged into the dungeon before it runs, same as
+ *   `DUNGEON_MASTER`'s second argument. Lets CI verify a production-scale dungeon at a
+ *   small `numUsers` / `numEvents` without editing it.
+ * @returns {Promise<{ pass: boolean, results: Array<{ name: string, pass: boolean, detail?: string, rows?: Array<Object> }>, schemaReport: Object, validatedConfig: Object }>}
  */
-export async function verifyDungeon(config, checks) {
+export async function verifyDungeon(config, checks, overrides) {
 	if (!checks || !checks.length) throw new Error('verifyDungeon: at least one check required');
-	let result = await DUNGEON_MASTER(config);
-	if (Array.isArray(result)) result = result[0];
+	let result = await DUNGEON_MASTER(config, overrides);
+	if (Array.isArray(result)) {
+		// v1.6.2: an array input runs N dungeons but checks are written against ONE
+		// schema, and we used to silently verify `result[0]` and throw the rest away —
+		// a green report for dungeons that were never looked at. Call verifyDungeon
+		// per dungeon instead.
+		if (result.length !== 1) {
+			throw new Error(
+				`verifyDungeon: got ${result.length} dungeons, but checks apply to one. ` +
+				`Call verifyDungeon once per dungeon.`
+			);
+		}
+		result = result[0];
+	}
 	const events = Array.isArray(result.eventData) ? result.eventData : Array.from(result.eventData);
 	const profiles = Array.isArray(result.userProfilesData) ? result.userProfilesData : Array.from(result.userProfilesData);
-	const schemaReport = validateSchema(events, config);
+	// v1.6.2: schema-check against the config the run actually used. `config` may be a
+	// PATH STRING (no fields at all → every property reads as unexpected), and even for
+	// an object input a v1.5.1 dungeon keeps `hasAndroidDevices` / `hasBrowser` under
+	// `switches`, which `deriveExpectedSchema` only reads once flattened.
+	const schemaReport = validateSchema(events, result.validatedConfig || config);
 	const ctx = { events, profiles, schemaReport };
 	const results = [];
-	// `validateDungeonConfig` mutates funnels in place — by the time we read here,
-	// `conversionWindowDays` is populated and `order` is the validated value.
-	const validatedFunnels = (config && Array.isArray(config.funnels)) ? config.funnels : [];
+	// v1.6.2: read the funnels off the run's `validatedConfig` — `conversionWindowDays`
+	// and the resolved `order` live there. Before 1.6.2 we read them back off the
+	// caller's own `config`, relying on `validateDungeonConfig` enriching in place;
+	// that also meant a path/JSON input (no `.funnels` on the string) silently got an
+	// empty funnel list and thus an unbounded conversion window. Both are fixed here.
+	const validatedFunnels = Array.isArray(result?.validatedConfig?.funnels)
+		? result.validatedConfig.funnels
+		: (config && Array.isArray(config.funnels)) ? config.funnels : [];
 	for (const check of checks) {
 		try {
 			const breakdownArgs = applyFunnelDefaults(check.breakdown, validatedFunnels, profiles);
@@ -138,5 +163,7 @@ export async function verifyDungeon(config, checks) {
 		}
 	}
 	const pass = results.every(r => r.pass) && schemaReport.pass;
-	return { pass, results, schemaReport };
+	// v1.6.2: hand back the enriched config the run used, so callers can read
+	// resolved funnel/event values without re-validating.
+	return { pass, results, schemaReport, validatedConfig: result.validatedConfig };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ak--47/dungeon-master",
-	"version": "1.6.0",
+	"version": "1.6.2",
 	"description": "generate fancy datasets",
 	"type": "module",
 	"main": "index.js",
@@ -33,7 +33,7 @@
 		"HOOKS.md"
 	],
 	"engines": {
-		"node": ">=18.0.0"
+		"node": ">=20.20.0"
 	},
 	"publishConfig": {
 		"access": "public"
@@ -87,7 +87,7 @@
 		"dotenv": "^16.4.5",
 		"hyparquet-writer": "^0.6.1",
 		"mixpanel": "^0.18.0",
-		"mixpanel-import": "^3.3.2",
+		"mixpanel-import": "^3.5.1",
 		"p-limit": "^3.1.0",
 		"pino": "^9.0.0",
 		"pino-pretty": "^11.0.0",
@@ -108,4 +108,4 @@
 			"tmp/"
 		]
 	}
-}
+}

package/scripts/verify-stories.mjs

@@ -176,7 +176,8 @@ if (inMemory) {
 
 	// Funnel auto-threading reads VALIDATED funnel fields (conversionWindowDays,
 	// order). The dungeon was not run in this process, so validate the config
-	// here — validateDungeonConfig resolves those defaults in place.
+	// here and use the RETURN value — as of v1.6.2 validateDungeonConfig does not
+	// enrich the object you hand it.
 	const validated = validateDungeonConfig({ ...config, token: '' });
 	const identityMap = buildIdentityMap(profiles);
 

package/types.d.ts

@@ -1324,6 +1324,21 @@ export type Result = {
      * so `userProfilesData.length - profilesPushed` = dropped profile count.
      */
     profilesPushed?: number;
+    /**
+     * v1.6.2: the enriched config this run actually used. `validateDungeonConfig`
+     * no longer writes back to the object you passed in, so resolved values —
+     * `funnels[].conversionWindowDays`, `events[].isStrictEvent`, the resolved
+     * dataset window — must be read here rather than off your own config.
+     *
+     * READ-ONLY. Do NOT feed this back into `DUNGEON_MASTER` — validation is not
+     * idempotent (each pass appends to the funnel set, eventually producing an
+     * empty `sequence`), and the object carries engine scratch fields. Re-run the
+     * ORIGINAL config instead.
+     *
+     * Credentials (`token`, `serviceAccount`, `serviceSecret`, `projectId`,
+     * `credentials`) are stripped — a Result is a thing hosts log.
+     */
+    validatedConfig?: Dungeon;
     /** Progress callback summary. Only present when `onProgress` was provided. */
     progress?: ProgressSummary;
 };
@@ -2358,6 +2373,8 @@ declare module '@ak--47/dungeon-master/utils' {
     export function objectList(template: Record<string, ValueValid>, options?: { min?: number; max?: number }): () => Array<Record<string, unknown>>;
     export function weighNumRange(min: number, max: number, skew?: number, size?: number): number[];
     export function pickAWinner(items: string[], mostChosenIndex?: number): () => string[];
+    /** Clears the per-run stable-winner + weighted-array caches (also called by initChance and at every run start). */
+    export function resetValueCaches(): void;
     export function initChance(seed?: string): unknown;
     export function initUserChance(seed?: string): unknown;
     export function getUserChance(): unknown;