@ak--47/dungeon-master 1.6.5 → 1.8.0

package/lib/utils/utils.js

@@ -63,6 +63,51 @@ function resetValueCaches() {
 	winnerEntryCache = new WeakMap();
 	winnerCache.clear();
 	weightedArrayCache.clear();
+	autoPowerLawEnabled = true;
+}
+
+// v1.7.0 (P2-1): run-level switch for the implicit power-law draw on 3–19-item
+// unique-string arrays. `choose()` is a free function with no config access, so
+// the orchestrator sets this from `validatedConfig.autoPowerLaw` at run start
+// (mirrors how the seeded chance singleton is managed). resetValueCaches()
+// restores the default so a prior run's opt-out never leaks.
+let autoPowerLawEnabled = true;
+
+/** @param {boolean} enabled */
+function setAutoPowerLaw(enabled) {
+	autoPowerLawEnabled = enabled !== false;
+}
+
+/** @returns {boolean} */
+function getAutoPowerLaw() {
+	return autoPowerLawEnabled;
+}
+
+/**
+ * v1.7.0 (P1-1): true for a value function that DECLARES a parameter (`(ctx) => …`)
+ * and is not a bound native (`chance.animal.bind(chance)` reports the native's
+ * arity but cannot read ctx). Context-aware functions skip the source-string
+ * cache and, on funnel steps, resolve inside makeEvent with the real event ctx.
+ * @param {unknown} value
+ * @returns {value is (ctx?: any) => any}
+ */
+function isContextAware(value) {
+	return typeof value === 'function' && value.length >= 1 && !Function.prototype.toString.call(value).includes('[native code]');
+}
+
+/**
+ * v1.7.0 (P2-1): `{ __weights: { free: 60, pro: 30, enterprise: 10 } }` — the
+ * declarative weighted form. True when `value` carries a non-empty `__weights`
+ * object whose values are finite, non-negative numbers.
+ * @param {unknown} value
+ * @returns {value is { __weights: Record<string, number> }}
+ */
+function isWeightsForm(value) {
+	if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
+	const w = /** @type {any} */ (value).__weights;
+	if (!w || typeof w !== 'object' || Array.isArray(w)) return false;
+	const entries = Object.entries(w);
+	return entries.length > 0 && entries.every(([, n]) => typeof n === 'number' && Number.isFinite(n) && n >= 0);
 }
 
 // v1.5.1: dataset-window state moved to AsyncLocalStorage scope
@@ -400,15 +445,24 @@ function objectList(template, options = {}) {
  * similar to pick
  * @param  {ValueValid} value
  */
-function choose(value) {
+function choose(value, ctx = undefined) {
 	if (value instanceof ListValue) return value;
 	const chance = getChance();
 
-	// most of the time this will receive a list of strings; 
+	// v1.7.0 (P2-1): declarative weights. The author's numbers ARE the
+	// distribution — no power law, no winner memo. Zero-weight keys never draw.
+	if (isWeightsForm(value)) {
+		const entries = Object.entries(value.__weights).filter(([, n]) => n > 0);
+		if (!entries.length) return "";
+		return chance.weighted(entries.map(([k]) => k), entries.map(([, n]) => n));
+	}
+
+	// most of the time this will receive a list of strings;
 	// when that is the case, we need to ensure some 'keywords' like 'variant' or 'test' aren't in the array
 	// next we want to see if the array is unweighted ... i.e. no dupe strings and each string only occurs once ['a', 'b', 'c', 'd']
 	// if all these are true we will pickAWinner(value)()
-	if (Array.isArray(value) && value.length > 2 && value.length < 20 && value.every(item => typeof item === 'string')) {
+	// v1.7.0 (P2-1): `autoPowerLaw: false` skips this branch → uniform pickone below.
+	if (autoPowerLawEnabled && Array.isArray(value) && value.length > 2 && value.length < 20 && value.every(item => typeof item === 'string')) {
 		// ensure terms 'variant' 'group' 'experiment' or 'population' are NOT in any of the items
 		if (!value.some(item => item.includes('variant') || item.includes('group') || item.includes('experiment') || item.includes('population'))) {
 			// check to make sure that each element in the array only occurs once...
@@ -439,21 +493,32 @@ function choose(value) {
 		// Functions tagged noCache (pickAWinner closures) skip the source-string
 		// cache: their toString() is identical across instances, so caching by
 		// source would hand one property's expansion to every other property.
+		//
+		// v1.7.0 (P1-1): value functions receive a `ValueContext` (`{ profile,
+		// event, time, config }`, members optional). Any function with declared
+		// arity >= 1 is context-aware and ALSO skips the source-string cache —
+		// its result legitimately differs per user/event, so caching by source
+		// would freeze the first evaluation and hand it to every later caller.
+		// Zero-arity functions keep the pre-1.7 cache behavior exactly.
 		while (typeof value === 'function') {
-			if (/** @type {any} */ (value).noCache === true) {
-				const result = value();
+			// Bound natives (`chance.animal.bind(chance)`) report the native's arity
+			// and cannot read ctx; call them exactly as before (no argument) so the
+			// options object chance methods take never sees the context.
+			const funcString = value.toString();
+			const isNative = funcString.includes('[native code]');
+			if (/** @type {any} */ (value).noCache === true || (value.length >= 1 && !isNative)) {
+				const result = isNative ? value() : value(ctx);
 				if (result instanceof ListValue) return result;
 				value = result;
 				continue;
 			}
-			const funcString = value.toString();
 
 			if (weightedArrayCache.has(funcString)) {
 				value = weightedArrayCache.get(funcString);
 				break;
 			}
 
-			const result = value();
+			const result = isNative ? value() : value(ctx);
 			if (result instanceof ListValue) return result;
 			if (Array.isArray(result) && result.length > 10) {
 				// Cache large arrays (likely weighted arrays)
@@ -463,6 +528,8 @@ function choose(value) {
 		}
 
 			if (value instanceof ListValue) return value;
+			// A function may return the weighted form.
+			if (isWeightsForm(value)) return choose(value, ctx);
 
 			if (Array.isArray(value) && value.length === 0) {
 			return ""; // Return empty string if the array is empty
@@ -748,10 +815,26 @@ function streamJSON(filePath, data, options = {}) {
 	});
 }
 
+function csvRow(item, columns) {
+	return columns.map(col => {
+		const value = item[col];
+
+		if (value === null || value === undefined) {
+			return '';
+		}
+
+		const serialized = typeof value === 'object'
+			? JSON.stringify(value)
+			: value.toString();
+
+		return `"${serialized.replace(/"/g, '""')}"`;
+	}).join(',');
+}
+
 function streamCSV(filePath, data, options = {}) {
 	return new Promise((resolve, reject) => {
 		let writeStream;
-		const { gzip = false } = options;
+		const { gzip = false, fixedColumns } = options;
 
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
@@ -775,20 +858,12 @@ function streamCSV(filePath, data, options = {}) {
 			}
 		}
 
-		// Extract all unique keys from the data array
-		const columns = getUniqueKeys(data);  // Assuming getUniqueKeys properly retrieves all keys
+		const columns = Array.isArray(fixedColumns) ? fixedColumns : getUniqueKeys(data);
 
-		// Stream the header
 		writeStream.write(columns.join(',') + '\n');
 
-		// Stream each data row
 		data.forEach(item => {
-			for (const key in item) {
-				// Ensure all nested objects are properly stringified
-				if (typeof item[key] === "object") item[key] = JSON.stringify(item[key]);
-			}
-			const row = columns.map(col => item[col] ? `"${item[col].toString().replace(/"/g, '""')}"` : "").join(',');
-			writeStream.write(row + '\n');
+			writeStream.write(csvRow(item, columns) + '\n');
 		});
 
 		writeStream.end();
@@ -1303,7 +1378,7 @@ META
  * @param  {Config} config
  */
 function buildFileNames(config) {
-	const { format = "csv", groupKeys = [], lookupTables = [] } = config;
+	const { format = "csv", groupKeys = [], lookupTables = [], warehouseMetrics = [] } = config;
 	let extension = "";
 	extension = format === "csv" ? "csv" : "json";
 	// const current = dayjs.utc().format("MM-DD-HH");
@@ -1321,10 +1396,12 @@ function buildFileNames(config) {
 		eventFiles: [path.join(writeDir, `${simName}-EVENTS.${extension}`)],
 		userFiles: [path.join(writeDir, `${simName}-USERS.${extension}`)],
 		adSpendFiles: [],
+		standaloneFiles: [],
 		scdFiles: [],
 		mirrorFiles: [],
 		groupFiles: [],
 		lookupFiles: [],
+		warehouseFiles: [],
 		folder: writeDir,
 	};
 	//add ad spend files
@@ -1332,6 +1409,11 @@ function buildFileNames(config) {
 		writePaths.adSpendFiles.push(path.join(writeDir, `${simName}-AD-SPEND.${extension}`));
 	}
 
+	//add standalone (identity-less snapshot) files — v1.8.0
+	if (Array.isArray(config?.standaloneEvents) && config.standaloneEvents.length > 0) {
+		writePaths.standaloneFiles.push(path.join(writeDir, `${simName}-STANDALONE.${extension}`));
+	}
+
 	//add SCD files
 	const scdKeys = Object.keys(config?.scdProps || {});
 	for (const key of scdKeys) {
@@ -1358,6 +1440,15 @@ function buildFileNames(config) {
 		);
 	}
 
+	for (const warehouseMetric of warehouseMetrics) {
+		const metricName = warehouseMetric?.name;
+		if (typeof metricName !== 'string') continue;
+		const metricFormat = warehouseMetric?.format || format || 'csv';
+		writePaths.warehouseFiles.push(
+			path.join(writeDir, `${simName}-WAREHOUSE-${metricName}.${metricFormat}`)
+		);
+	}
+
 	//add mirror files
 	const mirrorProps = config?.mirrorProps || {};
 	if (Object.keys(mirrorProps).length) {
@@ -1864,6 +1955,10 @@ export {
 	person,
 	pickAWinner,
 	resetValueCaches,
+	setAutoPowerLaw,
+	getAutoPowerLaw,
+	isWeightsForm,
+	isContextAware,
 	quickHash,
 	weighArray,
 	validateEventConfig,
@@ -1877,6 +1972,7 @@ export {
 	generateUser,
 	optimizedBoxMuller,
 	buildFileNames,
+	csvRow,
 	streamJSON,
 	streamCSV,
 	streamParquet,

package/lib/verify/index.js

@@ -17,6 +17,7 @@ export { verifyDungeon } from './verify-dungeon.js';
 // `evaluateStories` / `applyFunnelDefaults`. Use the RETURN value; it does not
 // enrich the config you pass in.
 export { validateDungeonConfig } from '../core/config-validator.js';
+export { pearson, computeWarehouseStats, computeWarehouseSourceRows, auditWarehouseRows } from './warehouse.js';
 export { deriveExpectedSchema, validateSchema } from './schema-validator.js';
 export {
 	evaluateFunnel,

package/lib/verify/schema-validator.js

@@ -41,6 +41,14 @@ export function deriveExpectedSchema(config) {
 		}
 	}
 
+	// v1.7.0 (P1-2): stickyEventProps are engine-stamped on every event of every
+	// user — legal on every event type, not flag stamping.
+	if (Array.isArray(config.stickyEventProps)) {
+		for (const key of config.stickyEventProps) {
+			if (typeof key === 'string' && key) globalKeys.add(key);
+		}
+	}
+
 	if (config.hasLocation) {
 		for (const k of LOCATION_KEYS) globalKeys.add(k);
 	}

package/lib/verify/story-runner.js

@@ -27,6 +27,7 @@
 
 import { emulateBreakdown } from './emulate-breakdown.js';
 import { applyFunnelDefaults } from './verify-dungeon.js';
+import { computeWarehouseSourceRows, computeWarehouseStats } from './warehouse.js';
 
 /** Closed archetype enum — MUST match lib/templates/story-spec.schema.json (unit-tested). */
 export const STORY_ARCHETYPES = [
@@ -322,6 +323,9 @@ export function validateStories(stories) {
 				err(ap, 'breakdown: required object with a string `type`');
 			} else if (a.breakdown.type === 'duckdb' && (typeof a.breakdown.sql !== 'string' || !a.breakdown.sql)) {
 				err(ap, 'breakdown: type "duckdb" requires a non-empty `sql`');
+			} else if ((a.breakdown.type === 'warehouse' || a.breakdown.type === 'warehouse-stats')
+				&& (typeof a.breakdown.table !== 'string' || !a.breakdown.table.trim())) {
+				err(ap, `breakdown: type "${a.breakdown.type}" requires a non-empty \`table\``);
 			}
 			if (a.expect === undefined && typeof a.assert !== 'function') {
 				err(ap, 'requires `expect` or a function-valued `assert`');
@@ -339,8 +343,8 @@ export function validateStories(stories) {
 					for (const [name, spec] of Object.entries(a.select)) {
 						if (!NAME_RE.test(name)) err(ap, `select: name "${name}" must be an identifier`);
 						selectNames.add(name);
-						if (!spec || typeof spec !== 'object' || !spec.where || typeof spec.where !== 'object' || !Object.keys(spec.where).length) {
-							err(ap, `select.${name}: requires a non-empty \`where\``);
+						if (!spec || typeof spec !== 'object' || !spec.where || typeof spec.where !== 'object') {
+							err(ap, `select.${name}: requires a \`where\` object`);
 							continue;
 						}
 						for (const [col, cond] of Object.entries(spec.where)) {
@@ -406,8 +410,8 @@ export function storiesToChecks(stories) {
 	for (const story of stories) {
 		story.assertions.forEach((assertion, i) => {
 			const name = `${story.id}[${i}]`;
-			if (assertion.breakdown.type === 'duckdb') {
-				console.warn(`[dungeon-master] ${name}: duckdb assertions only run in disk mode (scripts/verify-stories.mjs) — skipped in-memory.`);
+			if (assertion.breakdown.type === 'duckdb' || assertion.breakdown.type === 'warehouse' || assertion.breakdown.type === 'warehouse-stats') {
+				console.warn(`[dungeon-master] ${name}: ${assertion.breakdown.type} assertions only run in disk mode via scripts/verify-stories.mjs — skipped in verifyDungeon in-memory checks.`);
 				return;
 			}
 			checks.push({
@@ -441,11 +445,34 @@ export function storiesToChecks(stories) {
  *   auto-build over large profile sets is wasteful.
  * @param {(sql: string) => Promise<Array<Object>>} [opts.runSql] - duckdb
  *   executor (provided by the CLI in disk mode). Absent → duckdb assertions
- *   report NONE with an explanatory detail.
+ *   report NONE unless skipDiskOnlyDuckdb is set.
+ * @param {Record<string, Array<Object>>} [opts.warehouseRows] - Loaded
+ *   warehouse rows keyed by table name.
+ * @param {Record<string, Object>} [opts.warehouseSpecs] - Resolved warehouse
+ *   metric specs keyed by table name.
+ * @param {string|number} [opts.datasetStart] - Dataset window start for
+ *   in-window warehouse stats comparisons.
+ * @param {string|number} [opts.datasetEnd] - Dataset window end for in-window
+ *   warehouse stats comparisons.
+ * @param {boolean} [opts.skipDiskOnlyDuckdb] - Mark duckdb assertions skipped
+ *   instead of NONE when no SQL executor is available.
+ * @param {(message: string) => void} [opts.onWarning] - Warning sink for
+ *   skipped disk-only assertions.
  * @returns {Promise<Array<{ id: string, hook: string, archetype: string, verdict: string, assertions: Array<{ name: string, verdict: string, observed: number|null, detail: string }> }>>}
  */
 export async function evaluateStories(stories, events, opts) {
-	const { profiles, funnels, runSql, identityMap } = opts || {};
+	const {
+		profiles,
+		funnels,
+		runSql,
+		identityMap,
+		warehouseRows,
+		warehouseSpecs,
+		datasetStart,
+		datasetEnd,
+		skipDiskOnlyDuckdb,
+		onWarning,
+	} = opts || {};
 	const v = validateStories(stories);
 	if (!v.valid) {
 		throw new Error(`evaluateStories: invalid stories:\n  ${v.errors.join('\n  ')}`);
@@ -460,10 +487,21 @@ export async function evaluateStories(stories, events, opts) {
 			try {
 				if (assertion.breakdown.type === 'duckdb') {
 					if (!runSql) {
-						results.push({ name, verdict: 'NONE', observed: null, detail: 'duckdb assertion requires disk mode (no SQL executor available)' });
+						if (skipDiskOnlyDuckdb) {
+							onWarning?.(`[dungeon-master] ${name}: duckdb assertions only run in disk mode (scripts/verify-stories.mjs) — skipped in-memory.`);
+							results.push({ name, verdict: 'SKIPPED', observed: null, detail: 'duckdb assertion requires disk mode (skipped in-memory)' });
+						} else {
+							results.push({ name, verdict: 'NONE', observed: null, detail: 'duckdb assertion requires disk mode (no SQL executor available)' });
+						}
 						continue;
 					}
 					rows = await runSql(assertion.breakdown.sql);
+				} else if (assertion.breakdown.type === 'warehouse') {
+					rows = resolveWarehouseRows(assertion.breakdown.table, warehouseRows, warehouseSpecs);
+				} else if (assertion.breakdown.type === 'warehouse-stats') {
+					const spec = resolveWarehouseSpec(assertion.breakdown.table, warehouseSpecs);
+					const tableRows = resolveWarehouseRows(assertion.breakdown.table, warehouseRows, warehouseSpecs);
+					rows = [computeWarehouseStats(tableRows, spec, computeWarehouseSourceRows(events, spec), { datasetStart, datasetEnd })];
 				} else {
 					const bArgs = applyFunnelDefaults(assertion.breakdown, funnels, profiles);
 					if (identityMap && bArgs.identityMap === undefined) bArgs.identityMap = identityMap;
@@ -476,8 +514,33 @@ export async function evaluateStories(stories, events, opts) {
 			const res = evaluateAssertion(rows, assertion, { events, profiles });
 			results.push({ name, ...res });
 		}
-		const worst = results.reduce((w, r) => VERDICT_RANK[r.verdict] < VERDICT_RANK[w] ? r.verdict : w, 'NAILED');
+		const counted = results.filter((result) => result.verdict !== 'SKIPPED');
+		const worst = counted.length
+			? counted.reduce((w, r) => VERDICT_RANK[r.verdict] < VERDICT_RANK[w] ? r.verdict : w, 'NAILED')
+			: 'SKIPPED';
 		out.push({ id: story.id, hook: story.hook, archetype: story.archetype, verdict: worst, assertions: results });
 	}
 	return out;
 }
+
+function resolveWarehouseSpec(table, warehouseSpecs) {
+	const spec = warehouseSpecs?.[table];
+	if (!spec) throw new Error(`warehouse table "${table}" is not available`);
+	return spec;
+}
+
+function resolveWarehouseRows(table, warehouseRows, warehouseSpecs) {
+	const spec = resolveWarehouseSpec(table, warehouseSpecs);
+	const rows = warehouseRows?.[table];
+	if (!Array.isArray(rows)) throw new Error(`warehouse rows for table "${table}" are not available`);
+	return rows.map((row) => withWarehouseTimestamp(row, spec));
+}
+
+function withWarehouseTimestamp(row, spec) {
+	if (Number.isFinite(row?.__t)) return row;
+	const raw = row?.[spec.timeColumn];
+	const parsed = typeof raw === 'string'
+		? Date.parse(/T/.test(raw) ? raw : `${raw}T00:00:00Z`)
+		: NaN;
+	return { ...row, __t: Number.isFinite(parsed) ? parsed / 1000 : null };
+}