@ak--47/dungeon-master 1.7.0 → 1.8.0

package/lib/orchestrators/mixpanel-sender.js

@@ -71,6 +71,7 @@ async function _sendToMixpanel(context) {
 	const { config, storage } = context;
 	const {
 		adSpendData,
+		standaloneEventData,
 		eventData,
 		groupProfilesData,
 		scdTableData,
@@ -208,6 +209,29 @@ async function _sendToMixpanel(context) {
 		importResults.adSpend = imported;
 	}
 
+	// Import standalone (identity-less) metric snapshots — v1.8.0.
+	// Same shape as the ad-spend path: its own stream, imported as events.
+	// Gated on the config so a batch-mode run without `standaloneEvents` does
+	// not attempt an empty file read.
+	const hasStandaloneConfig = Array.isArray(config.standaloneEvents) && config.standaloneEvents.length > 0;
+	if (hasStandaloneConfig && (standaloneEventData?.length > 0 || isBATCH_MODE)) {
+		log(`  Standalone Events`);
+		let standaloneToImport = u.deepClone(standaloneEventData);
+		const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && standaloneEventData && standaloneEventData.length === 0);
+		if (shouldReadFromFiles && standaloneEventData?.getWrittenFiles) {
+			const files = standaloneEventData.getWrittenFiles();
+			if (files.length > 0) standaloneToImport = files;
+		}
+		const standaloneTotal = Array.isArray(standaloneToImport) ? standaloneToImport.length : 0;
+		const imported = await mp(creds, standaloneToImport, {
+			recordType: "event",
+			...commonOpts,
+			progressCallback: makeProgressCallback(standaloneTotal),
+		});
+		log(`  -> ${comma(imported.success)} standalone events sent\n`);
+		importResults.standalone = imported;
+	}
+
 	// Import group profiles
 	if (groupProfilesData && Array.isArray(groupProfilesData) && groupProfilesData.length > 0) {
 		for (const groupEntity of groupProfilesData) {
@@ -403,11 +427,12 @@ function logProblems(problems) {
  */
 function collectWrittenFiles(storage) {
 	const files = [];
+	if (storage.warehouseManifestFile) files.push(storage.warehouseManifestFile);
 	for (const container of [storage.eventData, storage.userProfilesData, storage.adSpendData,
-		storage.mirrorEventData, storage.groupEventData]) {
+		storage.standaloneEventData, storage.mirrorEventData, storage.groupEventData]) {
 		if (container?.getWrittenFiles) files.push(...container.getWrittenFiles());
 	}
-	for (const arr of [storage.groupProfilesData, storage.scdTableData, storage.lookupTableData]) {
+	for (const arr of [storage.groupProfilesData, storage.scdTableData, storage.lookupTableData, storage.warehouseMetricData]) {
 		if (Array.isArray(arr)) {
 			for (const c of arr) {
 				if (c?.getWrittenFiles) files.push(...c.getWrittenFiles());

package/lib/orchestrators/user-loop.js

@@ -978,6 +978,7 @@ export async function userLoop(context) {
 				}
 			}
 
+			context.warehouseAccumulator?.ingest(usersEvents);
 			await eventData.hookPush(usersEvents, { profile });
 		});
 

package/lib/templates/story-spec.schema.json

@@ -85,7 +85,6 @@
 						"properties": {
 							"where": {
 								"type": "object",
-								"minProperties": 1,
 								"description": "Column → value (equality) or { op, value } comparison. All clauses must match (AND).",
 								"additionalProperties": {
 									"oneOf": [
@@ -123,23 +122,49 @@
 			"properties": {
 				"type": { "type": "string", "minLength": 1 }
 			},
-			"if": {
-				"properties": { "type": { "const": "duckdb" } }
-			},
-			"then": {
-				"required": ["type", "sql"],
-				"properties": {
-					"type": { "const": "duckdb" },
-					"sql": {
-						"type": "string",
-						"minLength": 1,
-						"description": "DuckDB SQL escape hatch, for bespoke shapes only. The runner shells out to the `duckdb` CLI (no npm dep) and substitutes the literal token {{PREFIX}} with the run's data prefix path (e.g. data/verify-<name>), so globs read read_json_auto('{{PREFIX}}-EVENTS*.json'). Result rows feed select/expect like emulator rows. Disk mode only — skipped (with a warning) under --in-memory."
+			"allOf": [
+				{
+					"if": {
+						"properties": { "type": { "const": "duckdb" } }
+					},
+					"then": {
+						"required": ["type", "sql"],
+						"properties": {
+							"type": { "const": "duckdb" },
+							"sql": {
+								"type": "string",
+								"minLength": 1,
+								"description": "DuckDB SQL escape hatch, for bespoke shapes only. The runner shells out to the `duckdb` CLI (no npm dep) and substitutes the literal token {{PREFIX}} with the run's data prefix path (e.g. data/verify-<name>), so globs read read_json_auto('{{PREFIX}}-EVENTS*.json'). Result rows feed select/expect like emulator rows. Disk mode only — skipped (with a warning) under --in-memory."
+							}
+						}
+					}
+				},
+				{
+					"if": {
+						"properties": { "type": { "const": "warehouse" } }
+					},
+					"then": {
+						"required": ["type", "table"],
+						"properties": {
+							"type": { "const": "warehouse" },
+							"table": { "type": "string", "minLength": 1 }
+						}
+					}
+				},
+				{
+					"if": {
+						"properties": { "type": { "const": "warehouse-stats" } }
+					},
+					"then": {
+						"required": ["type", "table"],
+						"properties": {
+							"type": { "const": "warehouse-stats" },
+							"table": { "type": "string", "minLength": 1 }
+						}
 					}
 				}
-			},
-			"else": {
-				"description": "Anything other than 'duckdb' is passed byte-compatible to emulateBreakdown / verifyDungeon (frequencyByFrequency, funnelFrequency, aggregatePerUser, timeToConvert, attributedBy, sessionMetrics, retention, distinctCount, eventBreakdown, uniques, lifecycle, topPaths)."
-			}
+			],
+			"description": "Anything other than 'duckdb', 'warehouse', or 'warehouse-stats' is passed byte-compatible to emulateBreakdown / verifyDungeon (frequencyByFrequency, funnelFrequency, aggregatePerUser, timeToConvert, attributedBy, sessionMetrics, retention, distinctCount, eventBreakdown, uniques, lifecycle, topPaths)."
 		},
 		"expect": {
 			"type": "object",

package/lib/utils/utils.js

@@ -815,10 +815,26 @@ function streamJSON(filePath, data, options = {}) {
 	});
 }
 
+function csvRow(item, columns) {
+	return columns.map(col => {
+		const value = item[col];
+
+		if (value === null || value === undefined) {
+			return '';
+		}
+
+		const serialized = typeof value === 'object'
+			? JSON.stringify(value)
+			: value.toString();
+
+		return `"${serialized.replace(/"/g, '""')}"`;
+	}).join(',');
+}
+
 function streamCSV(filePath, data, options = {}) {
 	return new Promise((resolve, reject) => {
 		let writeStream;
-		const { gzip = false } = options;
+		const { gzip = false, fixedColumns } = options;
 
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
@@ -842,20 +858,12 @@ function streamCSV(filePath, data, options = {}) {
 			}
 		}
 
-		// Extract all unique keys from the data array
-		const columns = getUniqueKeys(data);  // Assuming getUniqueKeys properly retrieves all keys
+		const columns = Array.isArray(fixedColumns) ? fixedColumns : getUniqueKeys(data);
 
-		// Stream the header
 		writeStream.write(columns.join(',') + '\n');
 
-		// Stream each data row
 		data.forEach(item => {
-			for (const key in item) {
-				// Ensure all nested objects are properly stringified
-				if (typeof item[key] === "object") item[key] = JSON.stringify(item[key]);
-			}
-			const row = columns.map(col => item[col] ? `"${item[col].toString().replace(/"/g, '""')}"` : "").join(',');
-			writeStream.write(row + '\n');
+			writeStream.write(csvRow(item, columns) + '\n');
 		});
 
 		writeStream.end();
@@ -1370,7 +1378,7 @@ META
  * @param  {Config} config
  */
 function buildFileNames(config) {
-	const { format = "csv", groupKeys = [], lookupTables = [] } = config;
+	const { format = "csv", groupKeys = [], lookupTables = [], warehouseMetrics = [] } = config;
 	let extension = "";
 	extension = format === "csv" ? "csv" : "json";
 	// const current = dayjs.utc().format("MM-DD-HH");
@@ -1388,10 +1396,12 @@ function buildFileNames(config) {
 		eventFiles: [path.join(writeDir, `${simName}-EVENTS.${extension}`)],
 		userFiles: [path.join(writeDir, `${simName}-USERS.${extension}`)],
 		adSpendFiles: [],
+		standaloneFiles: [],
 		scdFiles: [],
 		mirrorFiles: [],
 		groupFiles: [],
 		lookupFiles: [],
+		warehouseFiles: [],
 		folder: writeDir,
 	};
 	//add ad spend files
@@ -1399,6 +1409,11 @@ function buildFileNames(config) {
 		writePaths.adSpendFiles.push(path.join(writeDir, `${simName}-AD-SPEND.${extension}`));
 	}
 
+	//add standalone (identity-less snapshot) files — v1.8.0
+	if (Array.isArray(config?.standaloneEvents) && config.standaloneEvents.length > 0) {
+		writePaths.standaloneFiles.push(path.join(writeDir, `${simName}-STANDALONE.${extension}`));
+	}
+
 	//add SCD files
 	const scdKeys = Object.keys(config?.scdProps || {});
 	for (const key of scdKeys) {
@@ -1425,6 +1440,15 @@ function buildFileNames(config) {
 		);
 	}
 
+	for (const warehouseMetric of warehouseMetrics) {
+		const metricName = warehouseMetric?.name;
+		if (typeof metricName !== 'string') continue;
+		const metricFormat = warehouseMetric?.format || format || 'csv';
+		writePaths.warehouseFiles.push(
+			path.join(writeDir, `${simName}-WAREHOUSE-${metricName}.${metricFormat}`)
+		);
+	}
+
 	//add mirror files
 	const mirrorProps = config?.mirrorProps || {};
 	if (Object.keys(mirrorProps).length) {
@@ -1948,6 +1972,7 @@ export {
 	generateUser,
 	optimizedBoxMuller,
 	buildFileNames,
+	csvRow,
 	streamJSON,
 	streamCSV,
 	streamParquet,

package/lib/verify/index.js

@@ -17,6 +17,7 @@ export { verifyDungeon } from './verify-dungeon.js';
 // `evaluateStories` / `applyFunnelDefaults`. Use the RETURN value; it does not
 // enrich the config you pass in.
 export { validateDungeonConfig } from '../core/config-validator.js';
+export { pearson, computeWarehouseStats, computeWarehouseSourceRows, auditWarehouseRows } from './warehouse.js';
 export { deriveExpectedSchema, validateSchema } from './schema-validator.js';
 export {
 	evaluateFunnel,

package/lib/verify/story-runner.js

@@ -27,6 +27,7 @@
 
 import { emulateBreakdown } from './emulate-breakdown.js';
 import { applyFunnelDefaults } from './verify-dungeon.js';
+import { computeWarehouseSourceRows, computeWarehouseStats } from './warehouse.js';
 
 /** Closed archetype enum — MUST match lib/templates/story-spec.schema.json (unit-tested). */
 export const STORY_ARCHETYPES = [
@@ -322,6 +323,9 @@ export function validateStories(stories) {
 				err(ap, 'breakdown: required object with a string `type`');
 			} else if (a.breakdown.type === 'duckdb' && (typeof a.breakdown.sql !== 'string' || !a.breakdown.sql)) {
 				err(ap, 'breakdown: type "duckdb" requires a non-empty `sql`');
+			} else if ((a.breakdown.type === 'warehouse' || a.breakdown.type === 'warehouse-stats')
+				&& (typeof a.breakdown.table !== 'string' || !a.breakdown.table.trim())) {
+				err(ap, `breakdown: type "${a.breakdown.type}" requires a non-empty \`table\``);
 			}
 			if (a.expect === undefined && typeof a.assert !== 'function') {
 				err(ap, 'requires `expect` or a function-valued `assert`');
@@ -339,8 +343,8 @@ export function validateStories(stories) {
 					for (const [name, spec] of Object.entries(a.select)) {
 						if (!NAME_RE.test(name)) err(ap, `select: name "${name}" must be an identifier`);
 						selectNames.add(name);
-						if (!spec || typeof spec !== 'object' || !spec.where || typeof spec.where !== 'object' || !Object.keys(spec.where).length) {
-							err(ap, `select.${name}: requires a non-empty \`where\``);
+						if (!spec || typeof spec !== 'object' || !spec.where || typeof spec.where !== 'object') {
+							err(ap, `select.${name}: requires a \`where\` object`);
 							continue;
 						}
 						for (const [col, cond] of Object.entries(spec.where)) {
@@ -406,8 +410,8 @@ export function storiesToChecks(stories) {
 	for (const story of stories) {
 		story.assertions.forEach((assertion, i) => {
 			const name = `${story.id}[${i}]`;
-			if (assertion.breakdown.type === 'duckdb') {
-				console.warn(`[dungeon-master] ${name}: duckdb assertions only run in disk mode (scripts/verify-stories.mjs) — skipped in-memory.`);
+			if (assertion.breakdown.type === 'duckdb' || assertion.breakdown.type === 'warehouse' || assertion.breakdown.type === 'warehouse-stats') {
+				console.warn(`[dungeon-master] ${name}: ${assertion.breakdown.type} assertions only run in disk mode via scripts/verify-stories.mjs — skipped in verifyDungeon in-memory checks.`);
 				return;
 			}
 			checks.push({
@@ -441,11 +445,34 @@ export function storiesToChecks(stories) {
  *   auto-build over large profile sets is wasteful.
  * @param {(sql: string) => Promise<Array<Object>>} [opts.runSql] - duckdb
  *   executor (provided by the CLI in disk mode). Absent → duckdb assertions
- *   report NONE with an explanatory detail.
+ *   report NONE unless skipDiskOnlyDuckdb is set.
+ * @param {Record<string, Array<Object>>} [opts.warehouseRows] - Loaded
+ *   warehouse rows keyed by table name.
+ * @param {Record<string, Object>} [opts.warehouseSpecs] - Resolved warehouse
+ *   metric specs keyed by table name.
+ * @param {string|number} [opts.datasetStart] - Dataset window start for
+ *   in-window warehouse stats comparisons.
+ * @param {string|number} [opts.datasetEnd] - Dataset window end for in-window
+ *   warehouse stats comparisons.
+ * @param {boolean} [opts.skipDiskOnlyDuckdb] - Mark duckdb assertions skipped
+ *   instead of NONE when no SQL executor is available.
+ * @param {(message: string) => void} [opts.onWarning] - Warning sink for
+ *   skipped disk-only assertions.
  * @returns {Promise<Array<{ id: string, hook: string, archetype: string, verdict: string, assertions: Array<{ name: string, verdict: string, observed: number|null, detail: string }> }>>}
  */
 export async function evaluateStories(stories, events, opts) {
-	const { profiles, funnels, runSql, identityMap } = opts || {};
+	const {
+		profiles,
+		funnels,
+		runSql,
+		identityMap,
+		warehouseRows,
+		warehouseSpecs,
+		datasetStart,
+		datasetEnd,
+		skipDiskOnlyDuckdb,
+		onWarning,
+	} = opts || {};
 	const v = validateStories(stories);
 	if (!v.valid) {
 		throw new Error(`evaluateStories: invalid stories:\n  ${v.errors.join('\n  ')}`);
@@ -460,10 +487,21 @@ export async function evaluateStories(stories, events, opts) {
 			try {
 				if (assertion.breakdown.type === 'duckdb') {
 					if (!runSql) {
-						results.push({ name, verdict: 'NONE', observed: null, detail: 'duckdb assertion requires disk mode (no SQL executor available)' });
+						if (skipDiskOnlyDuckdb) {
+							onWarning?.(`[dungeon-master] ${name}: duckdb assertions only run in disk mode (scripts/verify-stories.mjs) — skipped in-memory.`);
+							results.push({ name, verdict: 'SKIPPED', observed: null, detail: 'duckdb assertion requires disk mode (skipped in-memory)' });
+						} else {
+							results.push({ name, verdict: 'NONE', observed: null, detail: 'duckdb assertion requires disk mode (no SQL executor available)' });
+						}
 						continue;
 					}
 					rows = await runSql(assertion.breakdown.sql);
+				} else if (assertion.breakdown.type === 'warehouse') {
+					rows = resolveWarehouseRows(assertion.breakdown.table, warehouseRows, warehouseSpecs);
+				} else if (assertion.breakdown.type === 'warehouse-stats') {
+					const spec = resolveWarehouseSpec(assertion.breakdown.table, warehouseSpecs);
+					const tableRows = resolveWarehouseRows(assertion.breakdown.table, warehouseRows, warehouseSpecs);
+					rows = [computeWarehouseStats(tableRows, spec, computeWarehouseSourceRows(events, spec), { datasetStart, datasetEnd })];
 				} else {
 					const bArgs = applyFunnelDefaults(assertion.breakdown, funnels, profiles);
 					if (identityMap && bArgs.identityMap === undefined) bArgs.identityMap = identityMap;
@@ -476,8 +514,33 @@ export async function evaluateStories(stories, events, opts) {
 			const res = evaluateAssertion(rows, assertion, { events, profiles });
 			results.push({ name, ...res });
 		}
-		const worst = results.reduce((w, r) => VERDICT_RANK[r.verdict] < VERDICT_RANK[w] ? r.verdict : w, 'NAILED');
+		const counted = results.filter((result) => result.verdict !== 'SKIPPED');
+		const worst = counted.length
+			? counted.reduce((w, r) => VERDICT_RANK[r.verdict] < VERDICT_RANK[w] ? r.verdict : w, 'NAILED')
+			: 'SKIPPED';
 		out.push({ id: story.id, hook: story.hook, archetype: story.archetype, verdict: worst, assertions: results });
 	}
 	return out;
 }
+
+function resolveWarehouseSpec(table, warehouseSpecs) {
+	const spec = warehouseSpecs?.[table];
+	if (!spec) throw new Error(`warehouse table "${table}" is not available`);
+	return spec;
+}
+
+function resolveWarehouseRows(table, warehouseRows, warehouseSpecs) {
+	const spec = resolveWarehouseSpec(table, warehouseSpecs);
+	const rows = warehouseRows?.[table];
+	if (!Array.isArray(rows)) throw new Error(`warehouse rows for table "${table}" are not available`);
+	return rows.map((row) => withWarehouseTimestamp(row, spec));
+}
+
+function withWarehouseTimestamp(row, spec) {
+	if (Number.isFinite(row?.__t)) return row;
+	const raw = row?.[spec.timeColumn];
+	const parsed = typeof raw === 'string'
+		? Date.parse(/T/.test(raw) ? raw : `${raw}T00:00:00Z`)
+		: NaN;
+	return { ...row, __t: Number.isFinite(parsed) ? parsed / 1000 : null };
+}