@ak--47/dungeon-master 1.7.0 → 1.8.1

package/lib/hook-helpers/shape.js

@@ -114,6 +114,9 @@ export function applyLifecycleWave(events, uid, opts) {
  * For users where `hashFloat(uid) < share`, injects the `path` sequence —
  * each step cloned from the user's OWN existing event of that name — right
  * after the user's FIRST `anchor` occurrence, with tight monotonic gaps.
+ * This is append-only: original traffic is never moved or removed. Events
+ * already between the anchor and clones can interrupt the immediate branch.
+ * `share` selects injection recipients, not the measured Flows branch share.
  *
  * Why these rules (HOOKS.md §2.17): Flows' unique mode reads only the FIRST
  * flow per user, so the injection anchors on the first occurrence; gaps are
@@ -192,12 +195,23 @@ export function applyPathBias(events, uid, opts) {
  * Boundary guarantees (what makes derived sessions deterministic against
  * jitter — HOOKS.md §2.13): intra-session gaps stay well under Mixpanel's
  * 30-min timeout (even spacing capped at 20min + bounded jitter, worst case
- * <28min); inter-session gaps stay well over it (sessions land on distinct
- * days when possible; same-day sessions are centered in equal partitions of
- * the day, spaced ≥¼ partition — guaranteed >30min up to 8 sessions/day);
+ * <28min); with explicit bounds, inter-session gaps stay strictly over it (sessions land on distinct
+ * days when possible; same-day slots reserve at least 30min + 1ms between
+ * clusters, compressing their spans when necessary);
  * and no engineered session crosses UTC midnight (a day-boundary split would
  * cut it — `session_query.cpp` daySplit). Valid precisely because of P2.1:
  * session_ids are re-derived after the everything hook.
+ * With both bounds omitted, placement keeps the legacy full-UTC-day slots
+ * between the user's first and last active days. Overfull legacy requests
+ * still run, but their clusters can merge under the 30-minute timeout.
+ * Optional bounds are additive: supply known dataset limits to constrain
+ * placement; the helper cannot infer datasetEnd from a user's last event.
+ * An omitted side defaults to the corresponding full UTC day edge. Partial
+ * days compress clusters, including zero-span clusters. Only explicit-bound
+ * mode throws for insufficient per-week capacity, before any record changes.
+ * The helper never silently reduces the cluster target or drops events;
+ * legacy full-day placement can exceed an unknown dataset end and be clipped
+ * later by the engine. Pass metadata bounds to prevent that clipping.
  *
  * @param {Array<Object>} events - Full user event array (everything hook).
  * @param {string} uid - Unused for hashing here; kept for atom-signature
@@ -207,10 +221,15 @@ export function applyPathBias(events, uid, opts) {
  * @param {number} opts.sessionsPerWeek - Target clusters per week (≥1).
  * @param {number} opts.eventsPerSession - Target events per cluster (≥1).
  * @param {number} opts.sessionMinutes - Max cluster span in minutes.
+ * @param {string|number} [opts.datasetStart] - Inclusive lower bound (ISO,
+ *   unix seconds, or unix milliseconds); accepts meta.datasetStart directly.
+ * @param {string|number} [opts.datasetEnd] - Inclusive upper bound (same units);
+ *   accepts meta.datasetEnd directly.
+ * @throws {RangeError} Invalid explicit bounds or insufficient bounded 30-minute-session capacity.
  * @returns {Array<Object>} The SAME array, timestamps rewritten.
  */
 export function applySessionShape(events, uid, opts) {
-	const { sessionsPerWeek, eventsPerSession, sessionMinutes } = opts || {};
+	const { sessionsPerWeek, eventsPerSession, sessionMinutes, datasetStart, datasetEnd } = opts || {};
 	if (!Array.isArray(events) || !events.length) return events;
 	if (!isPos(sessionsPerWeek) || !isPos(eventsPerSession) || !isPos(sessionMinutes)) return events;
 
@@ -221,6 +240,13 @@ export function applySessionShape(events, uid, opts) {
 	const N = timed.length;
 	const firstMs = toMs(timed[0].time);
 	const lastMs = toMs(timed[N - 1].time);
+	const bounded = datasetStart !== undefined || datasetEnd !== undefined;
+	const lowerMs = datasetStart === undefined ? Math.floor(firstMs / DAY_MS) * DAY_MS : toMs(datasetStart);
+	const upperMs = datasetEnd === undefined ? (Math.floor(lastMs / DAY_MS) + 1) * DAY_MS - 1 : toMs(datasetEnd);
+	if (!Number.isFinite(lowerMs) || !Number.isFinite(upperMs) || lowerMs > upperMs ||
+		Math.abs(lowerMs) > 8.64e15 || Math.abs(upperMs) > 8.64e15) {
+		throw new RangeError('applySessionShape: invalid dataset bounds');
+	}
 	const weeks = Math.max(1, Math.ceil((lastMs - firstMs + 1) / WEEK_MS));
 	const numSessions = Math.max(1, Math.min(
 		Math.floor(sessionsPerWeek) * weeks,
@@ -235,6 +261,15 @@ export function applySessionShape(events, uid, opts) {
 	const chance = getChance();
 	const firstDay = Math.floor(firstMs / DAY_MS);
 	const lastDay = Math.floor(lastMs / DAY_MS);
+	const separationMs = 30 * MIN_MS + 1;
+	const perDayCount = new Map();
+	const dayBounds = day => [Math.ceil(Math.max(day * DAY_MS, lowerMs)),
+		Math.floor(Math.min((day + 1) * DAY_MS - 1, upperMs))];
+	const remainingCapacity = day => {
+		if (!bounded) return Infinity;
+		const [start, end] = dayBounds(day);
+		return (end >= start ? Math.floor((end - start) / separationMs) + 1 : 0) - (perDayCount.get(day) || 0);
+	};
 
 	// Pick one day per session, week by week: prefer the user's original
 	// active days in the week, fill from the rest of the week's days, and
@@ -251,22 +286,38 @@ export function applySessionShape(events, uid, opts) {
 			const t = toMs(ev.time);
 			if (t >= weekStartMs && t < weekStartMs + WEEK_MS) originalDays.add(Math.floor(t / DAY_MS));
 		}
-		const pool = [...originalDays].filter(d => d >= dayLo && d <= dayHi);
+		const pool = [...originalDays].filter(d => d >= dayLo && d <= dayHi && remainingCapacity(d) > 0);
 		const others = [];
-		for (let d = dayLo; d <= dayHi; d++) if (!originalDays.has(d)) others.push(d);
+		for (let d = dayLo; d <= dayHi; d++) if (!originalDays.has(d) && remainingCapacity(d) > 0) others.push(d);
 		const picked = chance.pickset(pool, Math.min(need, pool.length));
 		if (picked.length < need) picked.push(...chance.pickset(others, Math.min(need - picked.length, others.length)));
-		let i = 0;
-		while (picked.length < need) picked.push(picked[i++ % Math.max(1, picked.length)]); // reuse days: > days/week sessions
+		for (const day of picked) perDayCount.set(day, (perDayCount.get(day) || 0) + 1);
+		const candidates = [...pool, ...others];
+		while (picked.length < need) {
+			if (!bounded) {
+				let reuseIndex = 0;
+				while (picked.length < need) {
+					const day = picked[reuseIndex++ % Math.max(1, picked.length)];
+					picked.push(day);
+					perDayCount.set(day, (perDayCount.get(day) || 0) + 1);
+				}
+				break;
+			}
+			const available = candidates.filter(day => remainingCapacity(day) > 0);
+			if (!available.length) {
+				throw new RangeError(`applySessionShape: insufficient session capacity in week ${w + 1} for ${need} clusters`);
+			}
+			for (const day of available) {
+				picked.push(day);
+				perDayCount.set(day, (perDayCount.get(day) || 0) + 1);
+				if (picked.length === need) break;
+			}
+		}
 		picked.sort((a, b) => a - b);
 		sessionDays.push(...picked);
 	}
 
-	// Place sessions within days. Same-day sessions get equal partitions of
-	// the day; the session is centered in its partition with bounded jitter,
-	// which guarantees the inter-session and midnight invariants above.
-	const perDayCount = new Map();
-	for (const d of sessionDays) perDayCount.set(d, (perDayCount.get(d) || 0) + 1);
+	// Bound each day's slots and reserve the strict timeout gap between them.
 	const perDaySeen = new Map();
 	const sesMs = sessionMinutes * MIN_MS;
 
@@ -280,10 +331,15 @@ export function applySessionShape(events, uid, opts) {
 		const j = perDaySeen.get(day) || 0;
 		perDaySeen.set(day, j + 1);
 
-		const seg = DAY_MS / m;
-		const span = Math.min(sesMs, seg * 0.5);
-		const center = day * DAY_MS + j * seg + (seg - span) / 2;
-		const q = Math.floor((seg - span) / 4);
+		const [dayStart, dayEnd] = dayBounds(day);
+		const seg = (dayEnd - dayStart - (m - 1) * separationMs) / m;
+		const slotStart = Math.floor(dayStart + j * (seg + separationMs));
+		const slotEnd = Math.floor(dayStart + (j + 1) * seg + j * separationMs);
+		const slotWidth = bounded ? slotEnd - slotStart : DAY_MS / m;
+		const span = bounded ? Math.floor(Math.min(sesMs, slotWidth * 0.5)) : Math.min(sesMs, slotWidth * 0.5);
+		const center = bounded ? slotStart + Math.floor((slotWidth - span) / 2)
+			: day * DAY_MS + j * slotWidth + (slotWidth - span) / 2;
+		const q = Math.floor((slotWidth - span) / 4);
 		const start = center + (q > 0 ? chance.integer({ min: -q, max: q }) : 0);
 
 		if (size === 1) {

package/lib/orchestrators/mixpanel-sender.js

@@ -71,6 +71,7 @@ async function _sendToMixpanel(context) {
 	const { config, storage } = context;
 	const {
 		adSpendData,
+		standaloneEventData,
 		eventData,
 		groupProfilesData,
 		scdTableData,
@@ -208,6 +209,29 @@ async function _sendToMixpanel(context) {
 		importResults.adSpend = imported;
 	}
 
+	// Import standalone (identity-less) metric snapshots — v1.8.0.
+	// Same shape as the ad-spend path: its own stream, imported as events.
+	// Gated on the config so a batch-mode run without `standaloneEvents` does
+	// not attempt an empty file read.
+	const hasStandaloneConfig = Array.isArray(config.standaloneEvents) && config.standaloneEvents.length > 0;
+	if (hasStandaloneConfig && (standaloneEventData?.length > 0 || isBATCH_MODE)) {
+		log(`  Standalone Events`);
+		let standaloneToImport = u.deepClone(standaloneEventData);
+		const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && standaloneEventData && standaloneEventData.length === 0);
+		if (shouldReadFromFiles && standaloneEventData?.getWrittenFiles) {
+			const files = standaloneEventData.getWrittenFiles();
+			if (files.length > 0) standaloneToImport = files;
+		}
+		const standaloneTotal = Array.isArray(standaloneToImport) ? standaloneToImport.length : 0;
+		const imported = await mp(creds, standaloneToImport, {
+			recordType: "event",
+			...commonOpts,
+			progressCallback: makeProgressCallback(standaloneTotal),
+		});
+		log(`  -> ${comma(imported.success)} standalone events sent\n`);
+		importResults.standalone = imported;
+	}
+
 	// Import group profiles
 	if (groupProfilesData && Array.isArray(groupProfilesData) && groupProfilesData.length > 0) {
 		for (const groupEntity of groupProfilesData) {
@@ -403,11 +427,12 @@ function logProblems(problems) {
  */
 function collectWrittenFiles(storage) {
 	const files = [];
+	if (storage.warehouseManifestFile) files.push(storage.warehouseManifestFile);
 	for (const container of [storage.eventData, storage.userProfilesData, storage.adSpendData,
-		storage.mirrorEventData, storage.groupEventData]) {
+		storage.standaloneEventData, storage.mirrorEventData, storage.groupEventData]) {
 		if (container?.getWrittenFiles) files.push(...container.getWrittenFiles());
 	}
-	for (const arr of [storage.groupProfilesData, storage.scdTableData, storage.lookupTableData]) {
+	for (const arr of [storage.groupProfilesData, storage.scdTableData, storage.lookupTableData, storage.warehouseMetricData]) {
 		if (Array.isArray(arr)) {
 			for (const c of arr) {
 				if (c?.getWrittenFiles) files.push(...c.getWrittenFiles());

package/lib/orchestrators/user-loop.js

@@ -11,7 +11,7 @@ import pLimit from 'p-limit';
 import os from 'os';
 import * as u from "../utils/utils.js";
 import * as t from 'ak-tools';
-import { makeEvent } from "../generators/events.js";
+import { makeEvent, engineIdentity } from "../generators/events.js";
 import { makeFunnel } from "../generators/funnels.js";
 import { makeUserProfile } from "../generators/profiles.js";
 import { makeSCD } from "../generators/scd.js";
@@ -403,6 +403,8 @@ export async function userLoop(context) {
 			}
 
 			let userFirstEventTime;
+			let userUsageStartTime = context.FIXED_BEGIN;
+			const promisedAttemptEntries = new Set();
 
 			// ── v1.5 Active-day scheduling ──
 			// When `avgActiveDaysPerUser` is set, build a per-user day plan: a list
@@ -493,13 +495,9 @@ export async function userLoop(context) {
 				// Active-day mode: anchor the first funnel on a picked day so the user's
 				// signup lands within their planned active window. Legacy mode: anchor at
 				// adjustedCreated minus a noise offset.
-				let cursor;
-				if (dayPlan) {
-					const bounds = nextDayBounds();
-					cursor = bounds ? bounds.earliest : adjustedCreated.subtract(noise(), 'seconds').unix();
-				} else {
-					cursor = adjustedCreated.subtract(noise(), 'seconds').unix();
-				}
+				const lifecycleStart = Math.max(adjustedCreated.valueOf() / 1000, context.FIXED_BEGIN);
+				const firstDayBounds = dayPlan && !hasRetentionCurve ? nextDayBounds() : null;
+				let cursor = Math.max(lifecycleStart, firstDayBounds?.earliest ?? lifecycleStart);
 
 				// Resolve attempts plan. `attempts.{min,max}` count FAILED PRIORS; total
 				// passes = failedPriors + 1.  Validator coerced bounds; default both 0.
@@ -509,6 +507,7 @@ export async function userLoop(context) {
 				const failedPriors = (maxA > 0) ? chance.integer({ min: minA, max: maxA }) : 0;
 				const totalAttempts = failedPriors + 1;
 				let firstAttemptFirstEventTime = null;
+				if (failedPriors > 0) cursor = lifecycleStart;
 
 				for (let attemptNum = 1; attemptNum <= totalAttempts; attemptNum++) {
 					const isFinal = attemptNum === totalAttempts;
@@ -528,16 +527,23 @@ export async function userLoop(context) {
 						truncateBeforeAuth: !isFinal,
 						devicePool: userDevicePool,
 					};
+					const firstFeatureCtx = {
+						...featureCtx,
+						latestTime: firstDayBounds && failedPriors === 0 ? firstDayBounds.latest : context.FIXED_NOW,
+						pinFirstTime: hasRetentionCurve || failedPriors > 0,
+					};
 					const [data, converted, authMs] = await makeFunnel(
-						context, funnelToRun, user, cursor, profile, userSCD, persona, featureCtx, attemptMeta
+						context, funnelToRun, user, cursor, profile, userSCD, persona, firstFeatureCtx, attemptMeta
 					);
 					if (isFinal) userConverted = converted;
 					if (data && data.length) {
+						if (failedPriors > 0) promisedAttemptEntries.add(data[0].insert_id);
 						if (firstAttemptFirstEventTime === null) {
 							firstAttemptFirstEventTime = dayjs(data[0].time).unix();
 						}
 						// Advance the cursor for the next attempt by a small abandon-and-retry gap.
-						const lastTime = dayjs(data[data.length - 1].time).unix();
+						const lastTime = Math.max(...data.map(event => Date.parse(event.time))) / 1000;
+						userUsageStartTime = Math.max(userUsageStartTime, lastTime + 0.001);
 						cursor = lastTime + chance.integer({ min: 60, max: 30 * 60 }); // 1–30 min later
 						numEventsPreformed += data.length;
 						usersEvents = usersEvents.concat(data);
@@ -547,6 +553,13 @@ export async function userLoop(context) {
 						if (userAuthTimeMs === null) userAuthTimeMs = authMs;
 						userAuthed = true;
 					}
+					if (!isFinal && !data.length && config.events.find(event => event.event === firstFunnel.sequence[0])?.isAuthEvent) {
+						context.addWarning({
+							key: 'lifecycle.emptyPreAuthAttempt',
+							requested: 1, applied: 0, severity: 'warn',
+							reason: 'A failed prior has no pre-auth step because auth starts the funnel; the prior emits no rows and the final attempt still runs.',
+						});
+					}
 				}
 
 				userFirstEventTime = firstAttemptFirstEventTime !== null
@@ -614,6 +627,8 @@ export async function userLoop(context) {
 				// gets re-anchored to the picked day's start (subsequent funnel steps
 				// spill within `timeToConvert` hours; this is intentional).
 				const dayBounds = dayPlan ? nextDayBounds() : null;
+				const usageEarliest = Math.max(dayBounds?.earliest ?? userFirstEventTime, userUsageStartTime);
+				if (usageEarliest > (dayBounds?.latest ?? context.FIXED_NOW)) continue;
 				// Compute step1's `latestTime` so the funnel's relative span fits before
 				// FIXED_NOW. Without this, born-late users + long-ttc funnels generate
 				// large numbers of `_drop`'d events that consume budget cycles. The
@@ -639,7 +654,7 @@ export async function userLoop(context) {
 					const ttcSec = (currentFunnel.timeToConvert || 0) * 3600;
 					// Anchor cursor at picked day's start when active-day mode is on,
 					// otherwise pass userFirstEventTime (constant). NO cursor accumulation.
-					const funnelCursor = dayBounds ? dayBounds.earliest : userFirstEventTime;
+					const funnelCursor = usageEarliest;
 					// Constrain funnel step1's TimeSoup latestTime so the full funnel fits in
 					// window. Without this, late steps spill past FIXED_NOW and get `_drop`'d.
 					//
@@ -683,7 +698,7 @@ export async function userLoop(context) {
 					// — passing `true` here would pin every standalone event to the same
 					// `earliestTime`, which the now-deleted bunchIntoSessions used to paper
 					// over. With bunchIntoSessions removed, TimeSoup is the time source.
-					const standaloneEarliest = dayBounds ? dayBounds.earliest : userFirstEventTime;
+					const standaloneEarliest = usageEarliest;
 					// v1.5.1: pass `config.superProps` so standalone events get super-property
 					// stamping. Pre-1.5.1, standalone events received `{}` here, but the
 					// validator's auto-funnel (catch-all) consumed all non-strict events so
@@ -763,6 +778,8 @@ export async function userLoop(context) {
 					for (const ev of usersEvents) {
 						if (chance.bool({ likelihood: dataQuality.duplicateRate * 100 })) {
 							const dupe = { ...ev };
+							const identityDescriptor = Object.getOwnPropertyDescriptor(ev, engineIdentity);
+							if (identityDescriptor) Object.defineProperty(dupe, engineIdentity, identityDescriptor);
 							dupe.time = dayjs(ev.time).add(chance.integer({ min: 1, max: 60 }), 'seconds').toISOString();
 							dupe.insert_id = randomUUID();
 							dupes.push(dupe);
@@ -828,6 +845,13 @@ export async function userLoop(context) {
 				profile._drop = true;
 			}
 
+			const identityBeforeEverything = new Map(usersEvents.map(event => [event.insert_id, {
+				user_id: event.user_id, device_id: event.device_id,
+				original: event[engineIdentity],
+			}]));
+			const profileDropBeforeEverything = profile._drop;
+			let profileDropOverridden = false;
+
 			// Hook for processing all user events (hooks override everything)
 			if (config.hook) {
 				// `meta.isPreAuth(event)` predicate bound to this user's auth state.
@@ -843,7 +867,16 @@ export async function userLoop(context) {
 					return Number.isFinite(t) ? t < userAuthTimeMsLocal : false;
 				};
 				const newEvents = await config.hook(usersEvents, "everything", {
-					profile,
+					profile: new Proxy(profile, {
+						deleteProperty(target, key) {
+							if (key === '_drop') profileDropOverridden = true;
+							return Reflect.deleteProperty(target, key);
+						},
+						set(target, key, value) {
+							if (key === '_drop') profileDropOverridden = true;
+							return Reflect.set(target, key, value);
+						},
+					}),
 					scd: userSCD,
 					config,
 					userIsBornInDataset,
@@ -944,13 +977,44 @@ export async function userLoop(context) {
 			// stream to the remaining room with a seeded uniform sample (keeps the
 			// time shape; preserves array order). Only under the flag.
 			if (strictEventCount && usersEvents.length > 0) {
+				const promisedEntries = usersEvents.filter(event => promisedAttemptEntries.has(event.insert_id));
 				const room = numEvents - context.getStoredEventCount();
 				if (room <= 0) {
 					usersEvents = [];
 				} else if (usersEvents.length > room) {
-					const keep = new Set(chance.pickset(usersEvents, room));
+					const reserved = promisedEntries.slice(0, room);
+					const others = usersEvents.filter(event => !promisedAttemptEntries.has(event.insert_id));
+					const keep = new Set([...reserved, ...chance.pickset(others, Math.max(0, room - reserved.length))]);
 					usersEvents = usersEvents.filter(e => keep.has(e));
 				}
+				if (promisedEntries.some(event => !usersEvents.includes(event))) {
+					context.addWarning({
+						key: 'lifecycle.strictAttemptBudget',
+						requested: promisedEntries.length, applied: Math.max(0, room), severity: 'warn',
+						reason: 'The remaining strict event budget is smaller than the surviving first-funnel attempt entries; strictEventCount takes precedence and later entries are omitted.',
+					});
+				}
+			}
+
+			if (userIsBornInDataset && userDevicePool?.length) {
+				const authNames = new Set(config.events.filter(event => event.isAuthEvent).map(event => event.event));
+				const stitches = usersEvents.filter(event => authNames.has(event.event) && event.user_id && event.device_id);
+				const emittedAuthTime = stitches.length ? Math.min(...stitches.map(event => Date.parse(event.time))) : null;
+				for (const event of usersEvents) {
+					if (event.event === '$experiment_started') continue;
+					if (emittedAuthTime === null || Date.parse(event.time) < emittedAuthTime) {
+						const before = identityBeforeEverything.get(event.insert_id);
+						const original = event[engineIdentity] || before?.original;
+						if (!original) continue;
+						const explicitUser = event.user_id !== original.user_id;
+						const explicitDevice = before
+							? event.device_id !== before.device_id || (!before.device_id && !!original.device_id)
+							: event.device_id !== original.device_id;
+						if (!explicitUser) delete event.user_id;
+						if (!explicitUser && !explicitDevice) event.device_id ||= userDevicePool[0];
+					}
+				}
+				if (emittedAuthTime === null && !profileDropOverridden && profile._drop === profileDropBeforeEverything) profile._drop = true;
 			}
 
 			// Store all user data (skip profile push when a hook returned null
@@ -978,6 +1042,7 @@ export async function userLoop(context) {
 				}
 			}
 
+			context.warehouseAccumulator?.ingest(usersEvents);
 			await eventData.hookPush(usersEvents, { profile });
 		});
 
@@ -1107,7 +1172,10 @@ export function amplifyWorldEvents(events, worldEvents, chance, fixedNow) {
 			if (frac > 0 && chance.bool({ likelihood: frac * 100 })) copies++;
 			for (let c = 0; c < copies; c++) {
 				const t = chance.integer({ min: windowStart, max: windowEnd - 1 });
-				clones.push({ ...ev, time: dayjs.unix(t).toISOString(), insert_id: randomUUID() });
+				const clone = { ...ev, time: dayjs.unix(t).toISOString(), insert_id: randomUUID() };
+				const identityDescriptor = Object.getOwnPropertyDescriptor(ev, engineIdentity);
+				if (identityDescriptor) Object.defineProperty(clone, engineIdentity, identityDescriptor);
+				clones.push(clone);
 			}
 		}
 	}

package/lib/templates/story-spec.schema.json

@@ -85,7 +85,6 @@
 						"properties": {
 							"where": {
 								"type": "object",
-								"minProperties": 1,
 								"description": "Column → value (equality) or { op, value } comparison. All clauses must match (AND).",
 								"additionalProperties": {
 									"oneOf": [
@@ -123,23 +122,49 @@
 			"properties": {
 				"type": { "type": "string", "minLength": 1 }
 			},
-			"if": {
-				"properties": { "type": { "const": "duckdb" } }
-			},
-			"then": {
-				"required": ["type", "sql"],
-				"properties": {
-					"type": { "const": "duckdb" },
-					"sql": {
-						"type": "string",
-						"minLength": 1,
-						"description": "DuckDB SQL escape hatch, for bespoke shapes only. The runner shells out to the `duckdb` CLI (no npm dep) and substitutes the literal token {{PREFIX}} with the run's data prefix path (e.g. data/verify-<name>), so globs read read_json_auto('{{PREFIX}}-EVENTS*.json'). Result rows feed select/expect like emulator rows. Disk mode only — skipped (with a warning) under --in-memory."
+			"allOf": [
+				{
+					"if": {
+						"properties": { "type": { "const": "duckdb" } }
+					},
+					"then": {
+						"required": ["type", "sql"],
+						"properties": {
+							"type": { "const": "duckdb" },
+							"sql": {
+								"type": "string",
+								"minLength": 1,
+								"description": "DuckDB SQL escape hatch, for bespoke shapes only. The runner shells out to the `duckdb` CLI (no npm dep) and substitutes the literal token {{PREFIX}} with the run's data prefix path (e.g. data/verify-<name>), so globs read read_json_auto('{{PREFIX}}-EVENTS*.json'). Result rows feed select/expect like emulator rows. Disk mode only — skipped (with a warning) under --in-memory."
+							}
+						}
+					}
+				},
+				{
+					"if": {
+						"properties": { "type": { "const": "warehouse" } }
+					},
+					"then": {
+						"required": ["type", "table"],
+						"properties": {
+							"type": { "const": "warehouse" },
+							"table": { "type": "string", "minLength": 1 }
+						}
+					}
+				},
+				{
+					"if": {
+						"properties": { "type": { "const": "warehouse-stats" } }
+					},
+					"then": {
+						"required": ["type", "table"],
+						"properties": {
+							"type": { "const": "warehouse-stats" },
+							"table": { "type": "string", "minLength": 1 }
+						}
 					}
 				}
-			},
-			"else": {
-				"description": "Anything other than 'duckdb' is passed byte-compatible to emulateBreakdown / verifyDungeon (frequencyByFrequency, funnelFrequency, aggregatePerUser, timeToConvert, attributedBy, sessionMetrics, retention, distinctCount, eventBreakdown, uniques, lifecycle, topPaths)."
-			}
+			],
+			"description": "Anything other than 'duckdb', 'warehouse', or 'warehouse-stats' is passed byte-compatible to emulateBreakdown / verifyDungeon (frequencyByFrequency, funnelFrequency, aggregatePerUser, timeToConvert, attributedBy, sessionMetrics, retention, distinctCount, eventBreakdown, uniques, lifecycle, topPaths)."
 		},
 		"expect": {
 			"type": "object",

package/lib/utils/utils.js

@@ -815,10 +815,26 @@ function streamJSON(filePath, data, options = {}) {
 	});
 }
 
+function csvRow(item, columns) {
+	return columns.map(col => {
+		const value = item[col];
+
+		if (value === null || value === undefined) {
+			return '';
+		}
+
+		const serialized = typeof value === 'object'
+			? JSON.stringify(value)
+			: value.toString();
+
+		return `"${serialized.replace(/"/g, '""')}"`;
+	}).join(',');
+}
+
 function streamCSV(filePath, data, options = {}) {
 	return new Promise((resolve, reject) => {
 		let writeStream;
-		const { gzip = false } = options;
+		const { gzip = false, fixedColumns } = options;
 
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
@@ -842,20 +858,12 @@ function streamCSV(filePath, data, options = {}) {
 			}
 		}
 
-		// Extract all unique keys from the data array
-		const columns = getUniqueKeys(data);  // Assuming getUniqueKeys properly retrieves all keys
+		const columns = Array.isArray(fixedColumns) ? fixedColumns : getUniqueKeys(data);
 
-		// Stream the header
 		writeStream.write(columns.join(',') + '\n');
 
-		// Stream each data row
 		data.forEach(item => {
-			for (const key in item) {
-				// Ensure all nested objects are properly stringified
-				if (typeof item[key] === "object") item[key] = JSON.stringify(item[key]);
-			}
-			const row = columns.map(col => item[col] ? `"${item[col].toString().replace(/"/g, '""')}"` : "").join(',');
-			writeStream.write(row + '\n');
+			writeStream.write(csvRow(item, columns) + '\n');
 		});
 
 		writeStream.end();
@@ -1370,7 +1378,7 @@ META
  * @param  {Config} config
  */
 function buildFileNames(config) {
-	const { format = "csv", groupKeys = [], lookupTables = [] } = config;
+	const { format = "csv", groupKeys = [], lookupTables = [], warehouseMetrics = [] } = config;
 	let extension = "";
 	extension = format === "csv" ? "csv" : "json";
 	// const current = dayjs.utc().format("MM-DD-HH");
@@ -1388,10 +1396,12 @@ function buildFileNames(config) {
 		eventFiles: [path.join(writeDir, `${simName}-EVENTS.${extension}`)],
 		userFiles: [path.join(writeDir, `${simName}-USERS.${extension}`)],
 		adSpendFiles: [],
+		standaloneFiles: [],
 		scdFiles: [],
 		mirrorFiles: [],
 		groupFiles: [],
 		lookupFiles: [],
+		warehouseFiles: [],
 		folder: writeDir,
 	};
 	//add ad spend files
@@ -1399,6 +1409,11 @@ function buildFileNames(config) {
 		writePaths.adSpendFiles.push(path.join(writeDir, `${simName}-AD-SPEND.${extension}`));
 	}
 
+	//add standalone (identity-less snapshot) files — v1.8.0
+	if (Array.isArray(config?.standaloneEvents) && config.standaloneEvents.length > 0) {
+		writePaths.standaloneFiles.push(path.join(writeDir, `${simName}-STANDALONE.${extension}`));
+	}
+
 	//add SCD files
 	const scdKeys = Object.keys(config?.scdProps || {});
 	for (const key of scdKeys) {
@@ -1425,6 +1440,15 @@ function buildFileNames(config) {
 		);
 	}
 
+	for (const warehouseMetric of warehouseMetrics) {
+		const metricName = warehouseMetric?.name;
+		if (typeof metricName !== 'string') continue;
+		const metricFormat = warehouseMetric?.format || format || 'csv';
+		writePaths.warehouseFiles.push(
+			path.join(writeDir, `${simName}-WAREHOUSE-${metricName}.${metricFormat}`)
+		);
+	}
+
 	//add mirror files
 	const mirrorProps = config?.mirrorProps || {};
 	if (Object.keys(mirrorProps).length) {
@@ -1948,6 +1972,7 @@ export {
 	generateUser,
 	optimizedBoxMuller,
 	buildFileNames,
+	csvRow,
 	streamJSON,
 	streamCSV,
 	streamParquet,