@ak--47/dungeon-master 1.7.0 → 1.8.1

package/lib/core/config-validator.js

@@ -17,6 +17,8 @@ import { resolveSoup } from "../templates/soup-presets.js";
 import { resolveMacro } from "../templates/macro-presets.js";
 import { locations as LOCATION_TEMPLATE } from "../templates/defaults.js";
 import { CONDITION_OPERATORS } from "../utils/conditions.js";
+import { validateStandaloneEvents } from "../generators/standalone.js";
+import { validateWarehouseMetrics } from "../generators/warehouse.js";
 
 /**
  * v1.7.0 (P2-2): one entry per value the validator changed or flagged. Collected
@@ -653,6 +655,22 @@ export function validateDungeonConfig(config) {
 	/** @type {EngineWarning[]} */
 	const warnings = [];
 
+	// v1.8.0 — identity-less metric snapshots. Throws on anything malformed;
+	// a silent skip would drop a whole data stream without the author noticing.
+	const standaloneEvents = validateStandaloneEvents(config.standaloneEvents);
+	const {
+		warehouseMetrics,
+		warnings: warehouseMetricWarnings,
+	} = validateWarehouseMetrics(config);
+	for (const reason of warehouseMetricWarnings) {
+		const key = String(reason).split(' ')[0] || 'warehouseMetrics';
+		warnings.push({
+			key,
+			reason,
+			severity: /clamp/i.test(reason) ? 'clamp' : 'warn',
+		});
+	}
+
 	// v1.7.0 (R2-2): resolve singleCountry to the template's canonical name (accepts
 	// ISO code or full name, case-insensitive) or throw. Before 1.7.0 a miss
 	// filtered the location pool to empty and silently deleted every geo property.
@@ -1206,6 +1224,9 @@ export function validateDungeonConfig(config) {
 		groupKeys: normalizedGroupKeys,
 		groupProps,
 		lookupTables,
+		// v1.8.0 identity-less metric snapshots (normalized)
+		standaloneEvents,
+		warehouseMetrics,
 		hasAnonIds: hasAnonIdsResolved,
 		avgDevicePerUser,
 		hasSessionIds,

package/lib/core/dungeon-loader.js

@@ -278,7 +278,7 @@ export function validateDungeonShape(input) {
 		'events', 'numEvents', 'numUsers', 'numDays', 'funnels',
 		'userProps', 'superProps', 'hook', 'token', 'seed',
 		'scdProps', 'groupKeys', 'lookupTables', 'mirrorProps',
-		'hasAdSpend', 'soup', 'format', 'writeToDisk'
+		'hasAdSpend', 'standaloneEvents', 'warehouseMetrics', 'soup', 'format', 'writeToDisk'
 	];
 
 	const hasAnyDungeonKey = dungeonKeys.some(key => key in config);

package/lib/core/storage.js

@@ -91,15 +91,17 @@ export async function createHookArray(arr = [], opts) {
 		if (item === null || item === undefined) return false;
 		if (typeof item === 'object' && Object.keys(item).length === 0) return false;
 
+		const isWarehouse = type === "warehouse";
+
 		// Skip hook for types already hooked in generators/orchestrators to prevent double-firing
 		// Types hooked upstream: "event" (events.js), "user" (user-loop.js), "scd" (user-loop.js)
-		// Types only hooked here: "mirror", "ad-spend", "group", "lookup"
+		// Types only hooked here: "mirror", "ad-spend", "group", "lookup", "standalone"
 		const alreadyHooked = type === "event" || type === "user" || type === "scd";
 
 		// Performance optimization: skip hook overhead for passthrough hooks
 		// Only treat as passthrough if the function body is trivially simple (just returns its argument)
 		const hookStr = hook.toString();
-		const isPassthroughHook = hook.length === 1 || /^\s*function\s*\([^)]*\)\s*\{\s*return\s+\w+;?\s*\}\s*$/.test(hookStr) || /^\s*\(?[^)]*\)?\s*=>\s*\w+\s*$/.test(hookStr);
+		const isPassthroughHook = !isWarehouse && (hook.length === 1 || /^\s*function\s*\([^)]*\)\s*\{\s*return\s+\w+;?\s*\}\s*$/.test(hookStr) || /^\s*\(?[^)]*\)?\s*=>\s*\w+\s*$/.test(hookStr));
 
 		if (alreadyHooked || isPassthroughHook) {
 			// Fast path for passthrough hooks - no transformation needed
@@ -131,6 +133,10 @@ export async function createHookArray(arr = [], opts) {
 				for (const i of item) {
 					try {
 						const enriched = await hook(i, type, allMetaData);
+						if (isWarehouse) {
+							if (isValidEvent(i)) arr.push(i);
+							continue;
+						}
 						if (Array.isArray(enriched)) {
 							enriched.forEach(e => {
 								if (isValidEvent(e)) arr.push(e);
@@ -146,6 +152,10 @@ export async function createHookArray(arr = [], opts) {
 			} else {
 				try {
 					const enriched = await hook(item, type, allMetaData);
+					if (isWarehouse) {
+						if (isValidEvent(item)) arr.push(item);
+						return Promise.resolve(false);
+					}
 					if (Array.isArray(enriched)) {
 						enriched.forEach(e => {
 							if (isValidEvent(e)) arr.push(e);
@@ -161,7 +171,7 @@ export async function createHookArray(arr = [], opts) {
 		}
 
 		// Check batch size and handle writes synchronously to prevent race conditions
-		if (arr.length > BATCH_SIZE && !isWriting) {
+		if (!isWarehouse && arr.length > BATCH_SIZE && !isWriting) {
 			isWriting = true; // Lock to prevent concurrent writes
 			isBatchMode = true;
 			runtime.isBatchMode = true; // Update runtime state
@@ -200,6 +210,9 @@ export async function createHookArray(arr = [], opts) {
 		const streamOptions = {
 			gzip: config.gzip || false
 		};
+		if (type === "warehouse" && Array.isArray(rest.fixedColumns)) {
+			streamOptions.fixedColumns = rest.fixedColumns;
+		}
 
 		switch (format) {
 			case "csv":
@@ -253,6 +266,8 @@ export async function createHookArray(arr = [], opts) {
 	enrichedArray.getWriteDir = getWriteDir;
 	enrichedArray.getWritePath = getWritePath;
 	enrichedArray.getWrittenFiles = () => [...writtenFiles];
+	enrichedArray.type = type;
+	enrichedArray.format = format;
 
 	// Add additional properties from rest
 	for (const key in rest) {
@@ -309,9 +324,20 @@ export class StorageManager {
 				context: this.context
 			}),
 
+			// v1.8.0 — identity-less metric snapshots (`standaloneEvents`).
+			standaloneEventData: await createHookArray([], {
+				hook: config.hook,
+				type: "standalone",
+				filepath: `${config.name}-STANDALONE`,
+				format: config.format || "csv",
+				concurrency: config.concurrency || 1,
+				context: this.context
+			}),
+
 			scdTableData: [],
 			groupProfilesData: [],
 			lookupTableData: [],
+			warehouseMetricData: [],
 
 			mirrorEventData: await createHookArray([], {
 				hook: config.hook,
@@ -375,6 +401,28 @@ export class StorageManager {
 			}
 		}
 
+		if (config.warehouseMetrics && config.warehouseMetrics.length > 0) {
+			for (const warehouseMetric of config.warehouseMetrics) {
+				const fixedColumns = [
+					warehouseMetric.timeColumn,
+					...(warehouseMetric.source.groupBy || []),
+					warehouseMetric.valueColumn,
+					...Object.keys(warehouseMetric.columns || {}),
+				];
+				const warehouseArray = await createHookArray([], {
+					hook: config.hook,
+					type: "warehouse",
+					filepath: `${config.name}-WAREHOUSE-${warehouseMetric.name}`,
+					format: warehouseMetric.format || config.format || "csv",
+					concurrency: config.concurrency || 1,
+					context: this.context,
+					metricName: warehouseMetric.name,
+					fixedColumns,
+				});
+				storage.warehouseMetricData.push(warehouseArray);
+			}
+		}
+
 		return storage;
 	}
 

package/lib/generators/events.js

@@ -17,6 +17,8 @@ import { dataLogger as logger } from "../utils/logger.js";
 // Keys that must never be nulled by data quality gremlins
 const NULL_EXEMPT_KEYS = new Set(['event', 'time', 'insert_id', 'user_id', 'device_id', 'distinct_id', '_drop', '_anomaly', '_persona']);
 
+export const engineIdentity = Symbol('engineIdentity');
+
 
 /**
  * Creates a Mixpanel event with a flat shape
@@ -293,6 +295,9 @@ export async function makeEvent(
 
 	eventTemplate.insert_id = randomUUID();
 
+    const originalIdentity = { user_id: eventTemplate.user_id, device_id: eventTemplate.device_id };
+    Object.defineProperty(eventTemplate, engineIdentity, { value: originalIdentity, configurable: true });
+
     // Call hook if configured (hooks override everything — they are the final authority)
     const { hook } = config;
     if (hook) {
@@ -305,6 +310,7 @@ export async function makeEvent(
         });
         // If hook returns a modified event, use it; otherwise use original
         if (hookedEvent && typeof hookedEvent === 'object') {
+            Object.defineProperty(hookedEvent, engineIdentity, { value: originalIdentity, configurable: true });
             return hookedEvent;
         }
     }

package/lib/generators/funnels.js

@@ -331,6 +331,21 @@ export async function makeFunnel(context, funnel, user, firstEventTime, profile
 		dataQuality: featureCtx.dataQuality || context.config.dataQuality || null,
 		latestTime: Number.isFinite(featureCtx.latestTime) ? featureCtx.latestTime : undefined,
 	};
+	if (attemptInfo.isFirstFunnel && attemptInfo.isBorn && funnelEventsWithTiming.length) {
+		const spanMs = Math.max(0, ...funnelEventsWithTiming.map(event => event.relativeTimeMs || 0));
+		const latestStart = Math.min(funnelFeatureCtx.latestTime ?? context.FIXED_NOW, context.FIXED_NOW - spanMs / 1000);
+		if (firstEventTime > latestStart) {
+			context.addWarning({
+				key: 'lifecycle.firstFunnelClipped',
+				requested: spanMs,
+				applied: Math.max(0, (context.FIXED_NOW - firstEventTime) * 1000),
+				reason: 'First-funnel timing exceeds the remaining dataset window; emit the in-window prefix and omit future steps without compressing configured TTC.',
+				severity: 'warn',
+			});
+		}
+		funnelFeatureCtx.latestTime = Math.max(firstEventTime, latestStart);
+		if (featureCtx.pinFirstTime) funnelFeatureCtx.fixedTimeMs = firstEventTime * 1000;
+	}
 
 	// Pre-compute per-step stamping modes for execution order. For isFirstFunnel + isBorn
 	// runs, the first event in execution order whose config has `isAuthEvent: true` is

package/lib/generators/standalone.js

@@ -0,0 +1,248 @@
+/**
+ * Standalone event generator module (v1.8.0)
+ *
+ * Standalone events are IDENTITY-LESS metric snapshots. They carry no `user_id`
+ * and no `device_id` — they describe a system, not a person. Think daily CDN
+ * egress per region, weekly billing rollups per plan tier, hourly queue depth
+ * per cluster. `$ad_spend` is the same idea, hard-coded; this is the general form.
+ *
+ * One record is emitted per cadence tick per dimension cross-product row:
+ *
+ *   cadence: 'day', dimensions: { region: ['us','eu'], tier: ['a','b'] }
+ *   → 4 records per day (us/a, us/b, eu/a, eu/b)
+ *
+ * `distinct_id` is synthetic. It is the value of the dimension named by
+ * `distinctIdFrom`, or the event name when no dimension is named. It exists only
+ * so Mixpanel accepts the record; it never maps to a person.
+ */
+
+/** @typedef {import('../../types').Context} Context */
+/** @typedef {import('../../types').StandaloneEventConfig} StandaloneEventConfig */
+/** @typedef {import('../../types').ResolvedStandaloneEventConfig} ResolvedStandaloneEventConfig */
+
+import { randomUUID } from "node:crypto";
+import dayjs from "dayjs";
+import utc from "dayjs/plugin/utc.js";
+import * as u from "../utils/utils.js";
+
+dayjs.extend(utc);
+
+/** Seconds per cadence tick. */
+const CADENCE_SECONDS = {
+	hour: 60 * 60,
+	day: 24 * 60 * 60,
+	week: 7 * 24 * 60 * 60,
+};
+
+/** Cadence names the validator accepts. */
+export const VALID_CADENCES = Object.freeze(Object.keys(CADENCE_SECONDS));
+
+/**
+ * Expands a `dimensions` map into every combination of its values.
+ * `{ a: [1,2], b: ['x'] }` → `[{a:1,b:'x'}, {a:2,b:'x'}]`.
+ * An empty/absent map yields a single empty row, so an undimensioned
+ * standalone event emits exactly one record per tick.
+ *
+ * @param {Record<string, any[]>} dimensions
+ * @returns {Record<string, any>[]}
+ */
+export function expandDimensions(dimensions) {
+	const keys = Object.keys(dimensions || {});
+	if (keys.length === 0) return [{}];
+
+	let rows = [{}];
+	for (const key of keys) {
+		const values = dimensions[key];
+		/** @type {Record<string, any>[]} */
+		const next = [];
+		for (const row of rows) {
+			for (const value of values) {
+				next.push({ ...row, [key]: value });
+			}
+		}
+		rows = next;
+	}
+	return rows;
+}
+
+/**
+ * Every tick timestamp for one standalone event across the dataset window.
+ *
+ * Ticks start at `datasetStart` and step by the cadence. The final tick is the
+ * last one that lands at or before `datasetEnd` — a standalone event never
+ * emits a record in the future, matching the engine-wide future-time guard.
+ *
+ * @param {number} datasetStart - unix seconds
+ * @param {number} datasetEnd - unix seconds
+ * @param {'hour'|'day'|'week'} cadence
+ * @returns {number[]} unix seconds, ascending
+ */
+export function buildTicks(datasetStart, datasetEnd, cadence) {
+	const step = CADENCE_SECONDS[cadence];
+	if (!step) throw new Error(`unknown cadence: ${cadence}`);
+
+	/** @type {number[]} */
+	const ticks = [];
+	for (let t = datasetStart; t <= datasetEnd; t += step) {
+		ticks.push(t);
+	}
+	return ticks;
+}
+
+/**
+ * Builds every record for a single standalone event config.
+ *
+ * @param {Context} context
+ * @param {ResolvedStandaloneEventConfig} spec
+ * @returns {Record<string, any>[]}
+ */
+export function makeStandaloneEvents(context, spec) {
+	const { config } = context;
+	const datasetStart = context.FIXED_BEGIN;
+	const datasetEnd = context.FIXED_NOW;
+
+	const ticks = buildTicks(datasetStart, datasetEnd, spec.cadence);
+	const rows = expandDimensions(spec.dimensions);
+	const propKeys = Object.keys(spec.properties);
+
+	/** @type {Record<string, any>[]} */
+	const records = [];
+
+	for (let tickIndex = 0; tickIndex < ticks.length; tickIndex++) {
+		const tickUnix = ticks[tickIndex];
+		const isoTime = dayjs.unix(tickUnix).utc().toISOString();
+
+		for (const dimensions of rows) {
+			context.incrementOperations();
+
+			// distinct_id is synthetic: the named dimension's value, else the
+			// event name. Never a person. Stable across the run so the record
+			// series groups cleanly in Mixpanel.
+			const distinctId = spec.distinctIdFrom
+				? String(dimensions[spec.distinctIdFrom])
+				: spec.event;
+
+			/** @type {Record<string, any>} */
+			const record = {
+				event: spec.event,
+				time: isoTime,
+				insert_id: randomUUID(),
+				distinct_id: distinctId,
+				...dimensions,
+			};
+
+			// The context handed to every property function. Shares the
+			// ValueContext members (`time`, `config`) so a function written for
+			// a normal event property still works here, plus the standalone-only
+			// members a snapshot needs to shape a trend.
+			const valueContext = {
+				time: tickUnix * 1000,
+				config,
+				dimensions,
+				tickIndex,
+				tickCount: ticks.length,
+				cadence: spec.cadence,
+				event: record,
+			};
+
+			for (const key of propKeys) {
+				record[key] = u.choose(spec.properties[key], valueContext);
+			}
+
+			records.push(record);
+		}
+	}
+
+	return records;
+}
+
+/**
+ * Validates and normalizes the `standaloneEvents` config array.
+ * Throws on anything malformed — a silent skip would hide a whole data stream.
+ *
+ * @param {unknown} standaloneEvents
+ * @returns {ResolvedStandaloneEventConfig[]}
+ */
+export function validateStandaloneEvents(standaloneEvents) {
+	if (standaloneEvents === undefined || standaloneEvents === null) return [];
+	if (!Array.isArray(standaloneEvents)) {
+		throw new Error("standaloneEvents must be an array");
+	}
+
+	const seen = new Set();
+
+	return standaloneEvents.map((spec, index) => {
+		const label = `standaloneEvents[${index}]`;
+
+		if (!spec || typeof spec !== "object" || Array.isArray(spec)) {
+			throw new Error(`${label} must be an object`);
+		}
+		if (typeof spec.event !== "string" || !spec.event.trim()) {
+			throw new Error(`${label}.event must be a non-empty string`);
+		}
+		if (seen.has(spec.event)) {
+			throw new Error(`${label}.event "${spec.event}" is declared more than once`);
+		}
+		seen.add(spec.event);
+
+		const cadence = spec.cadence || "day";
+		if (!CADENCE_SECONDS[cadence]) {
+			throw new Error(
+				`${label}.cadence must be one of ${VALID_CADENCES.join(", ")} (got "${cadence}")`
+			);
+		}
+
+		/** @type {Record<string, any[]>} */
+		const dimensions = {};
+		if (spec.dimensions !== undefined && spec.dimensions !== null) {
+			if (typeof spec.dimensions !== "object" || Array.isArray(spec.dimensions)) {
+				throw new Error(`${label}.dimensions must be an object of arrays`);
+			}
+			for (const [key, values] of Object.entries(spec.dimensions)) {
+				if (!Array.isArray(values) || values.length === 0) {
+					throw new Error(`${label}.dimensions.${key} must be a non-empty array`);
+				}
+				dimensions[key] = values;
+			}
+		}
+
+		if (spec.distinctIdFrom !== undefined && spec.distinctIdFrom !== null) {
+			if (typeof spec.distinctIdFrom !== "string") {
+				throw new Error(`${label}.distinctIdFrom must be a string`);
+			}
+			if (!(spec.distinctIdFrom in dimensions)) {
+				throw new Error(
+					`${label}.distinctIdFrom "${spec.distinctIdFrom}" is not a declared dimension`
+				);
+			}
+		}
+
+		if (spec.properties !== undefined && spec.properties !== null) {
+			if (typeof spec.properties !== "object" || Array.isArray(spec.properties)) {
+				throw new Error(`${label}.properties must be an object`);
+			}
+		}
+
+		// Schema-first: a property key must not collide with a dimension key or
+		// with a reserved record key. A collision would silently overwrite one
+		// of them and the author would never see it.
+		const properties = spec.properties || {};
+		const reserved = new Set(["event", "time", "insert_id", "distinct_id", "user_id", "device_id"]);
+		for (const key of Object.keys(properties)) {
+			if (reserved.has(key)) {
+				throw new Error(`${label}.properties.${key} collides with a reserved record key`);
+			}
+			if (key in dimensions) {
+				throw new Error(`${label}.properties.${key} collides with a dimension of the same name`);
+			}
+		}
+
+		return {
+			event: spec.event,
+			cadence,
+			dimensions,
+			distinctIdFrom: spec.distinctIdFrom || null,
+			properties,
+		};
+	});
+}