@ak--47/dungeon-master 1.7.0 → 1.8.1

package/lib/verify/funnel-engine.js

@@ -90,8 +90,10 @@
  *   trend-interval end and the data-pull range — fix-round C6).
  * @property {boolean} [graceperiod=true] - Enable the 2-second grace window
  *   on ordering checks. Disable only for tests that need strict ordering.
- * @property {boolean} [reentry=false] - When true, after completing all steps,
- *   reset to step 0 and continue scanning. Increments `completions`.
+ * @property {boolean} [reentry=false] - When true, restart after the inclusive
+ *   2-second completion grace. An event recording the ordered last step and
+ *   matching the ordered first step also anchors the next attempt immediately.
+ *   `graceperiod: false` disables the completion wait. Increments `completions`.
  *   Attempts ALSO restart when the conversion window expires (fix-round
  *   B2+C5): an incoming event past the window from step 0 finalizes the
  *   live attempt as a drop-off and processes against a fresh one — ARB
@@ -124,8 +126,8 @@
  *   fresh one processes the same event (funnel_query.cpp:1608-1613 — for
  *   GENERAL_WO_REPEAT termination checks ONLY history_is_past_conversion_
  *   window, not history_is_mutable; re-birth :1663-1680). Contrast
- *   `reentry: true` (GENERAL), which also restarts right after each
- *   completion/exclusion, permitting repeat conversions within one window.
+ *   `reentry: true` (GENERAL), which also restarts after completion grace or
+ *   exclusion, permitting repeat conversions within one window.
  *   Requires `countMode: 'totals'`; mutually exclusive with `reentry` and
  *   `sessionScoped`.
  * @property {boolean} [sessionScoped=false] - **@deprecated — verifier-only, NOT
@@ -373,7 +375,7 @@ function emptyResult(trackStepProperties) {
  * @returns {{ result: FunnelResult, nextIdx: number, terminatedByExclusion: boolean, expiredByWindow: boolean }}
  */
 function runOneAttempt(sorted, startIdx, steps, exclusionSteps, options) {
-	const { windowCheck, graceperiod, trackStepProperties, anchorOk, isAnyOrder, prevAnchor, nextAnchor, expireOnWindow = false, woRepeat = false } = options;
+	const { windowCheck, graceperiod, trackStepProperties, anchorOk, isAnyOrder, prevAnchor, nextAnchor, expireOnWindow = false, woRepeat = false, restartSharedEdge = false } = options;
 	const numSteps = steps.length;
 	// Per-SLOT recorded candidates (history->steps): the latest match for
 	// anchors, the first eligible match for active any-order chunk members.
@@ -402,8 +404,9 @@ function runOneAttempt(sorted, startIdx, steps, exclusionSteps, options) {
 	let reached = -1;
 	let terminatedByExclusion = false;
 	let expiredByWindow = false;
+	let sharedEdgeCompleted = false;
 	let tailAnchorMs = 0; // terminating exclusion time, or last-step time on completion
-	let endIdx = -1;      // consumption boundary — frozen the moment the attempt is decided
+	let endIdx = -1;      // first event eligible for the next attempt
 	let i = startIdx;
 
 	// Does exclusion `ex` apply to gap g? afterStep/beforeStep bound the
@@ -586,8 +589,10 @@ function runOneAttempt(sorted, startIdx, steps, exclusionSteps, options) {
 
 		if (reached === numSteps - 1 && !terminatedByExclusion) {
 			tailAnchorMs = timeAtPos(numSteps - 1);
-			endIdx = i + 1;
-			// No break — the 2s tail may still kill the completion.
+			sharedEdgeCompleted = !woRepeat && numSteps > 1 && s === numSteps - 1
+				&& !isAnyOrder[0] && !isAnyOrder[numSteps - 1] && eventMatchesStep(ev, steps[0]);
+			endIdx = sharedEdgeCompleted && restartSharedEdge ? i : i + 1;
+			// Ordinary completions remain open for the 2s exclusion tail.
 		}
 		return true;
 	};
@@ -606,10 +611,10 @@ function runOneAttempt(sorted, startIdx, steps, exclusionSteps, options) {
 		// nextIdx = i, not i + 1. For GENERAL_WO_REPEAT (woRepeat) expiry is
 		// the ONLY termination — it fires even on completed/excluded attempts
 		// (:1611-1613). For GENERAL (the reentry loop) completion/exclusion
-		// keep their own restart below, so expiry only finalizes LIVE
-		// attempts (C5: failed-attempt windows).
+		// keep their own restart below; independent window expiry also
+		// finalizes a completion still inside its grace period.
 		if (expireOnWindow && reached >= 0
-			&& (woRepeat || (!terminatedByExclusion && reached !== numSteps - 1))
+			&& (woRepeat || !terminatedByExclusion)
 			&& !windowCheck(t, timeAtPos(0), ev, eventAtPos(0))
 		) {
 			expiredByWindow = true;
@@ -654,9 +659,15 @@ function runOneAttempt(sorted, startIdx, steps, exclusionSteps, options) {
 			// the event on their own: anchors need reached < slot; claimed
 			// any-order chunk members are first-match-sealed — so
 			// exclusions-only here matches ARB.)
-			if (!hasExclusions || !graceperiod || t > tailAnchorMs + OUT_OF_ORDER_MS) break;
-			for (let g = 0; g < numSteps - 1; g++) {
-				if (tryExclusionAtGap(g, ev, t)) break;
+			if (!graceperiod || t > tailAnchorMs + OUT_OF_ORDER_MS) {
+				endIdx = i;
+				break;
+			}
+			endIdx = i + 1;
+			if (hasExclusions) {
+				for (let g = 0; g < numSteps - 1; g++) {
+					if (tryExclusionAtGap(g, ev, t)) break;
+				}
 			}
 			continue;
 		}
@@ -693,6 +704,7 @@ function runOneAttempt(sorted, startIdx, steps, exclusionSteps, options) {
 				break;
 			}
 		}
+		if (sharedEdgeCompleted) break;
 	}
 
 	// Result surfaces the PATH (positions), not the slot table: position p was
@@ -772,6 +784,17 @@ function runOneAttempt(sorted, startIdx, steps, exclusionSteps, options) {
  * @returns {FunnelResult | FunnelResult[]}
  */
 export function evaluateFunnel(events, steps, options = {}) {
+	return evaluateFunnelWithContext(events, steps, options);
+}
+
+/**
+ * @param {Array<Object>} events
+ * @param {FunnelStep[]} steps
+ * @param {FunnelOptions} options
+ * @param {Map<Object, number>} [fullStreamOrdinals]
+ * @returns {FunnelResult | FunnelResult[]}
+ */
+function evaluateFunnelWithContext(events, steps, options, fullStreamOrdinals) {
 	if (options.countMode === 'sessions') {
 		// Mixpanel's funnel "count by Sessions" is an API rewrite, not an
 		// engine mode: count_type session REQUIRES window (session, 1) and
@@ -790,14 +813,14 @@ export function evaluateFunnel(events, steps, options = {}) {
 		if (typeof options.conversionWindowMs === 'number' || (cw != null && !(cw.unit === 'sessions' && cw.n === 1))) {
 			throw new Error("evaluateFunnel: cannot use countMode 'sessions' without conversion window = 1 session");
 		}
-		return evaluateFunnel(events, steps, {
+		return evaluateFunnelWithContext(events, steps, {
 			...options,
 			countMode: 'totals',
 			reentry: false,
 			woRepeat: true,
 			conversionWindowMs: undefined,
 			conversionWindow: { unit: 'sessions', n: 1 },
-		});
+		}, fullStreamOrdinals);
 	}
 	if (!Array.isArray(steps) || steps.length === 0) {
 		const empty = emptyResult(options.trackStepProperties);
@@ -871,7 +894,7 @@ export function evaluateFunnel(events, steps, options = {}) {
 		// the data-pull range (:402, :1408-1412) — the timeBucket wrapper's
 		// [start, stop + n×day) spill slice mirrors exactly that. Spec P1.6.1's
 		// dual per-step condition was a misreading; dropped per fix-round C6.
-		const ordinals = sessionOrdinals(sorted);
+		const ordinals = fullStreamOrdinals ?? sessionOrdinals(sorted);
 		windowCheck = (t, t0, ev, step0Ev) => {
 			const o = ordinals.get(ev);
 			const o0 = ordinals.get(step0Ev);
@@ -891,7 +914,7 @@ export function evaluateFunnel(events, steps, options = {}) {
 		}
 		const allResults = [];
 		for (const [sid, evs] of bySession) {
-			const sub = evaluateFunnel(evs, steps, { ...options, sessionScoped: false });
+			const sub = evaluateFunnelWithContext(evs, steps, { ...options, sessionScoped: false }, fullStreamOrdinals);
 			if (Array.isArray(sub)) {
 				for (const r of sub) { r.sessionId = sid; allResults.push(r); }
 			} else {
@@ -952,7 +975,7 @@ export function evaluateFunnel(events, steps, options = {}) {
 	const completedAttempts = [];
 	let idx = 0;
 	let lastResult = emptyResult(trackStepProperties);
-	const reentryOpts = { ...opts, expireOnWindow: true };
+	const reentryOpts = { ...opts, expireOnWindow: true, restartSharedEdge: true };
 	while (idx < sorted.length) {
 		const { result, nextIdx, terminatedByExclusion } = runOneAttempt(sorted, idx, normSteps, exclusionSteps, reentryOpts);
 		// Always advance — runOneAttempt returns nextIdx > idx when it processed an event.
@@ -990,7 +1013,8 @@ export function evaluateFunnel(events, steps, options = {}) {
  * `Map<propertyValue, FunnelResult>`.
  *
  * Each sub-funnel runs independently (a user CAN convert in one HPC value
- * group and drop off in another simultaneously).
+ * group and drop off in another simultaneously). Session windows derive
+ * ordinals from the full user stream before routing events to HPC buckets.
  *
  * Reference: `funnel_query.cpp` lines 749-784 (`aggregate_hash_get_key_cursor`).
  *
@@ -1017,6 +1041,9 @@ export function evaluateFunnelHPC(events, steps, holdProperty, options = {}) {
 		throw new Error('evaluateFunnelHPC: an anyOrder block cannot be the first funnel step');
 	}
 	const step0Name = flat[0].event;
+	const fullStreamOrdinals = options.countMode === 'sessions' || options.conversionWindow?.unit === 'sessions'
+		? sessionOrdinals((events || []).filter(ev => ev && typeof ev.event === 'string'))
+		: undefined;
 
 	// Bucket events by HPC value. The step-0 events define the universe of
 	// HPC values for this user; later events only populate buckets whose
@@ -1042,15 +1069,15 @@ export function evaluateFunnelHPC(events, steps, holdProperty, options = {}) {
 
 	const out = new Map();
 	for (const [v, evs] of valueBuckets) {
-		out.set(v, evaluateFunnel(evs, steps, options));
+		out.set(v, evaluateFunnelWithContext(evs, steps, options, fullStreamOrdinals));
 	}
 	return out;
 }
 
 /**
- * Resolve the property snapshot for a given segment mode against a result's
- * `stepProperties`. Use to mimic Mixpanel's FIRST_TOUCH / LAST_TOUCH / STEP
- * funnel segment modes.
+ * Merge reached `stepProperties` in recorded path order for FIRST_TOUCH or
+ * LAST_TOUCH. Undefined never replaces a defined value; null never replaces
+ * a non-null defined value. STEP returns its selected snapshot unchanged.
  *
  * @param {FunnelResult} result
  * @param {'first'|'last' | { step: number }} mode
@@ -1058,8 +1085,21 @@ export function evaluateFunnelHPC(events, steps, holdProperty, options = {}) {
  */
 export function resolveFunnelSegment(result, mode) {
 	if (!result || !Array.isArray(result.stepProperties) || !result.stepProperties.length) return undefined;
-	if (mode === 'first') return result.stepProperties[0];
-	if (mode === 'last')  return result.stepProperties[result.reached >= 0 ? result.reached : result.stepProperties.length - 1];
+	if (mode === 'first' || mode === 'last') {
+		const merged = {};
+		const snapshots = result.stepProperties.slice(0, result.reached + 1);
+		if (mode === 'first') snapshots.reverse();
+		for (const snapshot of snapshots) {
+			for (const [key, value] of Object.entries(snapshot || {})) {
+				const current = Object.prototype.hasOwnProperty.call(merged, key) ? merged[key] : undefined;
+				if (current === undefined || (current === null && value !== undefined)
+					|| (value !== undefined && value !== null)) {
+					Object.defineProperty(merged, key, { value, enumerable: true, configurable: true, writable: true });
+				}
+			}
+		}
+		return merged;
+	}
 	if (mode && typeof mode === 'object' && typeof mode.step === 'number') {
 		return result.stepProperties[mode.step];
 	}

package/lib/verify/index.js

@@ -17,6 +17,7 @@ export { verifyDungeon } from './verify-dungeon.js';
 // `evaluateStories` / `applyFunnelDefaults`. Use the RETURN value; it does not
 // enrich the config you pass in.
 export { validateDungeonConfig } from '../core/config-validator.js';
+export { pearson, computeWarehouseStats, computeWarehouseSourceRows, auditWarehouseRows } from './warehouse.js';
 export { deriveExpectedSchema, validateSchema } from './schema-validator.js';
 export {
 	evaluateFunnel,

package/lib/verify/story-runner.js

@@ -27,6 +27,7 @@
 
 import { emulateBreakdown } from './emulate-breakdown.js';
 import { applyFunnelDefaults } from './verify-dungeon.js';
+import { computeWarehouseSourceRows, computeWarehouseStats } from './warehouse.js';
 
 /** Closed archetype enum — MUST match lib/templates/story-spec.schema.json (unit-tested). */
 export const STORY_ARCHETYPES = [
@@ -322,6 +323,9 @@ export function validateStories(stories) {
 				err(ap, 'breakdown: required object with a string `type`');
 			} else if (a.breakdown.type === 'duckdb' && (typeof a.breakdown.sql !== 'string' || !a.breakdown.sql)) {
 				err(ap, 'breakdown: type "duckdb" requires a non-empty `sql`');
+			} else if ((a.breakdown.type === 'warehouse' || a.breakdown.type === 'warehouse-stats')
+				&& (typeof a.breakdown.table !== 'string' || !a.breakdown.table.trim())) {
+				err(ap, `breakdown: type "${a.breakdown.type}" requires a non-empty \`table\``);
 			}
 			if (a.expect === undefined && typeof a.assert !== 'function') {
 				err(ap, 'requires `expect` or a function-valued `assert`');
@@ -339,8 +343,8 @@ export function validateStories(stories) {
 					for (const [name, spec] of Object.entries(a.select)) {
 						if (!NAME_RE.test(name)) err(ap, `select: name "${name}" must be an identifier`);
 						selectNames.add(name);
-						if (!spec || typeof spec !== 'object' || !spec.where || typeof spec.where !== 'object' || !Object.keys(spec.where).length) {
-							err(ap, `select.${name}: requires a non-empty \`where\``);
+						if (!spec || typeof spec !== 'object' || !spec.where || typeof spec.where !== 'object') {
+							err(ap, `select.${name}: requires a \`where\` object`);
 							continue;
 						}
 						for (const [col, cond] of Object.entries(spec.where)) {
@@ -406,8 +410,8 @@ export function storiesToChecks(stories) {
 	for (const story of stories) {
 		story.assertions.forEach((assertion, i) => {
 			const name = `${story.id}[${i}]`;
-			if (assertion.breakdown.type === 'duckdb') {
-				console.warn(`[dungeon-master] ${name}: duckdb assertions only run in disk mode (scripts/verify-stories.mjs) — skipped in-memory.`);
+			if (assertion.breakdown.type === 'duckdb' || assertion.breakdown.type === 'warehouse' || assertion.breakdown.type === 'warehouse-stats') {
+				console.warn(`[dungeon-master] ${name}: ${assertion.breakdown.type} assertions only run in disk mode via scripts/verify-stories.mjs — skipped in verifyDungeon in-memory checks.`);
 				return;
 			}
 			checks.push({
@@ -441,11 +445,34 @@ export function storiesToChecks(stories) {
  *   auto-build over large profile sets is wasteful.
  * @param {(sql: string) => Promise<Array<Object>>} [opts.runSql] - duckdb
  *   executor (provided by the CLI in disk mode). Absent → duckdb assertions
- *   report NONE with an explanatory detail.
+ *   report NONE unless skipDiskOnlyDuckdb is set.
+ * @param {Record<string, Array<Object>>} [opts.warehouseRows] - Loaded
+ *   warehouse rows keyed by table name.
+ * @param {Record<string, Object>} [opts.warehouseSpecs] - Resolved warehouse
+ *   metric specs keyed by table name.
+ * @param {string|number} [opts.datasetStart] - Dataset window start for
+ *   in-window warehouse stats comparisons.
+ * @param {string|number} [opts.datasetEnd] - Dataset window end for in-window
+ *   warehouse stats comparisons.
+ * @param {boolean} [opts.skipDiskOnlyDuckdb] - Mark duckdb assertions skipped
+ *   instead of NONE when no SQL executor is available.
+ * @param {(message: string) => void} [opts.onWarning] - Warning sink for
+ *   skipped disk-only assertions.
  * @returns {Promise<Array<{ id: string, hook: string, archetype: string, verdict: string, assertions: Array<{ name: string, verdict: string, observed: number|null, detail: string }> }>>}
  */
 export async function evaluateStories(stories, events, opts) {
-	const { profiles, funnels, runSql, identityMap } = opts || {};
+	const {
+		profiles,
+		funnels,
+		runSql,
+		identityMap,
+		warehouseRows,
+		warehouseSpecs,
+		datasetStart,
+		datasetEnd,
+		skipDiskOnlyDuckdb,
+		onWarning,
+	} = opts || {};
 	const v = validateStories(stories);
 	if (!v.valid) {
 		throw new Error(`evaluateStories: invalid stories:\n  ${v.errors.join('\n  ')}`);
@@ -460,10 +487,21 @@ export async function evaluateStories(stories, events, opts) {
 			try {
 				if (assertion.breakdown.type === 'duckdb') {
 					if (!runSql) {
-						results.push({ name, verdict: 'NONE', observed: null, detail: 'duckdb assertion requires disk mode (no SQL executor available)' });
+						if (skipDiskOnlyDuckdb) {
+							onWarning?.(`[dungeon-master] ${name}: duckdb assertions only run in disk mode (scripts/verify-stories.mjs) — skipped in-memory.`);
+							results.push({ name, verdict: 'SKIPPED', observed: null, detail: 'duckdb assertion requires disk mode (skipped in-memory)' });
+						} else {
+							results.push({ name, verdict: 'NONE', observed: null, detail: 'duckdb assertion requires disk mode (no SQL executor available)' });
+						}
 						continue;
 					}
 					rows = await runSql(assertion.breakdown.sql);
+				} else if (assertion.breakdown.type === 'warehouse') {
+					rows = resolveWarehouseRows(assertion.breakdown.table, warehouseRows, warehouseSpecs);
+				} else if (assertion.breakdown.type === 'warehouse-stats') {
+					const spec = resolveWarehouseSpec(assertion.breakdown.table, warehouseSpecs);
+					const tableRows = resolveWarehouseRows(assertion.breakdown.table, warehouseRows, warehouseSpecs);
+					rows = [computeWarehouseStats(tableRows, spec, computeWarehouseSourceRows(events, spec), { datasetStart, datasetEnd })];
 				} else {
 					const bArgs = applyFunnelDefaults(assertion.breakdown, funnels, profiles);
 					if (identityMap && bArgs.identityMap === undefined) bArgs.identityMap = identityMap;
@@ -476,8 +514,33 @@ export async function evaluateStories(stories, events, opts) {
 			const res = evaluateAssertion(rows, assertion, { events, profiles });
 			results.push({ name, ...res });
 		}
-		const worst = results.reduce((w, r) => VERDICT_RANK[r.verdict] < VERDICT_RANK[w] ? r.verdict : w, 'NAILED');
+		const counted = results.filter((result) => result.verdict !== 'SKIPPED');
+		const worst = counted.length
+			? counted.reduce((w, r) => VERDICT_RANK[r.verdict] < VERDICT_RANK[w] ? r.verdict : w, 'NAILED')
+			: 'SKIPPED';
 		out.push({ id: story.id, hook: story.hook, archetype: story.archetype, verdict: worst, assertions: results });
 	}
 	return out;
 }
+
+function resolveWarehouseSpec(table, warehouseSpecs) {
+	const spec = warehouseSpecs?.[table];
+	if (!spec) throw new Error(`warehouse table "${table}" is not available`);
+	return spec;
+}
+
+function resolveWarehouseRows(table, warehouseRows, warehouseSpecs) {
+	const spec = resolveWarehouseSpec(table, warehouseSpecs);
+	const rows = warehouseRows?.[table];
+	if (!Array.isArray(rows)) throw new Error(`warehouse rows for table "${table}" are not available`);
+	return rows.map((row) => withWarehouseTimestamp(row, spec));
+}
+
+function withWarehouseTimestamp(row, spec) {
+	if (Number.isFinite(row?.__t)) return row;
+	const raw = row?.[spec.timeColumn];
+	const parsed = typeof raw === 'string'
+		? Date.parse(/T/.test(raw) ? raw : `${raw}T00:00:00Z`)
+		: NaN;
+	return { ...row, __t: Number.isFinite(parsed) ? parsed / 1000 : null };
+}