@ak--47/dungeon-master 1.4.5 → 1.5.1

package/lib/verify/counting.js

@@ -0,0 +1,360 @@
+/**
+ * Counting and aggregation helpers matching Mixpanel's analytics semantics.
+ *
+ * These primitives differ from naive SQL in important ways. Each helper
+ * documents the Mixpanel source file that defines the rule and the
+ * specific divergence from `COUNT(*)` / `AVG(x)` / etc.
+ *
+ * References (from `mixpanel/analytics`):
+ *   - `backend/arb/reader/queries/addiction_query.cpp` — distinct-period counting
+ *   - `backend/arb/reader/queries/normal_query.cpp` — null-aware AVG/SUM/MIN/MAX
+ */
+
+import { toMs } from '../hook-helpers/_internal.js';
+
+const SECONDS_PER_UNIT = {
+	hour: 3600,
+	day: 86400,
+	week: 7 * 86400,
+};
+
+/**
+ * Count distinct time periods on which a user fired a given event.
+ *
+ * Two related rules exist in Mixpanel:
+ *
+ * 1. **Calendar bucket** (default here, `algorithm: 'calendar'`):
+ *    `COUNT(DISTINCT date_trunc(unit, time))` in UTC. This is what the
+ *    Mixpanel UI presents — frequency distribution charts bucket events
+ *    into calendar hours/days/weeks.
+ *
+ * 2. **Rolling window** (`algorithm: 'rolling'`): the addiction_query.cpp
+ *    rule, `qtz_time >= last_counted + seconds_for_unit(unit)`. This is
+ *    Mixpanel's internal C++ implementation. It diverges from calendar
+ *    bucketing at unit boundaries — events at 23:59 and 00:01 next day
+ *    register as 1 rolling-window period (gap 120s < 86400s) but 2
+ *    calendar-day periods.
+ *
+ * The default is `calendar` because:
+ *   - It matches what users actually see in Mixpanel reports.
+ *   - It aligns with `injectOnNewDays`, which classifies days by
+ *     `Math.floor(t / DAY_MS)` to find empty days. Mixing the two
+ *     algorithms causes the atom and verifier to disagree at boundaries.
+ *
+ * Use `algorithm: 'rolling'` only when you're verifying behavior that
+ * specifically depends on the C++ rolling-window check.
+ *
+ * Reference: `mixpanel/analytics`
+ *   - calendar bucketing: implicit in the UI / Insights reports
+ *   - rolling-window: `backend/arb/reader/queries/addiction_query.cpp`
+ *
+ * @param {Object[]} events - Events to scan (mixed types OK).
+ * @param {string} eventName - Event name to filter for.
+ * @param {('hour'|'day'|'week')} [unit='day']
+ * @param {Object} [options]
+ * @param {('calendar'|'rolling')} [options.algorithm='calendar']
+ * @returns {number} Distinct period count.
+ */
+export function countDistinctPeriods(events, eventName, unit = 'day', options = {}) {
+	const seconds = SECONDS_PER_UNIT[unit];
+	if (!seconds) throw new Error(`countDistinctPeriods: unsupported unit "${unit}"`);
+	if (!Array.isArray(events) || !events.length) return 0;
+	const matches = events
+		.filter(e => e && e.event === eventName)
+		.map(e => toMs(e.time))
+		.filter(ms => Number.isFinite(ms));
+	if (!matches.length) return 0;
+	const unitMs = seconds * 1000;
+	const { algorithm = 'calendar' } = options;
+
+	if (algorithm === 'calendar') {
+		// Calendar bucket — UTC floor by unit. Matches what Mixpanel's UI
+		// shows and what `injectOnNewDays` uses internally.
+		const buckets = new Set();
+		for (const t of matches) buckets.add(Math.floor(t / unitMs));
+		return buckets.size;
+	}
+
+	// Rolling window — addiction_query.cpp semantics.
+	matches.sort((a, b) => a - b);
+	let count = 0;
+	let lastCountedMs = -Infinity;
+	for (const t of matches) {
+		if (t >= lastCountedMs + unitMs) {
+			count++;
+			lastCountedMs = t;
+		}
+	}
+	return count;
+}
+
+/**
+ * Null-aware average matching Mixpanel's aggregation semantics.
+ *
+ * Reference: `backend/arb/reader/queries/normal_query.cpp` ACTION_TYPE_AVERAGE:
+ *
+ *     if (action_value.type == VALUE_TYPE_NUMBER && !std::isnan(value)) {
+ *         v->average.sum += number;
+ *         v->average.count += upsampling_factor;
+ *     }
+ *
+ * Skips null, undefined, NaN, and non-numeric values from BOTH numerator
+ * and denominator. Returns null when no numeric values exist.
+ *
+ * Differs from naive `SUM(x) / COUNT(*)` which inflates the denominator
+ * by counting rows where x is missing — diluting the average toward 0.
+ *
+ * @param {*[]} values
+ * @returns {number|null}
+ */
+export function nullAwareAvg(values) {
+	if (!Array.isArray(values) || !values.length) return null;
+	let sum = 0;
+	let count = 0;
+	for (const v of values) {
+		if (typeof v === 'number' && Number.isFinite(v)) {
+			sum += v;
+			count++;
+		}
+	}
+	return count ? sum / count : null;
+}
+
+/**
+ * Null-aware sum. Skips null/undefined/NaN/non-numeric silently.
+ *
+ * Reference: `normal_query.cpp` ACTION_TYPE_SUM — same numeric guard as AVG.
+ * Differs from naive SQL SUM only when missing values are coalesced to 0
+ * upstream; in JS arrays missing values are typically `undefined` which
+ * produces NaN under `+`.
+ *
+ * @param {*[]} values
+ * @returns {number}
+ */
+export function nullAwareSum(values) {
+	if (!Array.isArray(values) || !values.length) return 0;
+	let sum = 0;
+	for (const v of values) {
+		if (typeof v === 'number' && Number.isFinite(v)) sum += v;
+	}
+	return sum;
+}
+
+/**
+ * Null-aware min/max. Returns null when no numeric values exist.
+ *
+ * Reference: `normal_query.cpp` ACTION_TYPE_EXTREMES — only records numeric
+ * values. Mixpanel starts max at -Infinity and min at +Infinity; we return
+ * null instead of those sentinels when no values were recorded.
+ *
+ * @param {*[]} values
+ * @param {('min'|'max')} mode
+ * @returns {number|null}
+ */
+export function nullAwareExtreme(values, mode) {
+	if (!Array.isArray(values) || !values.length) return null;
+	let extreme = mode === 'min' ? Infinity : -Infinity;
+	let any = false;
+	for (const v of values) {
+		if (typeof v === 'number' && Number.isFinite(v)) {
+			any = true;
+			if (mode === 'min') {
+				if (v < extreme) extreme = v;
+			} else {
+				if (v > extreme) extreme = v;
+			}
+		}
+	}
+	return any ? extreme : null;
+}
+
+/**
+ * v1.5.1: count distinct values of a flat property across events.
+ *
+ * Mixpanel parity: `COUNT_DISTINCT(prop)` aggregator (Insights). Skips
+ * null/undefined/empty-string values. Returns the distinct-value count plus
+ * the top-N most-frequent values (default 25, matching Mixpanel UI default).
+ *
+ * Property keys are FLAT on event records per the dungeon-master schema
+ * contract (see HOOKS.md §1) — no dot-path support.
+ *
+ * @param {Object[]} events
+ * @param {string} property - Flat property name.
+ * @param {Object} [options]
+ * @param {string} [options.event] - Optional event-name filter.
+ * @param {number} [options.topN=25] - Number of top values to include in the result.
+ * @returns {{ distinct_count: number, top_values: Array<{ value: any, count: number }> }}
+ */
+export function countDistinctValues(events, property, options = {}) {
+	if (!Array.isArray(events)) throw new Error('countDistinctValues: events must be an array');
+	if (typeof property !== 'string' || !property) throw new Error('countDistinctValues: property is required');
+	const topN = Number.isFinite(options.topN) && options.topN > 0 ? Math.floor(options.topN) : 25;
+	const filterEvent = typeof options.event === 'string' && options.event ? options.event : null;
+	const valueCounts = new Map();
+	for (const e of events) {
+		if (!e || typeof e !== 'object') continue;
+		if (filterEvent && e.event !== filterEvent) continue;
+		const v = e[property];
+		if (v === null || v === undefined || v === '') continue;
+		// Hashable normalization — Map keys distinguish primitives but objects
+		// use reference identity. For Mixpanel parity, stringify non-primitives.
+		const key = (typeof v === 'object') ? JSON.stringify(v) : v;
+		valueCounts.set(key, (valueCounts.get(key) || 0) + 1);
+	}
+	const sorted = [...valueCounts.entries()]
+		.sort((a, b) => b[1] - a[1])
+		.slice(0, topN)
+		.map(([value, count]) => ({ value, count }));
+	return { distinct_count: valueCounts.size, top_values: sorted };
+}
+
+/**
+ * Partition events into time buckets by UTC calendar (`day`, `week`, or
+ * `month`). Used by `emulateBreakdown` when `timeBucket` is set to slice any
+ * breakdown into a trend over time.
+ *
+ * Period labels:
+ *   - `'day'`   → `YYYY-MM-DD`
+ *   - `'week'`  → ISO week `YYYY-Www` (Monday-anchored)
+ *   - `'month'` → `YYYY-MM`
+ *
+ * Mixpanel parity: bucket boundaries are computed in UTC. Production Mixpanel
+ * uses query timezone (qtz); pass timestamps already shifted to qtz if you
+ * need that behavior. ISO week is Monday-anchored
+ * (matches `eval_node.c:3641-3643`).
+ *
+ * Empty-bucket backfill (Mixpanel `normal_query.cpp:352-356, 310-313` emits
+ * zero rows for empty intervals): when `options.from` AND `options.to` are
+ * provided, the result enumerates every bucket in `[from, to]` and emits
+ * `{ period, events: [] }` for buckets with no events. Without `from`/`to`,
+ * only buckets that contain at least one event are returned.
+ *
+ * @param {Object[]} events
+ * @param {('day'|'week'|'month')} bucket
+ * @param {Object} [options]
+ * @param {number|string} [options.from] - Inclusive range start (ms or ISO).
+ * @param {number|string} [options.to] - Inclusive range end (ms or ISO).
+ * @returns {Array<{ period: string, events: Object[] }>}
+ */
+export function partitionByTimeBucket(events, bucket, options = {}) {
+	const groups = new Map();
+	if (Array.isArray(events)) {
+		for (const ev of events) {
+			const ms = toMs(ev && ev.time);
+			if (!Number.isFinite(ms)) continue;
+			const period = formatBucket(ms, bucket);
+			if (!groups.has(period)) groups.set(period, []);
+			groups.get(period).push(ev);
+		}
+	}
+	const fromMs = options.from != null ? toMs(options.from) : null;
+	const toMsBound = options.to != null ? toMs(options.to) : null;
+	if (Number.isFinite(fromMs) && Number.isFinite(toMsBound)) {
+		// Enumerate every bucket period in [from, to] and seed empties.
+		for (const period of enumerateBucketPeriods(fromMs, toMsBound, bucket)) {
+			if (!groups.has(period)) groups.set(period, []);
+		}
+	}
+	return [...groups.entries()]
+		.map(([period, evs]) => ({ period, events: evs }))
+		.sort((a, b) => a.period.localeCompare(b.period));
+}
+
+/**
+ * Enumerate canonical bucket period labels covering `[fromMs, toMs]` UTC.
+ * Used to backfill empty rows when the caller supplies a trend axis.
+ *
+ * @param {number} fromMs
+ * @param {number} toMs
+ * @param {('day'|'week'|'month')} bucket
+ * @returns {string[]}
+ */
+function enumerateBucketPeriods(fromMs, toMs, bucket) {
+	const out = [];
+	if (!(toMs >= fromMs)) return out;
+	if (bucket === 'day') {
+		const start = Math.floor(fromMs / 86400_000);
+		const end = Math.floor(toMs / 86400_000);
+		for (let d = start; d <= end; d++) out.push(formatBucket(d * 86400_000, 'day'));
+	} else if (bucket === 'week') {
+		// Walk by 7 days starting from fromMs; rely on label dedup via Set.
+		const seen = new Set();
+		for (let t = fromMs; t <= toMs; t += 7 * 86400_000) {
+			const p = formatBucket(t, 'week'); if (!seen.has(p)) { seen.add(p); out.push(p); }
+		}
+		const last = formatBucket(toMs, 'week');
+		if (!seen.has(last)) out.push(last);
+	} else if (bucket === 'month') {
+		const seen = new Set();
+		const start = new Date(fromMs);
+		const end = new Date(toMs);
+		let y = start.getUTCFullYear(), m = start.getUTCMonth();
+		const yEnd = end.getUTCFullYear(), mEnd = end.getUTCMonth();
+		while (y < yEnd || (y === yEnd && m <= mEnd)) {
+			const p = formatBucket(Date.UTC(y, m, 1), 'month');
+			if (!seen.has(p)) { seen.add(p); out.push(p); }
+			m++; if (m > 11) { m = 0; y++; }
+		}
+	}
+	return out;
+}
+
+function pad2(n) { return n < 10 ? `0${n}` : `${n}`; }
+
+function formatBucket(ms, bucket) {
+	const d = new Date(ms);
+	if (bucket === 'day') {
+		return `${d.getUTCFullYear()}-${pad2(d.getUTCMonth() + 1)}-${pad2(d.getUTCDate())}`;
+	}
+	if (bucket === 'month') {
+		return `${d.getUTCFullYear()}-${pad2(d.getUTCMonth() + 1)}`;
+	}
+	if (bucket === 'week') {
+		// ISO week: Mon-anchored. Algorithm: shift to Thursday of the week,
+		// take year of that Thursday + week number relative to Jan-1 of that
+		// year's Monday-of-Thursday.
+		const date = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
+		const dayNum = (date.getUTCDay() + 6) % 7; // Mon=0..Sun=6
+		date.setUTCDate(date.getUTCDate() - dayNum + 3); // Thursday of this week
+		const year = date.getUTCFullYear();
+		const jan4 = new Date(Date.UTC(year, 0, 4));
+		const jan4DayNum = (jan4.getUTCDay() + 6) % 7;
+		const week1Mon = new Date(Date.UTC(year, 0, 4 - jan4DayNum));
+		const weekNum = Math.floor((date.getTime() - week1Mon.getTime()) / (7 * 86400_000)) + 1;
+		return `${year}-W${pad2(weekNum)}`;
+	}
+	throw new Error(`partitionByTimeBucket: unsupported bucket "${bucket}"`);
+}
+
+/**
+ * Bin a user by their distinct-period count of an event. Combines
+ * `countDistinctPeriods` with bin classification for cohort assignment.
+ *
+ * Each bin entry is `[min, max]` with `min` inclusive, `max` exclusive.
+ * Returns the first matching bin name, or `null` if no bin matches.
+ *
+ * Uses calendar-bucket counting by default (matches Mixpanel UI). Pass
+ * `options.algorithm: 'rolling'` to use the addiction_query.cpp rule
+ * instead — see `countDistinctPeriods` for the difference.
+ *
+ * Replaces total-event counting for any analysis that targets Mixpanel's
+ * frequency distribution (which counts distinct periods, not total events).
+ *
+ * @param {Object[]} events
+ * @param {string} eventName
+ * @param {Object<string, [number, number]>} bins
+ * @param {('hour'|'day'|'week')} [unit='day']
+ * @param {Object} [options]
+ * @param {('calendar'|'rolling')} [options.algorithm='calendar']
+ * @returns {string|null}
+ */
+export function binByDistinctPeriods(events, eventName, bins, unit = 'day', options = {}) {
+	const periods = countDistinctPeriods(events, eventName, unit, options);
+	if (!bins || typeof bins !== 'object') return null;
+	for (const [name, range] of Object.entries(bins)) {
+		if (!Array.isArray(range) || range.length !== 2) continue;
+		const [min, max] = range;
+		if (periods >= min && periods < max) return name;
+	}
+	return null;
+}