@ak--47/dungeon-master 1.5.4 → 1.6.1

package/lib/verify/counting.js

@@ -11,6 +11,8 @@
  */
 
 import { toMs } from '../hook-helpers/_internal.js';
+import { matchesWhere } from './coerce.js';
+import { buildIdentityMap, resolveUserId } from './identity.js';
 
 const SECONDS_PER_UNIT = {
 	hour: 3600,
@@ -21,43 +23,43 @@ const SECONDS_PER_UNIT = {
 /**
  * Count distinct time periods on which a user fired a given event.
  *
- * Two related rules exist in Mixpanel:
+ * Two related rules exist in Mixpanel — v1.6.0 (P1.10, findings #3) names
+ * them for what they are:
  *
- * 1. **Calendar bucket** (default here, `algorithm: 'calendar'`):
- *    `COUNT(DISTINCT date_trunc(unit, time))` in UTC. This is what the
- *    Mixpanel UI presents — frequency distribution charts bucket events
- *    into calendar hours/days/weeks.
+ * 1. **`'ui-bucket'`** (default): `COUNT(DISTINCT date_trunc(unit, time))`
+ *    in UTC — calendar-bucket counting, the shape the Mixpanel UI presents
+ *    in frequency distribution charts.
  *
- * 2. **Rolling window** (`algorithm: 'rolling'`): the addiction_query.cpp
- *    rule, `qtz_time >= last_counted + seconds_for_unit(unit)`. This is
- *    Mixpanel's internal C++ implementation. It diverges from calendar
- *    bucketing at unit boundaries — events at 23:59 and 00:01 next day
- *    register as 1 rolling-window period (gap 120s < 86400s) but 2
- *    calendar-day periods.
+ * 2. **`'mixpanel-rolling'`**: the addiction_query.cpp rule,
+ *    `qtz_time >= interval->last_counted + seconds_for_unit(unit)`
+ *    (addiction_query_update_history, addiction_query.cpp:363-374). This
+ *    is what Mixpanel's C++ reader actually computes. It diverges from
+ *    calendar bucketing at unit boundaries — events at 23:59 and 00:01
+ *    next day register as 1 rolling period (gap 120s < 86400s) but 2
+ *    calendar days.
  *
- * The default is `calendar` because:
- *   - It matches what users actually see in Mixpanel reports.
- *   - It aligns with `injectOnNewDays`, which classifies days by
- *     `Math.floor(t / DAY_MS)` to find empty days. Mixing the two
- *     algorithms causes the atom and verifier to disagree at boundaries.
+ * **The default does NOT match addiction_query.cpp's rolling rule.** It is
+ * kept because it matches what users see in report buckets, and because it
+ * aligns with `injectOnNewDays` (which classifies days by
+ * `Math.floor(t / DAY_MS)`); mixing the two algorithms makes the atom and
+ * verifier disagree at boundaries. For the actual Frequency report output
+ * shape (per-interval rolling counters + histogram), use
+ * `frequencyHistogram` instead.
  *
- * Use `algorithm: 'rolling'` only when you're verifying behavior that
- * specifically depends on the C++ rolling-window check.
- *
- * Reference: `mixpanel/analytics`
- *   - calendar bucketing: implicit in the UI / Insights reports
- *   - rolling-window: `backend/arb/reader/queries/addiction_query.cpp`
+ * `'calendar'` and `'rolling'` remain accepted as silent back-compat
+ * aliases for `'ui-bucket'` / `'mixpanel-rolling'`.
  *
  * @param {Object[]} events - Events to scan (mixed types OK).
  * @param {string} eventName - Event name to filter for.
  * @param {('hour'|'day'|'week')} [unit='day']
  * @param {Object} [options]
- * @param {('calendar'|'rolling')} [options.algorithm='calendar']
+ * @param {('ui-bucket'|'mixpanel-rolling'|'calendar'|'rolling')} [options.algorithm='ui-bucket']
  * @returns {number} Distinct period count.
  */
 export function countDistinctPeriods(events, eventName, unit = 'day', options = {}) {
 	const seconds = SECONDS_PER_UNIT[unit];
 	if (!seconds) throw new Error(`countDistinctPeriods: unsupported unit "${unit}"`);
+	const algorithm = normalizeDistinctPeriodAlgorithm(options.algorithm);
 	if (!Array.isArray(events) || !events.length) return 0;
 	const matches = events
 		.filter(e => e && e.event === eventName)
@@ -65,9 +67,8 @@ export function countDistinctPeriods(events, eventName, unit = 'day', options =
 		.filter(ms => Number.isFinite(ms));
 	if (!matches.length) return 0;
 	const unitMs = seconds * 1000;
-	const { algorithm = 'calendar' } = options;
 
-	if (algorithm === 'calendar') {
+	if (algorithm === 'ui-bucket') {
 		// Calendar bucket — UTC floor by unit. Matches what Mixpanel's UI
 		// shows and what `injectOnNewDays` uses internally.
 		const buckets = new Set();
@@ -75,7 +76,7 @@ export function countDistinctPeriods(events, eventName, unit = 'day', options =
 		return buckets.size;
 	}
 
-	// Rolling window — addiction_query.cpp semantics.
+	// Rolling window — addiction_query.cpp:363-374 semantics.
 	matches.sort((a, b) => a - b);
 	let count = 0;
 	let lastCountedMs = -Infinity;
@@ -88,6 +89,28 @@ export function countDistinctPeriods(events, eventName, unit = 'day', options =
 	return count;
 }
 
+/**
+ * Normalize the countDistinctPeriods algorithm name. `'calendar'` and
+ * `'rolling'` are silent v1.5 back-compat aliases; unknown names throw
+ * (they previously fell through to the rolling branch silently).
+ *
+ * @param {string} [algorithm]
+ * @returns {('ui-bucket'|'mixpanel-rolling')}
+ */
+function normalizeDistinctPeriodAlgorithm(algorithm) {
+	switch (algorithm) {
+		case undefined:
+		case 'ui-bucket':
+		case 'calendar':
+			return 'ui-bucket';
+		case 'mixpanel-rolling':
+		case 'rolling':
+			return 'mixpanel-rolling';
+		default:
+			throw new Error(`countDistinctPeriods: unknown algorithm "${algorithm}" — use 'ui-bucket' or 'mixpanel-rolling'`);
+	}
+}
+
 /**
  * Null-aware average matching Mixpanel's aggregation semantics.
  *
@@ -104,17 +127,40 @@ export function countDistinctPeriods(events, eventName, unit = 'day', options =
  * Differs from naive `SUM(x) / COUNT(*)` which inflates the denominator
  * by counting rows where x is missing — diluting the average toward 0.
  *
+ * v1.6.0 (P1.10): `{ flatten: true }` mirrors the ACTION_TYPE_AVERAGE
+ * VALUE_TYPE_LIST branch (`normal_query.cpp:1601-1617`): a list-valued
+ * property contributes each numeric ITEM independently — every item adds
+ * to the numerator AND increments the denominator. One level only
+ * (`list_cursor_next` items must themselves be VALUE_TYPE_NUMBER; a
+ * nested list item is skipped, not recursed). Non-numeric items inside a
+ * list are skipped. Default stays non-flattening: without the opt-in, an
+ * array value is non-numeric and is skipped whole — the explicit flag
+ * avoids silently changing v1.5 results for 1-item-array data.
+ * (ARB's list branch carries no isnan guard — the scalar branch does —
+ * but NaN can't arrive via JSON ingestion; we keep the same finite guard
+ * on items as on scalars.)
+ *
  * @param {*[]} values
+ * @param {Object} [options]
+ * @param {boolean} [options.flatten=false]
  * @returns {number|null}
  */
-export function nullAwareAvg(values) {
+export function nullAwareAvg(values, options = {}) {
 	if (!Array.isArray(values) || !values.length) return null;
+	const { flatten = false } = options;
 	let sum = 0;
 	let count = 0;
 	for (const v of values) {
 		if (typeof v === 'number' && Number.isFinite(v)) {
 			sum += v;
 			count++;
+		} else if (flatten && Array.isArray(v)) {
+			for (const item of v) {
+				if (typeof item === 'number' && Number.isFinite(item)) {
+					sum += item;
+					count++;
+				}
+			}
 		}
 	}
 	return count ? sum / count : null;
@@ -128,14 +174,28 @@ export function nullAwareAvg(values) {
  * upstream; in JS arrays missing values are typically `undefined` which
  * produces NaN under `+`.
  *
+ * v1.6.0 (P1.10): `{ flatten: true }` mirrors the ACTION_TYPE_SUM
+ * VALUE_TYPE_LIST branch (`normal_query.cpp:1585-1600`): each numeric
+ * item of a list-valued property adds to the sum independently, one
+ * level only, non-numeric items skipped. Same explicit opt-in rationale
+ * as `nullAwareAvg`.
+ *
  * @param {*[]} values
+ * @param {Object} [options]
+ * @param {boolean} [options.flatten=false]
  * @returns {number}
  */
-export function nullAwareSum(values) {
+export function nullAwareSum(values, options = {}) {
 	if (!Array.isArray(values) || !values.length) return 0;
+	const { flatten = false } = options;
 	let sum = 0;
 	for (const v of values) {
 		if (typeof v === 'number' && Number.isFinite(v)) sum += v;
+		else if (flatten && Array.isArray(v)) {
+			for (const item of v) {
+				if (typeof item === 'number' && Number.isFinite(item)) sum += item;
+			}
+		}
 	}
 	return sum;
 }
@@ -299,6 +359,47 @@ function enumerateBucketPeriods(fromMs, toMs, bucket) {
 	return out;
 }
 
+/**
+ * Resolve a bucket period label back to its UTC millisecond bounds.
+ * `startMs` inclusive, `endMs` exclusive — the `[start, stop)` convention
+ * Mixpanel uses for trend intervals (funnel_query.cpp:1398-1401 anchors
+ * step 0 in `[start, stop)`). Inverse of the labels `partitionByTimeBucket`
+ * emits: `YYYY-MM-DD` (day), `YYYY-Www` (ISO week, Monday-anchored),
+ * `YYYY-MM` (month).
+ *
+ * @param {string} period
+ * @param {('day'|'week'|'month')} bucket
+ * @returns {{ startMs: number, endMs: number }}
+ */
+export function bucketBoundsMs(period, bucket) {
+	if (bucket === 'day') {
+		const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(period);
+		if (!m) throw new Error(`bucketBoundsMs: bad day period "${period}"`);
+		const startMs = Date.UTC(+m[1], +m[2] - 1, +m[3]);
+		return { startMs, endMs: startMs + 86400_000 };
+	}
+	if (bucket === 'month') {
+		const m = /^(\d{4})-(\d{2})$/.exec(period);
+		if (!m) throw new Error(`bucketBoundsMs: bad month period "${period}"`);
+		return {
+			startMs: Date.UTC(+m[1], +m[2] - 1, 1),
+			endMs: Date.UTC(+m[1], +m[2], 1),
+		};
+	}
+	if (bucket === 'week') {
+		const m = /^(\d{4})-W(\d{2})$/.exec(period);
+		if (!m) throw new Error(`bucketBoundsMs: bad week period "${period}"`);
+		// Monday of ISO week w: week 1 contains Jan 4 (same anchor formatBucket uses).
+		const year = +m[1], week = +m[2];
+		const jan4 = new Date(Date.UTC(year, 0, 4));
+		const jan4DayNum = (jan4.getUTCDay() + 6) % 7;
+		const week1MonMs = Date.UTC(year, 0, 4 - jan4DayNum);
+		const startMs = week1MonMs + (week - 1) * 7 * 86400_000;
+		return { startMs, endMs: startMs + 7 * 86400_000 };
+	}
+	throw new Error(`bucketBoundsMs: unsupported bucket "${bucket}"`);
+}
+
 function pad2(n) { return n < 10 ? `0${n}` : `${n}`; }
 
 function formatBucket(ms, bucket) {
@@ -333,9 +434,10 @@ function formatBucket(ms, bucket) {
  * Each bin entry is `[min, max]` with `min` inclusive, `max` exclusive.
  * Returns the first matching bin name, or `null` if no bin matches.
  *
- * Uses calendar-bucket counting by default (matches Mixpanel UI). Pass
- * `options.algorithm: 'rolling'` to use the addiction_query.cpp rule
- * instead — see `countDistinctPeriods` for the difference.
+ * Uses `'ui-bucket'` (calendar) counting by default — matches Mixpanel's
+ * UI buckets, NOT addiction_query.cpp's rolling rule. Pass
+ * `options.algorithm: 'mixpanel-rolling'` for the C++ rule — see
+ * `countDistinctPeriods` for the difference.
  *
  * Replaces total-event counting for any analysis that targets Mixpanel's
  * frequency distribution (which counts distinct periods, not total events).
@@ -345,7 +447,7 @@ function formatBucket(ms, bucket) {
  * @param {Object<string, [number, number]>} bins
  * @param {('hour'|'day'|'week')} [unit='day']
  * @param {Object} [options]
- * @param {('calendar'|'rolling')} [options.algorithm='calendar']
+ * @param {('ui-bucket'|'mixpanel-rolling'|'calendar'|'rolling')} [options.algorithm='ui-bucket']
  * @returns {string|null}
  */
 export function binByDistinctPeriods(events, eventName, bins, unit = 'day', options = {}) {
@@ -358,3 +460,139 @@ export function binByDistinctPeriods(events, eventName, bins, unit = 'day', opti
 	}
 	return null;
 }
+
+/**
+ * The actual Frequency (Addiction) report output shape — per report
+ * interval, a histogram of users bucketed by their ROLLING distinct-unit
+ * count. This is what `countDistinctPeriods`'s JSDoc points to for real
+ * Frequency-report parity.
+ *
+ * ARB semantics (from `mixpanel/analytics`):
+ *   - Rolling counter: an event counts iff
+ *     `qtz_time >= interval->last_counted + seconds_for_unit(unit)`; then
+ *     `last_counted = qtz_time`, `count++` (`addiction_query_update_history`,
+ *     `addiction_query.cpp:363-374`). `last_counted` lives per-(user,
+ *     interval) and is calloc-zeroed — the counter RESETS at every interval
+ *     boundary, so the first qualifying event in each interval always counts.
+ *   - Histogram: `intervals_counts[interval][count - 1] += upsampling_factor`
+ *     for every user history with `count > 0`. Users with ZERO qualifying
+ *     events in an interval are omitted entirely — there is no zero bucket
+ *     (`addiction_query_prepare_results` ACTION_TYPE_COUNT,
+ *     `addiction_query.cpp:546-573`). The print loop emits EVERY interval,
+ *     zero-filled ones included.
+ *   - Histogram length: `(interval_seconds + unit_seconds - 1) / unit_seconds`
+ *     — integer ceil (`addiction_units_per_interval`,
+ *     `libquery/time/unit.c:108-113`).
+ *   - Interval selection: forward tiling from range start,
+ *     `index = (event_time - from_time) / interval_length`
+ *     (`uniform_intervals_for`, `libquery/time/uniform_intervals.h:48-58`).
+ *
+ * Range stand-in: ARB takes explicit `--from-date`/`--to-date` query params.
+ * This verifier has no query range, so intervals tile forward from UTC
+ * midnight of the FIRST event in the stream (any event, not just matches)
+ * through the last — the same dataset-boundary discipline `lifecycle` uses.
+ * ARB selects the interval by ptz_time and gap-checks by qtz_time; both are
+ * raw UTC timestamps here — shift event times upstream for a
+ * query-timezone view.
+ *
+ * @param {Object[]} events
+ * @param {Object} [options]
+ * @param {string} [options.event] - Event name to count. Required at runtime — throws if missing.
+ * @param {('hour'|'day'|'week')} [options.unit='day'] - Rolling unit.
+ * @param {number} [options.intervalDays] - Report interval length in days (positive integer). Required at runtime.
+ * @param {Object[]} [options.profiles] - Profiles for device→user identity resolution.
+ * @returns {Array<{ interval: string, histogram: number[] }>} One row per
+ *   interval (label = ISO date of the interval start); `histogram[i]` =
+ *   number of users with rolling count `i + 1` in that interval.
+ */
+export function frequencyHistogram(events, options = {}) {
+	const { event, unit = 'day', intervalDays, profiles } = options;
+	if (typeof event !== 'string' || !event) {
+		throw new Error('frequencyHistogram: event is required');
+	}
+	const unitSeconds = SECONDS_PER_UNIT[unit];
+	if (!unitSeconds) throw new Error(`frequencyHistogram: unsupported unit "${unit}"`);
+	if (!Number.isInteger(intervalDays) || intervalDays <= 0) {
+		throw new Error('frequencyHistogram: intervalDays must be a positive integer');
+	}
+	if (!Array.isArray(events) || !events.length) return [];
+
+	const identityMap = profiles ? buildIdentityMap(profiles) : undefined;
+	let minMs = Infinity, maxMs = -Infinity;
+	const matches = [];
+	for (const e of events) {
+		if (!e) continue;
+		const ms = toMs(e.time);
+		if (!Number.isFinite(ms)) continue;
+		if (ms < minMs) minMs = ms;
+		if (ms > maxMs) maxMs = ms;
+		if (e.event !== event) continue;
+		const uid = resolveUserId(e, identityMap);
+		if (!uid) continue;
+		matches.push({ uid, ms });
+	}
+	if (!Number.isFinite(minMs)) return [];
+
+	const intervalMs = intervalDays * 86400_000;
+	const unitMs = unitSeconds * 1000;
+	// UTC midnight of the first event's day — deterministic from_date stand-in.
+	const fromMs = Math.floor(minMs / 86400_000) * 86400_000;
+	const numIntervals = Math.floor((maxMs - fromMs) / intervalMs) + 1;
+	// ceil — addiction_units_per_interval (unit.c:108-113)
+	const histLen = Math.ceil((intervalDays * 86400) / unitSeconds);
+
+	// Per-(user, interval) rolling histories (addiction_query.cpp:363-374).
+	// One Map per interval mirrors ARB's calloc'd per-interval history slabs.
+	/** @type {Array<Map<string, { count: number, lastCountedMs: number }>>} */
+	const perInterval = Array.from({ length: numIntervals }, () => new Map());
+	matches.sort((a, b) => a.ms - b.ms);
+	for (const { uid, ms } of matches) {
+		// forward tiling — uniform_intervals_for (uniform_intervals.h:48-58)
+		const idx = Math.floor((ms - fromMs) / intervalMs);
+		const histories = perInterval[idx];
+		const h = histories.get(uid);
+		if (!h) {
+			// calloc-zeroed history → first qualifying event always counts
+			histories.set(uid, { count: 1, lastCountedMs: ms });
+		} else if (ms >= h.lastCountedMs + unitMs) {
+			h.count++;
+			h.lastCountedMs = ms;
+		}
+	}
+
+	const out = [];
+	for (let i = 0; i < numIntervals; i++) {
+		const histogram = new Array(histLen).fill(0);
+		for (const { count } of perInterval[i].values()) {
+			// count > 0 by construction — zero-count users never enter the
+			// map, matching ARB's no-zero-bucket omission (:546-573).
+			histogram[count - 1] += 1;
+		}
+		out.push({ interval: formatBucket(fromMs + i * intervalMs, 'day'), histogram });
+	}
+	return out;
+}
+
+/**
+ * Count events, optionally filtered by event name and a `where` object —
+ * Mixpanel's plain ACTION_TYPE_COUNT (action.h:14; each qualifying event
+ * increments the interval count by one — normal_query.cpp
+ * `result->count += q->upsampling_factor`). `where` string comparison is
+ * case-INSENSITIVE per the WHERE-filter rulebook (see coerce.js).
+ *
+ * @param {Object[]} events
+ * @param {Object} [options]
+ * @param {string} [options.event] restrict to this event name
+ * @param {Object<string, *>} [options.where] `{ prop: value | { op, value } }`
+ * @returns {number}
+ */
+export function countEvents(events, options = {}) {
+	const { event, where } = options;
+	let count = 0;
+	for (const e of events) {
+		if (!e || (event && e.event !== event)) continue;
+		if (!matchesWhere(e, where)) continue;
+		count++;
+	}
+	return count;
+}