@ak--47/dungeon-master 1.6.5 → 1.8.0

package/docs/guides/1.8.0-upgrade-guide.md

@@ -0,0 +1,151 @@
+# 1.8.0 Upgrade Guide
+
+**TL;DR: 1.8.0 adds two new metric-table surfaces and one CSV serialization fix.**
+`standaloneEvents` is the identity-less event stream shipped earlier on this
+branch. `warehouseMetrics` is new in this release: warehouse source tables plus a
+manifest, derived from the run's own events. The one behavior change is that CSV
+output now preserves falsy cells, so `0` and `false` stop turning into empty
+strings.
+
+## What changed
+
+### 1. `standaloneEvents` is the general identity-less stream
+
+If you already adopted the branch version of `standaloneEvents`, 1.8.0 is the
+release that ships it. These rows describe a system, not a person. They carry no
+`user_id` and no `device_id`, but they do import to Mixpanel as normal events.
+
+```js
+standaloneEvents: [{
+  event: 'cdn_egress',
+  cadence: 'day',
+  dimensions: { region: ['us-east', 'us-west', 'eu'] },
+  distinctIdFrom: 'region',
+  properties: {
+    gb_out: (ctx) => 400 + ctx.tickIndex * 3,
+    cost_usd: (ctx) => (400 + ctx.tickIndex * 3) * 0.085,
+  },
+}]
+```
+
+Use it for infra, finance, and ops telemetry. If the thing should show up in
+Mixpanel as an event stream, use `standaloneEvents`.
+
+### 2. `warehouseMetrics` materializes local warehouse tables
+
+If the thing should become a warehouse metric source table, use
+`warehouseMetrics` instead. This pass runs after event generation, reads the
+run's final event stream, and emits local CSV/JSON tables plus a manifest.
+
+```js
+warehouseMetrics: [
+  {
+    name: 'daily_new_bookings',
+    source: { event: 'new_booking', measure: 'sum', property: 'booking_value' },
+    timeColumn: 'date',
+    valueColumn: 'bookings',
+  },
+  {
+    name: 'daily_active_subscriptions',
+    type: 'point-in-time',
+    source: { event: 'subscription_started', minus: 'subscription_cancelled', measure: 'count' },
+    baseline: 40,
+    timeColumn: 'date',
+    valueColumn: 'active_subscriptions',
+  },
+  {
+    name: 'monthly_arr_snapshot',
+    type: 'point-in-time',
+    grain: 'month',
+    sparse: true,
+    history: 18,
+    source: {
+      event: 'subscription_started',
+      minus: 'subscription_cancelled',
+      measure: 'sum',
+      property: 'monthly_value',
+    },
+    baseline: 24000,
+    scale: 12,
+    timeColumn: 'month',
+    valueColumn: 'arr_usd',
+  },
+]
+```
+
+Key rules:
+
+- `type`: `'additive'` or `'point-in-time'`.
+- `grain`: `'day'`, `'week'`, `'month'`.
+- `source.measure`: `'count'`, `'sum'`, `'avg'`, `'dau'`, `'users'`.
+- `sum` and `avg` require `source.property`.
+- `point-in-time` forbids `avg` and `dau`.
+- `source.groupBy` allows up to two declared keys.
+- `history` prepends synthetic buckets before the live window.
+- `sparse: true` is only valid on point-in-time metrics and emits the first row
+  plus changed values.
+
+Artifacts:
+
+- `result.warehouseMetricData`
+- `result.warehouseManifest`
+- `<name>-WAREHOUSE-<table>.csv|json`
+- `<name>-WAREHOUSE-MANIFEST.json`
+
+`token` does not import these tables. They stay local until you deploy them.
+
+### 3. Warehouse deploy is a separate, confirm-before-live step
+
+The shipped path is `/warehouse-metrics` or:
+
+```bash
+node .claude/skills/warehouse-metrics/deploy.mjs <dungeon-path> --data-prefix <verified-prefix> --dry-run
+```
+
+Review the SQL files and `warehouse/GAPS.md` first. Live mode uses
+`bq load --replace`, so it overwrites destination tables. The direct live script
+does not prompt, so the operator or agent must obtain explicit consent before
+running the live command.
+
+If the warehouse metric CRUD docs route is unavailable (`GET
+ /crud/getWarehouseMetrics` returns 404), deploy still loads the tables and
+connects the source, then writes `warehouse/GAPS.md` for the manual create flow.
+
+Two real gotchas are now documented in the shipped flow:
+
+- Manifest `recommendedAggregation: 'last value'` becomes API
+  `aggregation: 'last_value'`.
+- `previewWarehouseMetric` blocks raw substrings like `CREATE` and `UPDATE`, so
+  identifiers like `created_at` and `updated_at` fail preview. A fake alias does
+  not help if the blocked text still appears anywhere in the SQL.
+
+### 4. CSV falsy cells stop collapsing to empty strings
+
+Before 1.8.0, CSV serialization wrote `0` and `false` as blank cells. That made
+some downstream previews and ad hoc SQL look fine until a real warehouse load or
+verification run compared them to JSON or in-memory output.
+
+Now:
+
+- `0` stays `0`
+- `false` stays `false`
+- only missing values stay empty
+
+If you have downstream logic that treated `''` as a stand-in for zero or false,
+fix that logic.
+
+## Migration checklist
+
+1. Keep `standaloneEvents` when you want Mixpanel events without a person.
+2. Add `warehouseMetrics` when you want local warehouse tables and a manifest.
+3. Run the dungeon once before deploy so the warehouse artifacts exist.
+4. Use `/warehouse-metrics` in dry-run mode, then obtain explicit operator
+  consent before the live `bq load --replace` step.
+5. Recheck any CSV consumers that depended on blank falsy cells.
+
+## Notes on row counts
+
+The shipped warehouse fixture is a good shape reference, not a row-count
+contract. It uses a 60-day live window and `history: 18` on the monthly ARR
+table. `grain`, `history`, `sparse`, and `groupBy` all change row counts, so
+sample numbers in review output are illustrative.

package/dungeons/technical/warehouse.js

@@ -0,0 +1,187 @@
+// ── IMPORTS ──
+/** @typedef {import('../../types').Dungeon} Config */
+
+// ── OVERVIEW ──
+/*
+ * NAME:       warehouse
+ * PURPOSE:    Minimal warehouse-metrics fixture covering the three canonical table shapes.
+ * SCALE:      200 users, 120 events, 60 days
+ * EVENTS (4): page_view (14) > subscription_started (2) > new_booking (1) > subscription_cancelled (1)
+ * FUNNELS:     none
+ * USER PROPS:  none
+ * SUPER PROPS: none
+ * GROUPS:      none
+ */
+
+// ── SCALE ──
+const SEED = 'warehouse-fixture';
+const DATASET_START = '2025-01-01T00:00:00Z';
+const DATASET_END = '2025-03-01T23:59:59Z';
+
+// ── CONFIG ──
+/** @type {Config} */
+const config = {
+	name: 'warehouse',
+	seed: SEED,
+	datasetStart: DATASET_START,
+	datasetEnd: DATASET_END,
+	numUsers: 200,
+	numEvents: 120,
+	format: 'csv',
+	writeToDisk: false,
+	verbose: false,
+	concurrency: 1,
+	credentials: {
+		token: '',
+		region: 'US',
+	},
+	switches: {
+		hasSessionIds: false,
+		hasAdSpend: false,
+		hasLocation: false,
+		hasAndroidDevices: false,
+		hasIOSDevices: false,
+		hasDesktopDevices: false,
+		hasBrowser: false,
+		hasCampaigns: false,
+		isAnonymous: false,
+		alsoInferFunnels: false,
+	},
+	events: [
+		{
+			event: 'page_view',
+			weight: 14,
+			isStrictEvent: false,
+			properties: {
+				page: ['/', '/pricing', '/reports', '/billing'],
+			},
+		},
+		{
+			event: 'new_booking',
+			weight: 1,
+			isStrictEvent: false,
+			properties: {
+				booking_value: [1200, 1800, 2400, 3600],
+			},
+		},
+		{
+			event: 'subscription_started',
+			weight: 2,
+			isStrictEvent: false,
+			properties: {
+				monthly_value: [100, 250, 500],
+			},
+		},
+		{
+			event: 'subscription_cancelled',
+			weight: 1,
+			isStrictEvent: false,
+			properties: {
+				monthly_value: [100, 250, 500],
+			},
+		},
+	],
+	warehouseMetrics: [
+		{
+			name: 'daily_new_bookings',
+			type: 'additive',
+			grain: 'day',
+			source: {
+				event: 'new_booking',
+				measure: 'sum',
+				property: 'booking_value',
+			},
+			timeColumn: 'date',
+			valueColumn: 'bookings',
+		},
+		{
+			name: 'daily_active_subscriptions',
+			type: 'point-in-time',
+			grain: 'day',
+			source: {
+				event: 'subscription_started',
+				minus: 'subscription_cancelled',
+				measure: 'count',
+			},
+			baseline: 40,
+			timeColumn: 'date',
+			valueColumn: 'active_subscriptions',
+		},
+		{
+			name: 'monthly_arr_snapshot',
+			type: 'point-in-time',
+			grain: 'month',
+			sparse: true,
+			history: 18,
+			source: {
+				event: 'subscription_started',
+				minus: 'subscription_cancelled',
+				measure: 'sum',
+				property: 'monthly_value',
+			},
+			baseline: 24000,
+			scale: 12,
+			timeColumn: 'month',
+			valueColumn: 'arr_usd',
+		},
+	],
+};
+
+export default config;
+
+export const stories = [
+	{
+		id: 'H1-bookings-corr',
+		hook: 'H1',
+		archetype: 'temporal-inflection',
+		narrative: 'The additive warehouse bookings table should track the generated booking revenue closely enough for a warehouse metric demo.',
+		assertions: [
+			{
+				breakdown: { type: 'warehouse-stats', table: 'daily_new_bookings' },
+				select: { s: { where: {} } },
+				expect: { metric: 's.corr', op: '>=', target: 0.9, floor: 0.7 },
+			},
+		],
+	},
+	{
+		id: 'H2-active-subs-shape',
+		hook: 'H2',
+		archetype: 'session-shape',
+		narrative: 'The dense active subscription snapshot should stay fully ordered, gap-free, and numerically populated across the full dataset window.',
+		assertions: [
+			{
+				breakdown: { type: 'warehouse-stats', table: 'daily_active_subscriptions' },
+				select: { s: { where: {} } },
+				expect: { metric: 's.gaps', op: '<=', target: 0 },
+			},
+			{
+				breakdown: { type: 'warehouse-stats', table: 'daily_active_subscriptions' },
+				select: { s: { where: {} } },
+				expect: { metric: 's.emptyNumericCells', op: '<=', target: 0 },
+			},
+			{
+				breakdown: { type: 'warehouse-stats', table: 'daily_active_subscriptions' },
+				select: { s: { where: {} } },
+				expect: { metric: 's.nonMonotonicTime', op: '<=', target: 0 },
+			},
+		],
+	},
+	{
+		id: 'H3-arr-history',
+		hook: 'H3',
+		archetype: 'composition-drift',
+		narrative: 'The sparse ARR snapshot should carry meaningful monthly history before the event window without a large seam jump into the live months.',
+		assertions: [
+			{
+				breakdown: { type: 'warehouse-stats', table: 'monthly_arr_snapshot' },
+				select: { s: { where: {} } },
+				expect: { metric: 's.buckets', op: '>=', target: 18 },
+			},
+			{
+				breakdown: { type: 'warehouse-stats', table: 'monthly_arr_snapshot' },
+				select: { s: { where: {} } },
+				expect: { metric: 's.seamJumpPct', op: '<=', target: 30 },
+			},
+		],
+	},
+];

package/index.js

@@ -23,14 +23,18 @@ import { userLoop } from './lib/orchestrators/user-loop.js';
 import { sendToMixpanel, collectWrittenFiles, releaseConnections } from './lib/orchestrators/mixpanel-sender.js';
 // Generators
 import { makeAdSpend } from './lib/generators/adspend.js';
+import { makeStandaloneEvents } from './lib/generators/standalone.js';
 import { makeMirror } from './lib/generators/mirror.js';
 import { makeGroupProfile, makeProfile } from './lib/generators/profiles.js';
+import { WarehouseAccumulator, materializeWarehouseMetrics, buildManifest } from './lib/generators/warehouse.js';
 
 // Utilities
-import { initChance, initUserChance, resetUserChance, resetValueCaches, setDatasetNow, setDatasetBegin, deleteFile } from './lib/utils/utils.js';
+import { initChance, initUserChance, resetUserChance, resetValueCaches, setAutoPowerLaw, setDatasetNow, setDatasetBegin, deleteFile, getChance } from './lib/utils/utils.js';
 import { runWithDataset } from './lib/utils/dataset-context.js';
 
 // External dependencies
+import { writeFile } from 'node:fs/promises';
+import path from 'node:path';
 import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 import { timer } from 'ak-tools';
@@ -150,6 +154,12 @@ async function runDungeon(config) {
 		// Step 1: Validate and enrich configuration (resolves dataset window)
 		validatedConfig = validateDungeonConfig(config);
 
+		// v1.7.0 (P2-1): `autoPowerLaw: false` turns off the implicit 45/25/15 draw on
+		// 3–19-item string arrays for this run. Module-level flag, like the seeded
+		// chance instance — `choose()` has no config access. resetValueCaches()
+		// above already restored the default (true) for this run.
+		setAutoPowerLaw(validatedConfig.autoPowerLaw !== false);
+
 		// validateDungeonConfig always resolves these to unix seconds, but the
 		// public Dungeon type accepts string | number on input. Narrow here.
 		const fixedNow = /** @type {number} */ (validatedConfig.datasetEnd);
@@ -178,6 +188,13 @@ async function runDungeon(config) {
 		storage = await storageManager.initializeContainers();
 		updateContextWithStorage(context, storage);
 
+		if (validatedConfig.warehouseMetrics?.length > 0) {
+			context.warehouseAccumulator = new WarehouseAccumulator(validatedConfig.warehouseMetrics, {
+				FIXED_BEGIN: context.FIXED_BEGIN,
+				FIXED_NOW: context.FIXED_NOW,
+			});
+		}
+
 		// ! DATA GENERATION STARTS HERE
 
 		// Step 4: Generate ad spend data (if enabled)
@@ -188,6 +205,14 @@ async function runDungeon(config) {
 			context.reportProgress({ phase: "step", step: "adspend", status: "complete", duration: Date.now() - _t4 });
 		}
 
+		// Step 4b: Generate standalone identity-less metric snapshots (if configured) — v1.8.0
+		if (validatedConfig.standaloneEvents?.length > 0) {
+			context.reportProgress({ phase: "step", step: "standalone", status: "start" });
+			const _t4b = Date.now();
+			await generateStandaloneData(context);
+			context.reportProgress({ phase: "step", step: "standalone", status: "complete", duration: Date.now() - _t4b });
+		}
+
 		if (context.config.verbose) logger.info('Starting user and event generation...');
 		// Step 5: Main user and event generation
 		context.reportProgress({ phase: "step", step: "users", status: "start" });
@@ -227,6 +252,13 @@ async function runDungeon(config) {
 			context.reportProgress({ phase: "step", step: "mirrors", status: "complete", duration: Date.now() - _t9 });
 		}
 
+		if (validatedConfig.warehouseMetrics?.length > 0) {
+			context.reportProgress({ phase: "step", step: "warehouse", status: "start" });
+			const _t9b = Date.now();
+			await generateWarehouseData(context);
+			context.reportProgress({ phase: "step", step: "warehouse", status: "complete", duration: Date.now() - _t9b });
+		}
+
 		if (context.config.verbose) logger.info('Data generation completed successfully');
 
 		// ! DATA GENERATION ENDS HERE
@@ -282,9 +314,24 @@ async function runDungeon(config) {
 		// population for downstream tools.
 		const profilesPushed = countProfilesPushed(storage.userProfilesData);
 
+		// v1.7.0 (P2-2): every value the engine changed or flagged, validator
+		// clamps first, then aggregated runtime warnings (conversionRate saturation,
+		// users matching no funnel, …). Always present, even when empty.
+		const warnings = [
+			...(Array.isArray(validatedConfig._warnings) ? validatedConfig._warnings : []),
+			...(Array.isArray(context.warehouseAccumulator?.warnings) ? context.warehouseAccumulator.warnings.map((reason) => ({
+				key: 'warehouseMetrics',
+				reason,
+				severity: 'warn',
+			})) : []),
+			...context.getWarnings(),
+		];
+
 		return {
 			...extractedData,
+			warehouseManifest: context.warehouseManifest,
 			importResults,
+			warnings,
 			files: extractFileInfo(storage),
 			time: { start, end, delta, human },
 			operations: context.getOperations(),
@@ -341,6 +388,79 @@ async function generateAdSpendData(context) {
 	}
 }
 
+/**
+ * Generate standalone identity-less metric snapshots — v1.8.0.
+ *
+ * One record per cadence tick per dimension cross-product row. Records carry no
+ * `user_id` and no `device_id`; they describe a system, not a person.
+ *
+ * @param {Context} context - Context object
+ */
+async function generateStandaloneData(context) {
+	const { config, storage } = context;
+	const specs = /** @type {import('./types').ResolvedStandaloneEventConfig[]} */ (config.standaloneEvents);
+
+	for (const spec of specs) {
+		const records = makeStandaloneEvents(context, spec);
+		for (const record of records) {
+			// The `standalone` hook fires on push, like ad-spend. Meta carries the
+			// stream's resolved spec so a hook can tell the streams apart.
+			// `datasetStart`/`datasetEnd` are added by hookPush itself.
+			await storage.standaloneEventData.hookPush(
+				/** @type {import('./types').EventSchema} */ (record),
+				{ spec, config }
+			);
+		}
+	}
+}
+
+/**
+ * Materialize configured warehouse metric tables after the user loop completes.
+ *
+ * The accumulator taps the final per-user event stream during Step 5. This step
+ * runs afterward so seeded noise and column callbacks cannot perturb event generation.
+ *
+ * @param {Context} context - Context object
+ */
+async function generateWarehouseData(context) {
+	const { config, storage } = context;
+	const specs = /** @type {import('./types').ResolvedWarehouseMetricConfig[]} */ (config.warehouseMetrics);
+	const accumulator = context.warehouseAccumulator;
+	if (!Array.isArray(specs) || specs.length === 0 || !accumulator) return;
+
+	const materialized = materializeWarehouseMetrics({
+		specs,
+		accumulator,
+		chance: getChance(),
+		FIXED_BEGIN: context.FIXED_BEGIN,
+		FIXED_NOW: context.FIXED_NOW,
+		configName: config.name,
+		config,
+	});
+
+	for (let index = 0; index < materialized.length; index += 1) {
+		const entry = materialized[index];
+		const container = storage.warehouseMetricData?.[index];
+		if (!container) continue;
+
+		for (let rowIndex = 0; rowIndex < entry.rows.length; rowIndex += 1) {
+			await container.hookPush(entry.rows[rowIndex], entry.metas[rowIndex]);
+		}
+	}
+
+	const postHookMaterialized = specs.map((spec, index) => ({
+		spec,
+		rows: Array.from(storage.warehouseMetricData?.[index] || []),
+	}));
+	context.warehouseManifest = buildManifest(specs, postHookMaterialized, config.name);
+
+	if (config.writeToDisk && storage.warehouseMetricData?.[0]?.getWriteDir) {
+		const manifestPath = path.join(storage.warehouseMetricData[0].getWriteDir(), `${config.name}-WAREHOUSE-MANIFEST.json`);
+		await writeFile(manifestPath, JSON.stringify(context.warehouseManifest, null, 2));
+		storage.warehouseManifestFile = manifestPath;
+	}
+}
+
 /**
  * Generate group profiles for all configured group keys
  * @param {Context} context - Context object
@@ -552,11 +672,12 @@ async function flushStorageToDisk(storage, config) {
 	if (storage.eventData?.flush) flushPromises.push(storage.eventData.flush());
 	if (storage.userProfilesData?.flush) flushPromises.push(storage.userProfilesData.flush());
 	if (storage.adSpendData?.flush) flushPromises.push(storage.adSpendData.flush());
+	if (storage.standaloneEventData?.flush) flushPromises.push(storage.standaloneEventData.flush());
 	if (storage.mirrorEventData?.flush) flushPromises.push(storage.mirrorEventData.flush());
 	if (storage.groupEventData?.flush) flushPromises.push(storage.groupEventData.flush());
 
 	// Flush arrays of HookedArrays (excluding lookup tables which are handled separately)
-	[storage.scdTableData, storage.groupProfilesData].forEach(arrayOfContainers => {
+	[storage.scdTableData, storage.groupProfilesData, storage.warehouseMetricData].forEach(arrayOfContainers => {
 		if (Array.isArray(arrayOfContainers)) {
 			arrayOfContainers.forEach(container => {
 				if (container?.flush) flushPromises.push(container.flush());
@@ -620,11 +741,19 @@ function countProfilesPushed(profilesContainer) {
  * @returns {object} Extracted data in Result format
  */
 function extractStorageData(storage) {
+	const warehouseMetricData = {};
+	for (const container of storage.warehouseMetricData || []) {
+		if (!container?.metricName) continue;
+		warehouseMetricData[container.metricName] = Array.from(container);
+	}
+
 	return {
 		eventData: storage.eventData || [],
 		mirrorEventData: storage.mirrorEventData || [],
 		userProfilesData: storage.userProfilesData || [],
 		adSpendData: storage.adSpendData || [],
+		standaloneEventData: storage.standaloneEventData || [],
+		warehouseMetricData,
 		// Keep arrays of HookedArrays as separate arrays (don't flatten)
 		scdTableData: storage.scdTableData || [],
 		groupProfilesData: storage.groupProfilesData || [],