@ak--47/dungeon-master 1.6.5 → 1.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ak--47/dungeon-master",
-	"version": "1.6.5",
+	"version": "1.8.0",
 	"description": "generate fancy datasets",
 	"type": "module",
 	"main": "index.js",
@@ -48,15 +48,7 @@
 		"dungeon:run": "node ./scripts/run-dungeon.mjs",
 		"dungeon:to-json": "node ./scripts/dungeon-to-json.mjs",
 		"dungeon:from-json": "node ./scripts/json-to-dungeon.mjs",
-		"dungeon:schema": "node ./scripts/extract-dungeon-schema.mjs",
-		"kodiak:deploy": "gcloud builds submit --config dungeons/user/kodiak/cloudbuild.yaml",
-		"kodiak:plan": "node dungeons/user/kodiak/kickoff.mjs --plan-only",
-		"kodiak:smoke:local": "bash dungeons/user/kodiak/smoke-local.sh",
-		"kodiak:smoke": "node dungeons/user/kodiak/kickoff.mjs --total-events 1000000000 --chunk-cap 10",
-		"kodiak:generate": "node dungeons/user/kodiak/kickoff.mjs --no-wait",
-		"kodiak:status": "node dungeons/user/kodiak/status.mjs",
-		"kodiak:run": "bash dungeons/user/kodiak/run-all.sh",
-		"kodiak:load": "node dungeons/user/kodiak/load-bq.mjs --replace --no-wait"
+		"dungeon:schema": "node ./scripts/extract-dungeon-schema.mjs"
 	},
 	"repository": {
 		"type": "git",
@@ -83,6 +75,7 @@
 		"@google-cloud/storage": "^7.14.0",
 		"ak-tools": "^1.1.12",
 		"chance": "^1.1.11",
+		"csv-parse": "^7.0.2",
 		"dayjs": "^1.11.11",
 		"dotenv": "^16.4.5",
 		"hyparquet-writer": "^0.6.1",
@@ -100,7 +93,8 @@
 		"@vitest/ui": "^2.1.9",
 		"nodemon": "^3.1.3",
 		"typescript": "^5.6.0",
-		"vitest": "^2.1.9"
+		"vitest": "^2.1.9",
+		"yaml": "^2.9.0"
 	},
 	"nodemonConfig": {
 		"ignore": [

package/scripts/verify-stories.mjs

@@ -34,15 +34,18 @@ import readline from 'readline';
 import { execFile } from 'child_process';
 import { promisify } from 'util';
 import { pathToFileURL } from 'url';
+import { parse as parseCsv } from 'csv-parse';
+import generate from '../index.js';
 import { extractComments } from '../lib/core/extract-comments.js';
 import { validateDungeonConfig } from '../lib/core/config-validator.js';
 import {
 	buildIdentityMap,
-	verifyDungeon,
 	VERDICT_RANK,
 	validateStories,
-	storiesToChecks,
+	validateSchema,
 	evaluateStories,
+	auditWarehouseRows,
+	computeWarehouseSourceRows,
 } from '../lib/verify/index.js';
 
 const USAGE = `Usage: node scripts/verify-stories.mjs <dungeon-path> [--data-prefix <prefix>] [--in-memory] [--json]
@@ -89,13 +92,16 @@ if (!fs.existsSync(abs)) die(`verify-stories: dungeon not found at ${abs}`);
 
 const mod = await import(pathToFileURL(abs).href);
 const config = mod.default;
-const stories = mod.stories;
+const stories = Array.isArray(mod.stories) ? mod.stories : [];
 if (!config || typeof config !== 'object') die(`verify-stories: ${dungeonPath} has no default-exported config object`);
-if (!Array.isArray(stories) || !stories.length) {
+const hasWarehouseMetrics = Array.isArray(config.warehouseMetrics) && config.warehouseMetrics.length > 0;
+if (!stories.length && !hasWarehouseMetrics) {
 	die(`verify-stories: ${dungeonPath} has no \`stories\` named export — add one (see lib/templates/story-spec.schema.json) or use scripts/verify-runner.mjs for ad-hoc checks.`);
 }
-const sv = validateStories(stories);
-if (!sv.valid) die(`verify-stories: invalid stories:\n  ${sv.errors.join('\n  ')}`);
+if (stories.length) {
+	const sv = validateStories(stories);
+	if (!sv.valid) die(`verify-stories: invalid stories:\n  ${sv.errors.join('\n  ')}`);
+}
 
 // ── coverage discipline ─────────────────────────────────────────────────────
 // Every numbered hook in the HOOK STORIES comment block needs >=1 story. One
@@ -129,28 +135,30 @@ const coverage = {
 
 let storyResults;   // Array<{ id, hook, archetype, verdict, assertions }>
 let schemaPass = true;
+let warehouseAudits = [];
 
 if (inMemory) {
-	const checks = storiesToChecks(stories); // warns + skips duckdb assertions
-	if (!checks.length) die('verify-stories: every assertion is duckdb (disk-mode-only) — nothing to run in-memory. Drop --in-memory.');
-	// token: '' prevents any Mixpanel send; in-memory verification never writes.
-	const report = await verifyDungeon({ ...config, token: '', writeToDisk: false }, checks);
-	schemaPass = !!report.schemaReport?.pass;
-	const byStory = new Map(stories.map(s => [s.id, { id: s.id, hook: s.hook, archetype: s.archetype, verdict: null, assertions: [] }]));
-	for (const r of report.results) {
-		const m = /^(.+)\[(\d+)\]$/.exec(r.name);
-		const st = m && byStory.get(m[1]);
-		if (!st) continue;
-		const verdict = (/^([A-Z]+) — /.exec(r.detail || '') || [])[1] || (r.pass ? 'STRONG' : 'NONE');
-		st.assertions.push({ name: r.name, verdict, observed: null, detail: r.detail || '' });
-	}
-	for (const st of byStory.values()) {
-		// duckdb-only stories have zero in-memory assertions: informational SKIPPED.
-		st.verdict = st.assertions.length
-			? st.assertions.reduce((w, a) => VERDICT_RANK[a.verdict] < VERDICT_RANK[w] ? a.verdict : w, 'NAILED')
-			: 'SKIPPED';
-	}
-	storyResults = [...byStory.values()];
+	const result = await generate({ ...config, token: '', writeToDisk: false });
+	const events = Array.isArray(result.eventData) ? result.eventData : Array.from(result.eventData || []);
+	const profiles = Array.isArray(result.userProfilesData) ? result.userProfilesData : Array.from(result.userProfilesData || []);
+	const validated = result.validatedConfig || validateDungeonConfig({ ...config, token: '' });
+	schemaPass = !!validateSchema(events, validated)?.pass;
+	const warehouseSpecs = Object.fromEntries((validated.warehouseMetrics || []).map((spec) => [spec.name, spec]));
+	const warehouseRows = result.warehouseMetricData || {};
+	storyResults = stories.length
+		? await evaluateStories(stories, events, {
+			profiles,
+			funnels: Array.isArray(validated.funnels) ? validated.funnels : [],
+			identityMap: buildIdentityMap(profiles),
+			warehouseRows,
+			warehouseSpecs,
+			datasetStart: validated.datasetStart,
+			datasetEnd: validated.datasetEnd,
+			skipDiskOnlyDuckdb: true,
+			onWarning: (message) => console.error(message),
+		})
+		: [];
+	warehouseAudits = buildWarehouseAudits(warehouseSpecs, warehouseRows, validated, events);
 } else {
 	const prefix = dataPrefix || `verify-${path.basename(abs).replace(/\.(js|mjs)$/, '')}`;
 	const prefixPath = prefix.includes('/') ? prefix : path.join('data', prefix);
@@ -180,6 +188,12 @@ if (inMemory) {
 	// enrich the object you hand it.
 	const validated = validateDungeonConfig({ ...config, token: '' });
 	const identityMap = buildIdentityMap(profiles);
+	schemaPass = !!validateSchema(events, validated)?.pass;
+	const warehouseSpecs = Object.fromEntries((validated.warehouseMetrics || []).map((spec) => [spec.name, spec]));
+	const warehouseManifest = loadWarehouseManifest(prefixPath);
+	const warehouseRows = Object.fromEntries(await Promise.all(
+		Object.values(warehouseSpecs).map(async (spec) => [spec.name, await loadWarehouseRows(prefixPath, spec, warehouseManifest)])
+	));
 
 	const execFileP = promisify(execFile);
 	const runSql = async (sql) => {
@@ -195,19 +209,27 @@ if (inMemory) {
 		return trimmed ? JSON.parse(trimmed) : [];
 	};
 
-	storyResults = await evaluateStories(stories, events, {
-		profiles,
-		funnels: Array.isArray(validated.funnels) ? validated.funnels : [],
-		identityMap,
-		runSql,
-	});
+	storyResults = stories.length
+		? await evaluateStories(stories, events, {
+			profiles,
+			funnels: Array.isArray(validated.funnels) ? validated.funnels : [],
+			identityMap,
+			runSql,
+			warehouseRows,
+			warehouseSpecs,
+			datasetStart: validated.datasetStart,
+			datasetEnd: validated.datasetEnd,
+		})
+		: [];
+	warehouseAudits = buildWarehouseAudits(warehouseSpecs, warehouseRows, validated, events, warehouseManifest);
 }
 
 // ── report ──────────────────────────────────────────────────────────────────
 
 const counted = storyResults.filter(s => s.verdict !== 'SKIPPED');
 const failing = counted.filter(s => VERDICT_RANK[s.verdict] < VERDICT_RANK.STRONG);
-const pass = failing.length === 0 && missing.length === 0 && schemaPass;
+const failingAudits = warehouseAudits.filter((audit) => !audit.pass);
+const pass = failing.length === 0 && failingAudits.length === 0 && missing.length === 0 && schemaPass;
 
 if (asJson) {
 	console.log(JSON.stringify({
@@ -215,26 +237,36 @@ if (asJson) {
 		mode: inMemory ? 'in-memory' : 'disk',
 		coverage,
 		stories: storyResults,
+		warehouseAudits,
 		schemaPass,
 		pass,
 	}, null, 2));
 } else {
-	const wId = Math.max(5, ...storyResults.map(s => s.id.length));
-	const wHook = Math.max(4, ...storyResults.map(s => String(s.hook).length));
-	const wArch = Math.max(9, ...storyResults.map(s => s.archetype.length));
-	console.log('');
-	console.log(`${'STORY'.padEnd(wId)}  ${'HOOK'.padEnd(wHook)}  ${'ARCHETYPE'.padEnd(wArch)}  VERDICT`);
-	for (const s of storyResults) {
-		console.log(`${s.id.padEnd(wId)}  ${String(s.hook).padEnd(wHook)}  ${s.archetype.padEnd(wArch)}  ${s.verdict}`);
-		for (const a of s.assertions) {
-			console.log(`  ${a.name.padEnd(wId)}  ${a.verdict} — ${a.detail.replace(/^[A-Z]+ — /, '')}`);
+	if (storyResults.length) {
+		const wId = Math.max(5, ...storyResults.map(s => s.id.length));
+		const wHook = Math.max(4, ...storyResults.map(s => String(s.hook).length));
+		const wArch = Math.max(9, ...storyResults.map(s => s.archetype.length));
+		console.log('');
+		console.log(`${'STORY'.padEnd(wId)}  ${'HOOK'.padEnd(wHook)}  ${'ARCHETYPE'.padEnd(wArch)}  VERDICT`);
+		for (const s of storyResults) {
+			console.log(`${s.id.padEnd(wId)}  ${String(s.hook).padEnd(wHook)}  ${s.archetype.padEnd(wArch)}  ${s.verdict}`);
+			for (const a of s.assertions) {
+				console.log(`  ${a.name.padEnd(wId)}  ${a.verdict} — ${a.detail.replace(/^[A-Z]+ — /, '')}`);
+			}
 		}
+		console.log('');
 	}
-	console.log('');
 	const tally = {};
 	for (const s of counted) tally[s.verdict] = (tally[s.verdict] || 0) + 1;
 	const skipped = storyResults.length - counted.length;
-	console.log(`${counted.length} stories: ${Object.entries(tally).map(([v, n]) => `${n} ${v}`).join(', ') || 'none'}${skipped ? ` (${skipped} skipped — duckdb-only, disk mode required)` : ''}`);
+	console.log(`${counted.length} stories: ${Object.entries(tally).map(([v, n]) => `${n} ${v}`).join(', ') || 'none'}${skipped ? ` (${skipped} skipped — disk mode required)` : ''}`);
+	if (warehouseAudits.length) {
+		console.log('warehouse audit:');
+		for (const audit of warehouseAudits) {
+			const summary = `stats corr=${formatNumber(audit.stats.corr)} buckets=${audit.stats.buckets} gaps=${audit.stats.gaps} empty=${audit.stats.emptyNumericCells}`;
+			console.log(`  ${audit.table}: ${audit.pass ? 'PASS' : 'FAIL'} — ${audit.failures.length ? audit.failures.join('; ') : summary}`);
+		}
+	}
 	if (coverage.note) console.log(`coverage: ${coverage.note}`);
 	else if (missing.length) console.log(`coverage: FAIL — hooks with no story: ${missing.map(n => `H${n}`).join(', ')} (declared H${coverage.declared.join(', H')})`);
 	else console.log(`coverage: ${coverage.declared.length} hooks declared in HOOK STORIES, all covered`);
@@ -243,3 +275,77 @@ if (asJson) {
 }
 
 process.exit(pass ? 0 : 1);
+
+function buildWarehouseAudits(warehouseSpecs, warehouseRows, validated, events = [], warehouseManifest = null) {
+	return Object.values(warehouseSpecs || {}).map((spec) => {
+		const rows = warehouseRows?.[spec.name] || [];
+		const sourceRows = events.length ? computeWarehouseSourceRows(events, spec) : [];
+		const manifestColumns = warehouseManifest?.tables?.find((table) => table.table === spec.name)?.columns || [];
+		return {
+			table: spec.name,
+			...auditWarehouseRows(rows, spec, {
+				datasetStart: validated.datasetStart,
+				datasetEnd: validated.datasetEnd,
+				sourceRows,
+				manifestColumns,
+			}),
+		};
+	});
+}
+
+function loadWarehouseManifest(prefixPath) {
+	const manifestPath = `${prefixPath}-WAREHOUSE-MANIFEST.json`;
+	if (!fs.existsSync(manifestPath)) return null;
+	return JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+}
+
+async function loadWarehouseRows(prefixPath, spec, manifest) {
+	const manifestEntry = manifest?.tables?.find((table) => table.table === spec.name) || null;
+	const fileBase = manifestEntry?.file || `${path.basename(prefixPath)}-WAREHOUSE-${spec.name}`;
+	const format = manifestEntry?.format || spec.format || 'csv';
+	const filePath = path.join(path.dirname(prefixPath), `${fileBase}.${format}`);
+	if (!fs.existsSync(filePath)) return [];
+	if (format === 'json') return loadNdjson(filePath);
+	return loadCsv(filePath, manifestEntry?.columns || []);
+}
+
+async function loadNdjson(filePath) {
+	const out = [];
+	const rl = readline.createInterface({ input: fs.createReadStream(filePath), crlfDelay: Infinity });
+	for await (const line of rl) {
+		if (line.trim()) out.push(JSON.parse(line));
+	}
+	return out;
+}
+
+async function loadCsv(filePath, columns) {
+	const out = [];
+	const parser = fs.createReadStream(filePath).pipe(parseCsv({
+		bom: true,
+		columns: true,
+		skip_empty_lines: true,
+	}));
+	for await (const record of parser) {
+		const row = {};
+		for (const [header, rawValue] of Object.entries(record)) {
+			const type = columns.find((column) => column.name === header)?.bqType;
+			row[header] = coerceWarehouseCell(rawValue ?? '', type);
+		}
+		out.push(row);
+	}
+	return out;
+}
+
+function coerceWarehouseCell(value, bqType) {
+	if (bqType === 'FLOAT64') {
+		if (value === '') return '';
+		const numeric = Number(value);
+		return Number.isFinite(numeric) ? numeric : value;
+	}
+	if (bqType === 'BOOL') return value === 'true';
+	return value;
+}
+
+function formatNumber(value) {
+	return Number.isFinite(value) ? value.toFixed(3) : String(value);
+}