@ak--47/dungeon-master 1.6.1 → 1.6.3

package/lib/orchestrators/mixpanel-sender.js

@@ -10,14 +10,64 @@ import { comma, rm } from "ak-tools";
 import * as u from "../utils/utils.js";
 import mp from "mixpanel-import";
 
+export { collectWrittenFiles };
+
+/**
+ * Count of `sendToMixpanel` calls currently in flight in this process.
+ *
+ * `mp.destroy()` tears down mixpanel-import's PROCESS-GLOBAL undici pools, so a
+ * run that finishes while another is still importing would close the sockets out
+ * from under it — surfacing as `UND_ERR_CLOSED`, which mixpanel-import does not
+ * retry. Only the last run out turns off the lights.
+ */
+let _inFlightImports = 0;
+
+/**
+ * Release mixpanel-import's shared undici connection pools, if no import is still
+ * running. dungeon-master is a library — a host process that runs occasional
+ * imports shouldn't hold ingest sockets open in between. Pools are lazily
+ * re-created, so a later run is unaffected.
+ *
+ * Called automatically when the last in-flight import settles; exported so a host
+ * that drives `mixpanel-import` directly can force the same cleanup. Note the
+ * refcount only sees imports that go through `sendToMixpanel` — a host running its
+ * own `mixpanel-import` job concurrently with a dungeon run can still have its
+ * pools closed underneath it.
+ * @returns {Promise<boolean>} true if pools were torn down.
+ */
+export async function releaseConnections() {
+	// `destroy` only exists on mixpanel-import >= 3.5.1.
+	if (_inFlightImports > 0 || typeof mp.destroy !== 'function') return false;
+	try {
+		await mp.destroy();
+		return true;
+	} catch (err) {
+		// Never fail a completed import over socket cleanup.
+		log(`  !! connection pool cleanup failed: ${err.message}\n`);
+		return false;
+	}
+}
+
 /**
  * Sends the data to Mixpanel
  * @param {Context} context - Context object containing config, storage, etc.
  * @returns {Promise<Object>} Import results for all data types
  */
-export { collectWrittenFiles };
-
 export async function sendToMixpanel(context) {
+	_inFlightImports++;
+	try {
+		return await _sendToMixpanel(context);
+	} finally {
+		_inFlightImports--;
+		await releaseConnections();
+	}
+}
+
+/**
+ * @param {Context} context
+ * @returns {Promise<Object>}
+ */
+async function _sendToMixpanel(context) {
 	const { config, storage } = context;
 	const {
 		adSpendData,

package/lib/orchestrators/user-loop.js

@@ -750,6 +750,33 @@ export async function userLoop(context) {
 				});
 			}
 
+			// v1.6.3: guarantee unique `insert_id` across the user's final stream.
+			//
+			// Hooks clone events by spreading an existing one — the documented way
+			// to inject — and a spread copies `insert_id` along with everything
+			// else. Mixpanel deduplicates on `insert_id` at ingest, so those clones
+			// are accepted, reported as successful, and then silently dropped: the
+			// engineered volume never appears in the project. Nothing downstream
+			// catches it, because local verification never inspects `insert_id`.
+			//
+			// The hook-helper clone atoms stamp fresh ids themselves, but hooks are
+			// free to hand-roll a spread (and many shipped dungeons do), so the
+			// only reliable place to enforce this is here, over the finished set.
+			// A legitimate stream never contains two events with the same id, so
+			// any collision at this point is a clone that needs its own id.
+			// `> 0`, not `> 1`: a lone event cannot collide, but it can still be
+			// missing an id if a hook replaced it with a constructed object.
+			if (usersEvents.length > 0) {
+				const seenInsertIds = new Set();
+				for (const ev of usersEvents) {
+					if (!ev) continue;
+					if (!ev.insert_id || seenInsertIds.has(ev.insert_id)) {
+						ev.insert_id = randomUUID();
+					}
+					seenInsertIds.add(ev.insert_id);
+				}
+			}
+
 			// Defensive guard: drop any events whose timestamp landed past the
 			// configured dataset end. Hooks that duplicate events with time offsets
 			// (weekend surges, viral spreads) can leak a few past the boundary.

package/lib/verify/index.js

@@ -11,6 +11,12 @@
 
 export { emulateBreakdown } from './emulate-breakdown.js';
 export { verifyDungeon } from './verify-dungeon.js';
+// v1.6.2: exposed on the verify surface so a standalone verify script — one that
+// reads shards off disk instead of running the dungeon — can resolve funnel
+// defaults (`conversionWindowDays`, `order`) before handing funnels to
+// `evaluateStories` / `applyFunnelDefaults`. Use the RETURN value; it does not
+// enrich the config you pass in.
+export { validateDungeonConfig } from '../core/config-validator.js';
 export { deriveExpectedSchema, validateSchema } from './schema-validator.js';
 export {
 	evaluateFunnel,

package/lib/verify/verify-dungeon.js

@@ -59,9 +59,10 @@ function findMatchingFunnel(funnels, steps) {
  * args object — the input is not mutated. Extracted from `verifyDungeon` in
  * v1.6 so the story runner (P3.3) reuses the exact same threading.
  * @param {Object} breakdownArgs - Args destined for `emulateBreakdown`.
- * @param {Array<Object>} funnels - VALIDATED dungeon funnels (`validateDungeonConfig`
- *   mutates funnels in place, so post-run `config.funnels` carries resolved
- *   `conversionWindowDays` / `order`).
+ * @param {Array<Object>} funnels - VALIDATED dungeon funnels — i.e. the funnels off
+ *   `validateDungeonConfig`'s RETURN value (post-1.6.2 it no longer enriches the
+ *   caller's object), which carry resolved `conversionWindowDays` / `order`. A run
+ *   result exposes them as `result.validatedConfig.funnels`.
  * @param {Array<Object>} [profiles] - User profiles, threaded into `timeToConvert`.
  * @returns {Object}
  */
@@ -113,20 +114,44 @@ export function applyFunnelDefaults(breakdownArgs, funnels, profiles) {
 /**
  * @param {Object} config - Dungeon config (or path; passed straight to DUNGEON_MASTER).
  * @param {VerifyCheck[]} checks
- * @returns {Promise<{ pass: boolean, results: Array<{ name: string, pass: boolean, detail?: string, rows?: Array<Object> }>, schemaReport: Object }>}
+ * @param {Object} [overrides] - v1.6.2: merged into the dungeon before it runs, same as
+ *   `DUNGEON_MASTER`'s second argument. Lets CI verify a production-scale dungeon at a
+ *   small `numUsers` / `numEvents` without editing it.
+ * @returns {Promise<{ pass: boolean, results: Array<{ name: string, pass: boolean, detail?: string, rows?: Array<Object> }>, schemaReport: Object, validatedConfig: Object }>}
  */
-export async function verifyDungeon(config, checks) {
+export async function verifyDungeon(config, checks, overrides) {
 	if (!checks || !checks.length) throw new Error('verifyDungeon: at least one check required');
-	let result = await DUNGEON_MASTER(config);
-	if (Array.isArray(result)) result = result[0];
+	let result = await DUNGEON_MASTER(config, overrides);
+	if (Array.isArray(result)) {
+		// v1.6.2: an array input runs N dungeons but checks are written against ONE
+		// schema, and we used to silently verify `result[0]` and throw the rest away —
+		// a green report for dungeons that were never looked at. Call verifyDungeon
+		// per dungeon instead.
+		if (result.length !== 1) {
+			throw new Error(
+				`verifyDungeon: got ${result.length} dungeons, but checks apply to one. ` +
+				`Call verifyDungeon once per dungeon.`
+			);
+		}
+		result = result[0];
+	}
 	const events = Array.isArray(result.eventData) ? result.eventData : Array.from(result.eventData);
 	const profiles = Array.isArray(result.userProfilesData) ? result.userProfilesData : Array.from(result.userProfilesData);
-	const schemaReport = validateSchema(events, config);
+	// v1.6.2: schema-check against the config the run actually used. `config` may be a
+	// PATH STRING (no fields at all → every property reads as unexpected), and even for
+	// an object input a v1.5.1 dungeon keeps `hasAndroidDevices` / `hasBrowser` under
+	// `switches`, which `deriveExpectedSchema` only reads once flattened.
+	const schemaReport = validateSchema(events, result.validatedConfig || config);
 	const ctx = { events, profiles, schemaReport };
 	const results = [];
-	// `validateDungeonConfig` mutates funnels in place — by the time we read here,
-	// `conversionWindowDays` is populated and `order` is the validated value.
-	const validatedFunnels = (config && Array.isArray(config.funnels)) ? config.funnels : [];
+	// v1.6.2: read the funnels off the run's `validatedConfig` — `conversionWindowDays`
+	// and the resolved `order` live there. Before 1.6.2 we read them back off the
+	// caller's own `config`, relying on `validateDungeonConfig` enriching in place;
+	// that also meant a path/JSON input (no `.funnels` on the string) silently got an
+	// empty funnel list and thus an unbounded conversion window. Both are fixed here.
+	const validatedFunnels = Array.isArray(result?.validatedConfig?.funnels)
+		? result.validatedConfig.funnels
+		: (config && Array.isArray(config.funnels)) ? config.funnels : [];
 	for (const check of checks) {
 		try {
 			const breakdownArgs = applyFunnelDefaults(check.breakdown, validatedFunnels, profiles);
@@ -138,5 +163,7 @@ export async function verifyDungeon(config, checks) {
 		}
 	}
 	const pass = results.every(r => r.pass) && schemaReport.pass;
-	return { pass, results, schemaReport };
+	// v1.6.2: hand back the enriched config the run used, so callers can read
+	// resolved funnel/event values without re-validating.
+	return { pass, results, schemaReport, validatedConfig: result.validatedConfig };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ak--47/dungeon-master",
-	"version": "1.6.1",
+	"version": "1.6.3",
 	"description": "generate fancy datasets",
 	"type": "module",
 	"main": "index.js",
@@ -33,7 +33,7 @@
 		"HOOKS.md"
 	],
 	"engines": {
-		"node": ">=18.0.0"
+		"node": ">=20.20.0"
 	},
 	"publishConfig": {
 		"access": "public"
@@ -87,7 +87,7 @@
 		"dotenv": "^16.4.5",
 		"hyparquet-writer": "^0.6.1",
 		"mixpanel": "^0.18.0",
-		"mixpanel-import": "^3.3.2",
+		"mixpanel-import": "^3.6.1",
 		"p-limit": "^3.1.0",
 		"pino": "^9.0.0",
 		"pino-pretty": "^11.0.0",
@@ -108,4 +108,4 @@
 			"tmp/"
 		]
 	}
-}
+}

package/scripts/verify-stories.mjs

@@ -176,7 +176,8 @@ if (inMemory) {
 
 	// Funnel auto-threading reads VALIDATED funnel fields (conversionWindowDays,
 	// order). The dungeon was not run in this process, so validate the config
-	// here — validateDungeonConfig resolves those defaults in place.
+	// here and use the RETURN value — as of v1.6.2 validateDungeonConfig does not
+	// enrich the object you hand it.
 	const validated = validateDungeonConfig({ ...config, token: '' });
 	const identityMap = buildIdentityMap(profiles);
 

package/types.d.ts

@@ -1324,6 +1324,21 @@ export type Result = {
      * so `userProfilesData.length - profilesPushed` = dropped profile count.
      */
     profilesPushed?: number;
+    /**
+     * v1.6.2: the enriched config this run actually used. `validateDungeonConfig`
+     * no longer writes back to the object you passed in, so resolved values —
+     * `funnels[].conversionWindowDays`, `events[].isStrictEvent`, the resolved
+     * dataset window — must be read here rather than off your own config.
+     *
+     * READ-ONLY. Do NOT feed this back into `DUNGEON_MASTER` — validation is not
+     * idempotent (each pass appends to the funnel set, eventually producing an
+     * empty `sequence`), and the object carries engine scratch fields. Re-run the
+     * ORIGINAL config instead.
+     *
+     * Credentials (`token`, `serviceAccount`, `serviceSecret`, `projectId`,
+     * `credentials`) are stripped — a Result is a thing hosts log.
+     */
+    validatedConfig?: Dungeon;
     /** Progress callback summary. Only present when `onProgress` was provided. */
     progress?: ProgressSummary;
 };