@ak--47/dungeon-master 1.4.2 → 1.4.4

package/dungeons/vertical/sass.js

@@ -13,6 +13,7 @@ import utc from "dayjs/plugin/utc.js";
 import "dotenv/config";
 import * as u from "../../lib/utils/utils.js";
 import * as v from "ak-tools";
+import { findFirstSequence, scaleFunnelTTC } from "../../lib/hook-helpers/timing.js";
 
 dayjs.extend(utc);
 const chance = u.initChance(SEED);
@@ -52,15 +53,14 @@ const chance = u.initChance(SEED);
 
 /*
  * ═══════════════════════════════════════════════════════════════════════════════
- * ANALYTICS HOOKS (10 hooks)
+ * ANALYTICS HOOKS (11 hooks)
  * ═══════════════════════════════════════════════════════════════════════════════
  *
  * 10 deliberately architected patterns hidden in the data. NOTE: All cohort
  * effects are HIDDEN — no flag stamping. Discoverable via behavioral cohorts
  * or raw-prop breakdowns (company_size, day, doc_section). Adds:
- *   9. INCIDENT RESPONSE TIME-TO-CONVERT (Enterprise 1.4x faster vs Startup)
- *      [funnel-post: visible only in Mixpanel funnel median TTC; cross-event
- *      MIN→MIN SQL queries do NOT show this]
+ *   9. INCIDENT RESPONSE TIME-TO-CONVERT (Enterprise 0.67x gap vs Startup 1.5x)
+ *      [everything hook: scales response_time_mins and resolution_time_mins by company_size]
  *   10. DOCS MAGIC NUMBER (sweet 4-7 docs → +40% deploys; over 8+ → drop 25%)
  *
  * ─────────────────────────────────────────────────────────────────────────────
@@ -264,6 +264,120 @@ const chance = u.initChance(SEED);
  * customer segments — from a $99/mo startup to a $500K Fortune 500 contract.
  *
  * ─────────────────────────────────────────────────────────────────────────────
+ * 9. INCIDENT RESPONSE TTC (everything)
+ * ─────────────────────────────────────────────────────────────────────────────
+ *
+ * PATTERN: Enterprise companies resolve incidents faster; startups resolve
+ * slower. The hook reads company_size from the user profile and scales
+ * response_time_mins (on "alert acknowledged") and resolution_time_mins
+ * (on "alert resolved") by a factor: enterprise 0.67x, startup 1.5x,
+ * smb/mid_market unchanged. Additionally, the first incident-response
+ * funnel sequence (alert triggered → alert acknowledged → alert resolved)
+ * has its inter-step timestamps scaled via scaleFunnelTTC with the same
+ * factor, so Mixpanel funnel TTC reports reflect the gap. This compounds
+ * with Hook 4 (integration users) — an enterprise user with both Slack
+ * and PagerDuty configured gets the fastest resolution times.
+ *
+ * HOW TO FIND IT IN MIXPANEL:
+ *
+ *   Report 1: Avg Response Time by Company Size
+ *   - Report type: Insights
+ *   - Event: "alert acknowledged"
+ *   - Measure: Average of "response_time_mins"
+ *   - Breakdown: "company_size" (user property)
+ *   - Expected: enterprise ~ 0.67x startup
+ *
+ *   Report 2: Incident Funnel TTC by Company Size
+ *   - Report type: Funnels
+ *   - Steps: "alert triggered" → "alert acknowledged" → "alert resolved"
+ *   - Measure: Median time to convert
+ *   - Breakdown: "company_size"
+ *   - Expected: enterprise ~ 0.67x median TTC vs startup ~ 1.5x
+ *
+ *   Report 3: Avg Resolution Time by Company Size
+ *   - Report type: Insights
+ *   - Event: "alert resolved"
+ *   - Measure: Average of "resolution_time_mins"
+ *   - Breakdown: "company_size"
+ *   - Expected: enterprise ~ 0.67x startup
+ *
+ * REAL-WORLD ANALOGUE: Enterprise teams have dedicated SRE rotations,
+ * automated runbooks, and premium support contracts that compress
+ * incident timelines. Startups rely on smaller teams with less tooling.
+ *
+ * ─────────────────────────────────────────────────────────────────────────────
+ * 10. DOCS MAGIC NUMBER (everything)
+ * ─────────────────────────────────────────────────────────────────────────────
+ *
+ * PATTERN: Users in the 4-7 documentation-viewed sweet spot get 2-3 extra
+ * production "service deployed" events cloned into their stream (boosting
+ * deploy frequency ~40%). Users with 8+ documentation views are over-
+ * engaged browsers; 25% of their "service deployed" events are dropped.
+ * No flag is stamped — discoverable only by binning users on doc-view
+ * count and comparing per-user deploy volume.
+ *
+ * HOW TO FIND IT IN MIXPANEL:
+ *
+ *   Report 1: Deploys per User by Docs-View Bucket
+ *   - Report type: Insights (with cohort)
+ *   - Cohort A: users with 4-7 "documentation viewed" events
+ *   - Cohort B: users with 0-3 "documentation viewed" events
+ *   - Event: "service deployed"
+ *   - Measure: Total per user
+ *   - Expected: Cohort A ~ 1.4x deploys per user vs Cohort B
+ *
+ *   Report 2: Deploys per User on Heavy Doc Readers
+ *   - Report type: Insights (with cohort)
+ *   - Cohort C: users with >= 8 "documentation viewed" events
+ *   - Cohort A: users with 4-7
+ *   - Event: "service deployed"
+ *   - Measure: Total per user
+ *   - Expected: Cohort C ~ 25% fewer deploys per user vs Cohort A
+ *
+ * REAL-WORLD ANALOGUE: Engineers who read just enough docs deploy with
+ * confidence; those who read excessively may be stuck troubleshooting
+ * and never ship, or are evaluating the product without committing.
+ *
+ * ─────────────────────────────────────────────────────────────────────────────
+ * 11. DEPLOY PIPELINE EXPERIMENT (funnel experiment — engine-managed)
+ * ─────────────────────────────────────────────────────────────────────────────
+ *
+ * PATTERN: The deployment funnel (deployment pipeline run → service deployed
+ * → dashboard viewed) runs a "Canary Deploys" experiment starting 45 days
+ * before dataset end. Users are randomly assigned to Control or "Canary
+ * Deploys" variant. The Canary variant gets 1.2x conversion multiplier and
+ * 0.85x time-to-convert multiplier (faster + higher conversion). The engine
+ * emits `$experiment_started` events with `Experiment Name` and
+ * `Variant Name` properties. No hook code needed — the engine handles
+ * variant assignment and conversion/TTC scaling.
+ *
+ * HOW TO FIND IT IN MIXPANEL:
+ *
+ *   Report 1: Experiment Enrollment
+ *   - Report type: Insights
+ *   - Event: "$experiment_started"
+ *   - Measure: Total
+ *   - Breakdown: "Variant Name"
+ *   - Expected: roughly even split between "Control" and "Canary Deploys"
+ *
+ *   Report 2: Deploy Funnel by Variant
+ *   - Report type: Funnels
+ *   - Steps: "deployment pipeline run" → "service deployed" → "dashboard viewed"
+ *   - Breakdown: "Variant Name" (user property)
+ *   - Expected: Canary variant ~ 1.2x conversion vs Control
+ *
+ *   Report 3: Deploy TTC by Variant
+ *   - Report type: Funnels
+ *   - Steps: same as above
+ *   - Measure: Median time to convert
+ *   - Breakdown: "Variant Name"
+ *   - Expected: Canary variant ~ 0.85x TTC vs Control (faster)
+ *
+ * REAL-WORLD ANALOGUE: Teams A/B test canary deployment strategies.
+ * Canary deploys catch issues earlier, improving both success rate and
+ * deployment velocity.
+ *
+ * ─────────────────────────────────────────────────────────────────────────────
  * EXPECTED METRICS SUMMARY
  * ─────────────────────────────────────────────────────────────────────────────
  *
@@ -277,6 +391,12 @@ const chance = u.initChance(SEED);
  * Cost Overrun             | Scale-down after overrun | 50%       | 100%           | 2x
  * Failed Deploy Recovery   | Deploy duration (sec)    | ~500      | ~750           | 1.5x
  * Enterprise vs Startup    | ACV range                | $0-3.6K   | $50K-500K      | 100x+
+ * Incident Response TTC    | Enterprise response_time | 1x        | 0.67x          | -33%
+ * Incident Response TTC    | Startup response_time    | 1x        | 1.5x           | +50%
+ * Docs Magic Number        | sweet (4-7) deploys/user | 1x        | ~1.4x          | +40%
+ * Docs Magic Number        | over (8+) deploys/user   | 1x        | ~0.75x         | -25%
+ * Deploy Experiment        | Canary conversion        | 65%       | ~78%           | 1.2x
+ * Deploy Experiment        | Canary TTC               | 1d        | ~0.85d         | 0.85x
  *
  * ─────────────────────────────────────────────────────────────────────────────
  * ADVANCED ANALYSIS IDEAS
@@ -392,6 +512,14 @@ const config = {
 			conversionRate: 65,
 			timeToConvert: 1,
 			weight: 3,
+			experiment: {
+				name: "Canary Deploys",
+				variants: [
+					{ name: "Control" },
+					{ name: "Canary Deploys", conversionMultiplier: 1.2, ttcMultiplier: 0.85 },
+				],
+				startDaysBeforeEnd: 45,
+			},
 		},
 		{
 			// Infrastructure management
@@ -641,7 +769,9 @@ const config = {
 	/**
 	 * ARCHITECTED ANALYTICS HOOKS
 	 *
-	 * This hook function creates 8 deliberate patterns in the data:
+	 * This hook function creates 10 deliberate patterns in the data.
+	 * Hook 11 (Deploy Pipeline Experiment) is engine-managed via funnel
+	 * experiment config — no hook code needed.
 	 *
 	 * 1. END-OF-QUARTER SPIKE: Days 100-110 drive plan upgrades and team expansion
 	 * 2. CHURNED ACCOUNT SILENCING: ~10% of users go completely silent after month 1
@@ -651,6 +781,9 @@ const config = {
 	 * 6. COST OVERRUN PATTERN: Budget-exceeded users react by scaling down infrastructure
 	 * 7. FAILED DEPLOYMENT RECOVERY: Recovery deploys take 1.5x longer, tracked across calls
 	 * 8. ENTERPRISE VS STARTUP: Company size determines seat count, ACV, and health score
+	 * 9. INCIDENT RESPONSE TTC: Enterprise 0.67x faster, startup 1.5x slower incident resolution
+	 * 10. DOCS MAGIC NUMBER: Sweet 4-7 docs → extra deploys; over 8+ → drop 25% of deploys
+	 * 11. DEPLOY PIPELINE EXPERIMENT: Canary Deploys A/B test on deployment funnel (engine-managed)
 	 */
 	hook: function (record, type, meta) {
 		// (Hook 1a moved to everything hook for reliable datasetStart access)
@@ -803,25 +936,33 @@ const config = {
 			}
 
 			// HOOK 9: INCIDENT RESPONSE TTC — enterprise resolves faster,
-			// startup resolves slower. Scale the time gap between alert
-			// triggered → acknowledged → resolved for this user.
-			const segment = profile?.company_size;
+			// startup resolves slower. Scale response_time_mins on
+			// acknowledged events and resolution_time_mins on resolved
+			// events by company_size. This compounds with H4 (integration
+			// users) — realistic: enterprise + good tooling = fastest.
+			const companySegment = profile?.company_size;
 			const ttcFactor = (
-				segment === "enterprise" ? 0.5 :
-				segment === "startup" ? 1.8 :
+				companySegment === "enterprise" ? 0.67 :
+				companySegment === "startup" ? 1.5 :
 				1.0
 			);
 			if (ttcFactor !== 1.0) {
-				const alertEvents = userEvents
-					.filter(e => e.event === "alert triggered" || e.event === "alert acknowledged" || e.event === "alert resolved")
-					.sort((a, b) => a.time.localeCompare(b.time));
-				for (let i = 1; i < alertEvents.length; i++) {
-					const prev = dayjs(alertEvents[i - 1].time);
-					const curr = dayjs(alertEvents[i].time);
-					const gap = curr.diff(prev);
-					const newGap = Math.round(gap * ttcFactor);
-					alertEvents[i].time = prev.add(newGap, "milliseconds").toISOString();
-				}
+				// Timestamp shift: affects Mixpanel funnel TTC
+				const incidentSeq = findFirstSequence(
+					userEvents,
+					["alert triggered", "alert acknowledged", "alert resolved"],
+					60 * 24 * 30
+				);
+				if (incidentSeq) scaleFunnelTTC(incidentSeq, ttcFactor);
+				// Property scale: affects Insights AVG reports
+				userEvents.forEach(e => {
+					if (e.event === "alert acknowledged" && e.response_time_mins) {
+						e.response_time_mins = Math.max(1, Math.round(e.response_time_mins * ttcFactor));
+					}
+					if (e.event === "alert resolved" && e.resolution_time_mins) {
+						e.resolution_time_mins = Math.max(1, Math.round(e.resolution_time_mins * ttcFactor));
+					}
+				});
 			}
 		}
 

package/dungeons/vertical/social.js

@@ -252,6 +252,30 @@ const chance = u.initChance(SEED);
  * REAL-WORLD ANALOGUE: Moderate posters write thoughtful comments; the
  * over-prolific tend to spam and burn through audience patience.
  *
+ * -------------------------------------------------------------------------------------
+ * 10. ONBOARDING TIME-TO-CONVERT (funnel-post)
+ * -------------------------------------------------------------------------------------
+ *
+ * PATTERN: Creator and business account_type users complete the
+ * onboarding funnel (account created -> profile updated -> post created)
+ * 1.4x faster (factor 0.71). Personal account users complete it 1.25x
+ * slower (factor 1.25). The hook intercepts funnel-post arrays, computes
+ * the time gap between consecutive steps, and scales each gap by the
+ * account-type-specific factor before rewriting the step timestamps.
+ *
+ * HOW TO FIND IT IN MIXPANEL:
+ *
+ *   Report 1: Onboarding Funnel Median TTC by Account Type
+ *   - Report type: Funnels
+ *   - Steps: "account created" -> "profile updated" -> "post created"
+ *   - Measure: Median time to convert
+ *   - Breakdown: "account_type" (superProp)
+ *   - Expected: creator/business ~ 0.71x baseline; personal ~ 1.25x baseline
+ *
+ * REAL-WORLD ANALOGUE: Creators and businesses arrive with clear intent
+ * and complete profile setup faster; personal users browse casually and
+ * take longer to commit to their first post.
+ *
  * =====================================================================================
  * EXPECTED METRICS SUMMARY
  * =====================================================================================
@@ -268,6 +292,7 @@ const chance = u.initChance(SEED);
  * Weekend Surge             | Weekend vs weekday       | 1x       | ~ 1.3x      | 1.3x
  * Post-Created Magic Number | sweet comment_length     | 1x       | ~ 1.4x      | 1.4x
  * Post-Created Magic Number | over engagement/user     | 1x       | ~ 0.7x      | -30%
+ * Onboarding TTC            | median TTC by acct type  | 1x       | 0.71/1.25x  | ~ 1.8x range
  * =====================================================================================
  */
 

package/dungeons/vertical/travel.js

@@ -112,7 +112,7 @@ const destinationCities = ["New York", "London", "Paris", "Tokyo", "Barcelona",
  * ───────────────────────────────────────────────────────────────
  * 4. CANCELLATION BY BOOKING WINDOW (everything hook)
  * ───────────────────────────────────────────────────────────────
- * PATTERN: Users who book last-minute (< 7 days out) have 40% of
+ * PATTERN: Users who book last-minute (< 7 days out) have 60% of
  * their "booking cancelled" events dropped — they rarely cancel.
  * Advance bookers keep all cancellation events.
  *
@@ -192,6 +192,74 @@ const destinationCities = ["New York", "London", "Paris", "Tokyo", "Barcelona",
  * REAL-WORLD ANALOGUE: Business travelers book the same hotels
  * repeatedly, leading to faster, more confident conversions.
  *
+ * ───────────────────────────────────────────────────────────────
+ * 9. BOOKING TIME-TO-CONVERT (funnel-post hook)
+ * ───────────────────────────────────────────────────────────────
+ *
+ * PATTERN: Business travelers complete the Search-to-Book funnel
+ * 1.35x faster (time gaps scaled by 0.74). Budget and leisure-family
+ * users complete it 1.25x slower (gaps scaled by 1.25). The hook
+ * iterates over the funnel-post event array, compresses or stretches
+ * the inter-step time gaps based on the user's customer_segment from
+ * meta.profile, then rewrites each event's timestamp.
+ *
+ * HOW TO FIND IT IN MIXPANEL:
+ *
+ *   Report 1: Search-to-Book TTC by Segment
+ *   - Report type: Funnels
+ *   - Steps: "destination searched" -> "hotel viewed" -> "price compared" -> "booking completed"
+ *   - Breakdown: user property "customer_segment"
+ *   - Metric: Median time to convert
+ *   - Expected: business_traveler median TTC ~ 0.74x of budget/leisure TTC
+ *     (e.g., business ~ 24h vs budget ~ 43h)
+ *
+ *   NOTE: This effect is visible ONLY in Mixpanel funnel median TTC.
+ *   Cross-event MIN->MIN SQL queries on raw events do NOT show this
+ *   because funnel-post mutates timestamps after event generation but
+ *   before storage.
+ *
+ * REAL-WORLD ANALOGUE: Business travelers know their preferred hotel
+ * chains and corporate rates, moving from search to booking with
+ * minimal comparison. Leisure and budget travelers deliberate longer,
+ * comparing options and waiting for deals.
+ *
+ * ───────────────────────────────────────────────────────────────
+ * 10. HOTEL-VIEWED MAGIC NUMBER (everything hook)
+ * ───────────────────────────────────────────────────────────────
+ *
+ * PATTERN: Users who viewed 5-10 hotels sit in a "decisive comparison
+ * shopper" sweet spot -- all their nightly_rate values on "booking
+ * completed" events are boosted by +30% (factor 1.3), indicating they
+ * chose higher-tier rooms after deliberate comparison. Users who
+ * viewed 11+ hotels trigger analysis paralysis: 35% of their
+ * "booking completed" events are dropped entirely. No flag is
+ * stamped -- discoverable only by binning users on hotel-viewed
+ * COUNT and comparing booking revenue or conversion volume.
+ *
+ * HOW TO FIND IT IN MIXPANEL:
+ *
+ *   Report 1: Nightly Rate by Hotel-View Cohort
+ *   - Report type: Insights (with cohorts)
+ *   - Cohort A: users who did "hotel viewed" 5-10 times
+ *   - Cohort B: users who did "hotel viewed" 0-4 times
+ *   - Event: "booking completed"
+ *   - Measure: Average of "nightly_rate"
+ *   - Compare cohort A vs cohort B
+ *   - Expected: cohort A ~ 1.3x higher avg nightly_rate
+ *
+ *   Report 2: Bookings per User by Hotel-View Volume
+ *   - Report type: Insights (with cohorts)
+ *   - Cohort C: users who did "hotel viewed" 11+ times
+ *   - Cohort A: users who did "hotel viewed" 5-10 times
+ *   - Event: "booking completed"
+ *   - Measure: Total events per user
+ *   - Compare cohort C vs cohort A
+ *   - Expected: cohort C ~ 35% fewer bookings per user
+ *
+ * REAL-WORLD ANALOGUE: Travelers who compare a handful of hotels
+ * make confident, higher-value bookings; those who endlessly browse
+ * suffer decision fatigue and often abandon the search entirely.
+ *
  * ═══════════════════════════════════════════════════════════════
  * EXPECTED METRICS SUMMARY
  * ═══════════════════════════════════════════════════════════════
@@ -206,6 +274,9 @@ const destinationCities = ["New York", "London", "Paris", "Tokyo", "Barcelona",
  * Review Quality              | review_length       | 120      | 180/80  | 1.5x/0.67x
  * Business Profile            | travel_frequency    | mixed    | weekly  | —
  * Repeat Destination          | funnel conversion   | 40%      | 52%     | 1.3x
+ * Booking TTC                 | funnel median TTC   | 1x       | 0.74x   | 1.35x faster (business)
+ * Hotel-Viewed Magic Num      | sweet nightly_rate  | 1x       | 1.3x    | +30%
+ * Hotel-Viewed Magic Num      | over bookings/user  | 1x       | 0.65x   | -35%
  */
 
 /** @type {Config} */
@@ -774,10 +845,10 @@ const config = {
 				}
 			});
 
-			// Last-minute bookers rarely cancel — drop 40%.
+			// Last-minute bookers rarely cancel — drop 60%.
 			for (let i = events.length - 1; i >= 0; i--) {
 				if (events[i].event === "booking cancelled" && events[i].booking_window === "last_minute") {
-					if (chance.bool({ likelihood: 40 })) {
+					if (chance.bool({ likelihood: 60 })) {
 						events.splice(i, 1);
 					}
 				}

package/index.js

@@ -25,7 +25,7 @@ import { makeMirror } from './lib/generators/mirror.js';
 import { makeGroupProfile, makeProfile } from './lib/generators/profiles.js';
 
 // Utilities
-import { initChance, setDatasetNow, deleteFile } from './lib/utils/utils.js';
+import { initChance, setDatasetNow, setDatasetBegin, deleteFile } from './lib/utils/utils.js';
 
 // External dependencies
 import dayjs from "dayjs";
@@ -143,6 +143,7 @@ async function runDungeon(config) {
 		// dungeon configs. Without this, those factories produce values relative
 		// to process-start, which leaks wall-clock time into the output.
 		setDatasetNow(fixedNow);
+		setDatasetBegin(fixedBegin);
 
 		// Step 2: Create context with validated config (pass time constants explicitly)
 		const context = createContext(validatedConfig, null, { fixedNow, fixedBegin });

package/lib/core/config-validator.js

@@ -283,7 +283,7 @@ export function validateDungeonConfig(config) {
 		hasAnonIds = false,
 		hasSessionIds = false,
 		sessionTimeout = 30,
-		format = "csv",
+		format,
 		token = null,
 		region = "US",
 		writeToDisk = false,
@@ -592,7 +592,7 @@ export function validateDungeonConfig(config) {
 		avgDevicePerUser,
 		hasSessionIds,
 		sessionTimeout: (typeof sessionTimeout === 'number' && sessionTimeout > 0) ? sessionTimeout : 30,
-		format,
+		format: format || (typeof writeToDisk === 'string' && writeToDisk.startsWith('gs://') ? 'json' : 'csv'),
 		token,
 		region,
 		writeToDisk,

package/lib/orchestrators/mixpanel-sender.js

@@ -121,7 +121,8 @@ export async function sendToMixpanel(context) {
 	// Import group profiles
 	if (groupProfilesData && Array.isArray(groupProfilesData) && groupProfilesData.length > 0) {
 		for (const groupEntity of groupProfilesData) {
-			if (!groupEntity || groupEntity.length === 0) continue;
+			if (!groupEntity) continue;
+		if (groupEntity.length === 0 && !isBATCH_MODE) continue;
 			const groupKey = groupEntity?.groupKey;
 			log(`  Group Profiles (${groupKey})`);
 			let groupProfilesToImport = u.deepClone(groupEntity);
@@ -141,7 +142,7 @@ export async function sendToMixpanel(context) {
 	}
 
 	// Import group events
-	if (groupEventData?.length > 0) {
+	if (groupEventData?.length > 0 || (isBATCH_MODE && groupEventData)) {
 		log(`  Group Events`);
 		let groupEventDataToImport = u.deepClone(groupEventData);
 		const shouldReadFromFiles = isBATCH_MODE || (writeToDisk && groupEventData.length === 0);

package/lib/utils/utils.js

@@ -33,19 +33,29 @@ const weightedArrayCache = new Map();
 // window is resolved, so date helpers in dungeon configs produce deterministic
 // values relative to the dataset end (not the process start).
 let DATASET_NOW = dayjs.utc();
+let DATASET_BEGIN = dayjs.utc().subtract(30, 'day');
 
-/**
- * Override the reference "now" used by date()/day() factories. Called by the
- * orchestrator with the resolved dataset end so every date helper produces
- * deterministic values relative to the dataset window.
- * @param {number} unixSeconds
- */
+/** @param {number} unixSeconds */
 function setDatasetNow(unixSeconds) {
 	if (typeof unixSeconds === 'number' && Number.isFinite(unixSeconds)) {
 		DATASET_NOW = dayjs.unix(unixSeconds).utc();
 	}
 }
 
+/** @param {number} unixSeconds */
+function setDatasetBegin(unixSeconds) {
+	if (typeof unixSeconds === 'number' && Number.isFinite(unixSeconds)) {
+		DATASET_BEGIN = dayjs.unix(unixSeconds).utc();
+	}
+}
+
+
+class ListValue extends Array {
+	constructor(items) {
+		super(items.length);
+		for (let i = 0; i < items.length; i++) this[i] = items[i];
+	}
+}
 
 import { Storage as cloudStorage } from '@google-cloud/storage';
 const projectId = 'YOUR_PROJECT_ID';
@@ -198,11 +208,66 @@ function day(start, end) {
 
 };
 
+/**
+ * @param {string|number|dayjs.Dayjs} [start] - Defaults to DATASET_BEGIN.
+ * @param {string|number|dayjs.Dayjs} [end] - Defaults to DATASET_NOW.
+ * @param {string|null} [format='YYYY-MM-DDTHH:mm:ss'] - null for ISO string.
+ * @returns {() => string}
+ */
+function dateRange(start, end, format = 'YYYY-MM-DDTHH:mm:ss') {
+	return function () {
+		const chance = getChance();
+		const s = start != null ? dayjs(typeof start === 'number' ? start * 1000 : start).utc() : DATASET_BEGIN;
+		const e = end != null ? dayjs(typeof end === 'number' ? end * 1000 : end).utc() : DATASET_NOW;
+		const diffSec = e.diff(s, 'second');
+		const offsetSec = chance.integer({ min: 0, max: Math.max(0, diffSec) });
+		const result = s.add(offsetSec, 'second');
+		return format ? result.format(format) : result.toISOString();
+	};
+}
+
+/**
+ * @param {Array} pool - Values to pick from.
+ * @param {{ min?: number, max?: number }} [options]
+ * @returns {() => ListValue}
+ */
+function listOf(pool, options = {}) {
+	const { min = 1, max = 3 } = options;
+	return function () {
+		const chance = getChance();
+		const count = chance.integer({ min, max: Math.min(max, pool.length) });
+		return new ListValue(chance.pickset(pool, count));
+	};
+}
+
+/**
+ * @param {Record<string, import('../../types.js').ValueValid>} template
+ * @param {{ min?: number, max?: number }} [options]
+ * @returns {() => ListValue}
+ */
+function objectList(template, options = {}) {
+	const { min = 1, max = 5 } = options;
+	return function () {
+		const chance = getChance();
+		const count = chance.integer({ min, max });
+		const result = [];
+		for (let i = 0; i < count; i++) {
+			const obj = {};
+			for (const [key, val] of Object.entries(template)) {
+				obj[key] = choose(val);
+			}
+			result.push(obj);
+		}
+		return new ListValue(result);
+	};
+}
+
 /**
  * similar to pick
  * @param  {ValueValid} value
  */
 function choose(value) {
+	if (value instanceof ListValue) return value;
 	const chance = getChance();
 
 	// most of the time this will receive a list of strings; 
@@ -245,6 +310,7 @@ function choose(value) {
 			}
 
 			const result = value();
+			if (result instanceof ListValue) return result;
 			if (Array.isArray(result) && result.length > 10) {
 				// Cache large arrays (likely weighted arrays)
 				weightedArrayCache.set(funcString, result);
@@ -252,7 +318,9 @@ function choose(value) {
 			value = result;
 		}
 
-		if (Array.isArray(value) && value.length === 0) {
+			if (value instanceof ListValue) return value;
+
+			if (Array.isArray(value) && value.length === 0) {
 			return ""; // Return empty string if the array is empty
 		}
 
@@ -511,7 +579,16 @@ function streamJSON(filePath, data, options = {}) {
 
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
-			writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
+			const gcsStream = storage.bucket(bucket).file(file).createWriteStream();
+			gcsStream.on('finish', () => resolve(filePath));
+			gcsStream.on('error', reject);
+			if (gzip) {
+				const gzipStream = zlib.createGzip();
+				gzipStream.pipe(gcsStream);
+				writeStream = gzipStream;
+			} else {
+				writeStream = gcsStream;
+			}
 		}
 		else {
 			writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
@@ -525,9 +602,9 @@ function streamJSON(filePath, data, options = {}) {
 			writeStream.write(JSON.stringify(item) + '\n');
 		});
 		writeStream.end();
-		writeStream.on('finish', () => {
-			resolve(filePath);
-		});
+		if (!filePath?.startsWith('gs://')) {
+			writeStream.on('finish', () => resolve(filePath));
+		}
 		writeStream.on('error', reject);
 	});
 }
@@ -539,7 +616,16 @@ function streamCSV(filePath, data, options = {}) {
 
 		if (filePath?.startsWith('gs://')) {
 			const { uri, bucket, file } = parseGCSUri(filePath);
-			writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
+			const gcsStream = storage.bucket(bucket).file(file).createWriteStream();
+			gcsStream.on('finish', () => resolve(filePath));
+			gcsStream.on('error', reject);
+			if (gzip) {
+				const gzipStream = zlib.createGzip();
+				gzipStream.pipe(gcsStream);
+				writeStream = gzipStream;
+			} else {
+				writeStream = gcsStream;
+			}
 		}
 		else {
 			writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
@@ -567,9 +653,9 @@ function streamCSV(filePath, data, options = {}) {
 		});
 
 		writeStream.end();
-		writeStream.on('finish', () => {
-			resolve(filePath);
-		});
+		if (!filePath?.startsWith('gs://')) {
+			writeStream.on('finish', () => resolve(filePath));
+		}
 		writeStream.on('error', reject);
 	});
 }
@@ -642,20 +728,23 @@ async function streamParquet(filePath, data, options = {}) {
 	});
 
 	if (filePath?.startsWith('gs://')) {
-		// For GCS, write to buffer first, then upload
 		// @ts-ignore
 		const arrayBuffer = parquetWriteBuffer({ columnData });
 		const { bucket, file } = parseGCSUri(filePath);
-
-		const writeStream = storage.bucket(bucket).file(file).createWriteStream({
-			gzip: gzip || true // Always gzip for GCS
-		});
+		const gcsStream = storage.bucket(bucket).file(file).createWriteStream();
 
 		return new Promise((resolve, reject) => {
-			writeStream.write(Buffer.from(arrayBuffer));
-			writeStream.end();
-			writeStream.on('finish', () => resolve(filePath));
-			writeStream.on('error', reject);
+			gcsStream.on('finish', () => resolve(filePath));
+			gcsStream.on('error', reject);
+			if (gzip) {
+				const gzipStream = zlib.createGzip();
+				gzipStream.pipe(gcsStream);
+				gzipStream.write(Buffer.from(arrayBuffer));
+				gzipStream.end();
+			} else {
+				gcsStream.write(Buffer.from(arrayBuffer));
+				gcsStream.end();
+			}
 		});
 	} else {
 		// For local files
@@ -1660,5 +1749,10 @@ export {
 	assignSessionIds,
 	bunchIntoSessions,
 	setDatasetNow,
+	setDatasetBegin,
+	dateRange,
+	listOf,
+	objectList,
+	ListValue,
 	deleteFile,
 };