make-mp-data 2.1.11 → 3.0.1

package/lib/utils/mixpanel.js

@@ -0,0 +1,101 @@
+/**
+ * Server-side Mixpanel wrapper for AI observability
+ *
+ * Provides a clean interface for tracking AI job events and usage metrics.
+ * Non-blocking - tracking failures are logged but don't affect application flow.
+ */
+
+import Mixpanel from 'mixpanel';
+import { serverLogger as logger } from './logger.js';
+
+const MIXPANEL_TOKEN = '7608bedf7495b09ac092209387facb52';
+const { NODE_ENV = 'unknown' } = process.env;
+
+// Initialize Mixpanel
+let mixpanel = null;
+try {
+	mixpanel = Mixpanel.init(MIXPANEL_TOKEN);
+	logger.debug('Mixpanel initialized for server-side tracking');
+} catch (error) {
+	logger.error({ err: error }, 'Failed to initialize Mixpanel');
+}
+
+/**
+ * Track an event to Mixpanel (server-side)
+ * @param {string} eventName - Event name
+ * @param {object} properties - Event properties
+ * @param {string} [userId] - Optional user ID for distinct_id
+ */
+export function track(eventName, properties = {}, userId = null) {
+	if (!mixpanel) {
+		logger.warn({ eventName }, 'Mixpanel not initialized, skipping track');
+		return;
+	}
+
+	try {
+		const trackProps = {
+			...properties,
+			distinct_id: userId || 'anonymous',
+			source: 'dm4-server',
+			$insert_id: `${eventName}-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`,
+		};
+
+		mixpanel.track(eventName, trackProps);
+	} catch (error) {
+		// Non-blocking - log but don't throw
+		logger.error({ err: error, eventName }, 'Failed to track Mixpanel event');
+	}
+}
+
+/**
+ * Track an AI job completion event with structured metadata
+ * @param {object} params - AI job parameters
+ * @param {string} params.component - AI component (schema, hooks, refine, funnels, generic)
+ * @param {string} params.prompt - The prompt sent to the AI
+ * @param {object|string} params.response - The AI response
+ * @param {number} params.duration_ms - Duration in milliseconds
+ * @param {boolean} params.success - Whether the job succeeded
+ * @param {string} [params.error] - Error message if failed
+ * @param {object} [params.usage] - Token usage from ai.getLastUsage()
+ * @param {string} [params.user_id] - User ID for distinct_id
+ */
+export function trackAIJob(params) {
+	const {
+		component,
+		prompt,
+		response,
+		duration_ms,
+		success,
+		error = null,
+		usage = null,
+		user_id = null
+	} = params;
+
+	const responseStr = typeof response === 'string' ? response : JSON.stringify(response);
+
+	const eventProps = {
+		component: `ai-${component}`,
+		prompt_preview: prompt?.substring(0, 200) || '',
+		prompt_length: prompt?.length || 0,
+		output_length: responseStr?.length || 0,
+		duration_ms,
+		duration_human: `${(duration_ms / 1000).toFixed(2)} seconds`,
+		runtime: process.env.RUNTIME_CONTEXT || 'dm4',
+		environment: NODE_ENV,
+		success,
+		error,
+		// Token usage (all fields from getLastUsage)
+		...(usage && {
+			promptTokens: usage.promptTokens,
+			responseTokens: usage.responseTokens,
+			totalTokens: usage.totalTokens,
+			attempts: usage.attempts,
+			modelVersion: usage.modelVersion,
+			requestedModel: usage.requestedModel,
+		}),
+	};
+
+	track('AI Job', eventProps, user_id);
+}
+
+export default { track, trackAIJob };

package/lib/utils/project.js

@@ -1,5 +1,6 @@
 import 'dotenv/config';
 import * as akTools from 'ak-tools';
+import { dataLogger as logger } from './logger.js';
 const { rand, makeName } = akTools;
 let { OAUTH_TOKEN = "" } = process.env;
 const { NODE_ENV = "unknown" } = process.env;
@@ -114,7 +115,7 @@ async function getUser(oauthToken = OAUTH_TOKEN) {
 		}
 	}
 	catch (err) {
-		console.error('get user err', err);
+		logger.error({ err }, 'Error getting user information');
 	}
 
 	return user;
@@ -144,7 +145,7 @@ async function addGroupKeys(groupKeyDfns = [], projectId, oauthToken = OAUTH_TOK
 			results.push(data?.results);
 		}
 		catch (err) {
-			console.error('add group keys err', err);
+			logger.error({ err, property_name }, 'Error adding group key');
 			continue loopKeys;
 		}
 

package/lib/utils/utils.js

@@ -208,10 +208,10 @@ function choose(value) {
 
 	}
 
-	// if the thing has a .next() method, call that
+	// if the thing has a .next() method, call that (e.g., generators/iterators)
 	try {
-		if (value && typeof value.next === 'function') {
-			return value.next();
+		if (value && typeof /** @type {any} */ (value).next === 'function') {
+			return /** @type {any} */ (value).next();
 		}
 	} catch (e) {
 		console.error(`Error occurred while calling next(): ${e}`);
@@ -1070,7 +1070,42 @@ function buildFileNames(config) {
 }
 
 /**
- * @param  {[string, number][]} arrayOfArrays
+ * Human-readable byte size
+ * @param {number} bytes
+ * @param {number} dp - decimal places
+ * @param {boolean} si - use SI units
+ * @returns {string}
+ */
+function bytesHuman(bytes, dp = 2, si = true) {
+	const thresh = si ? 1000 : 1024;
+	if (Math.abs(bytes) < thresh) {
+		return bytes + ' B';
+	}
+	const units = si ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
+	let u = -1;
+	const r = 10 ** dp;
+	do {
+		bytes /= thresh;
+		++u;
+	} while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1);
+	return bytes.toFixed(dp) + ' ' + units[u];
+}
+
+/**
+ * Format milliseconds as HH:MM:SS
+ * @param {number} ms - Milliseconds
+ * @returns {string} Formatted duration string
+ */
+function formatDuration(ms) {
+	const seconds = Math.floor(ms / 1000);
+	const hours = Math.floor(seconds / 3600);
+	const minutes = Math.floor((seconds % 3600) / 60);
+	const secs = seconds % 60;
+	return `${String(hours).padStart(2, '0')}:${String(minutes).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;
+}
+
+/**
+ * @param  {[string, string | number][]} arrayOfArrays
  */
 function progress(arrayOfArrays) {
 	const terminalWidth = process.stdout.columns || 120;
@@ -1503,4 +1538,6 @@ export {
 	datesBetween,
 	weighChoices,
 	wrapFunc,
+	bytesHuman,
+	formatDuration,
 };

package/package.json

@@ -1,13 +1,13 @@
 {
 	"name": "make-mp-data",
-	"version": "2.1.11",
+	"version": "3.0.1",
 	"description": "builds all mixpanel primitives for a given project",
 	"type": "module",
 	"main": "index.js",
 	"types": "types.d.ts",
 	"files": [
 		"index.js",
-		"cli.js",
+		"entry.js",
 		"types.d.ts",
 		"lib/",
 		"dungeons/",
@@ -16,23 +16,16 @@
 		"README.md"
 	],
 	"scripts": {
-		"new:dungeon": "node ./scripts/create-dungeon.mjs",
-		"start": "node ./index.js",
-		"dev": "nodemon scratch.mjs --ignore ./data/*",
-		"prune": "rm -f ./data/* && rm -f ./tmp/* && rm -f vscode-profile-*",
 		"post": "npm publish",
-		"deps": "./scripts/update-deps.sh",
 		"test": "NODE_ENV=test vitest run",
-		"coverage": "vitest run --coverage && open ./coverage/index.html",
 		"typecheck": "tsc --noEmit",
-		"typecheck:build": "tsc --noEmit --project tsconfig.build.json",
-		"typecheck:strict": "echo '⚠️  Running pedantic strict checks (many false positives expected)...' && tsc --noEmit --project tsconfig.build.json --strict",
-		"typecheck:summary": "echo '🔍 Type-checking all shipped files...' && npm run typecheck:build && echo '✅ All TypeScript checks passed!'",
-		"test:quick": "node ./tests/benchmark/quick-test.mjs",
-		"exp:soup": "node ./tests/testSoup.mjs",
-		"func:local": "functions-framework --target=entry",
-		"func:deploy": "./scripts/deploy.sh",
-		"textGen": "node ./lib/templates/textQuickTest.js"
+		"dev": "nodemon scratch.mjs --ignore ./data/*",
+		"prune": "rm -f ./data/* && rm -f ./tmp/* && rm -f vscode-profile-*",
+		"deps": "./scripts/update-deps.sh",
+		"dungeon:new": "node ./scripts/create-dungeon.mjs",
+		"dungeon:run": "node ./scripts/run-dungeon.mjs",
+		"dungeon:to-json": "node ./scripts/dungeon-to-json.mjs",
+		"dungeon:from-json": "node ./scripts/json-to-dungeon.mjs"
 	},
 	"repository": {
 		"type": "git",
@@ -60,18 +53,19 @@
 	},
 	"homepage": "https://github.com/ak--47/make-mp-data#readme",
 	"dependencies": {
-		"@google-cloud/functions-framework": "^3.4.2",
 		"@google-cloud/storage": "^7.14.0",
-		"ak-fetch": "^2.0.12",
-		"ak-gemini": "^1.0.12",
+		"ak-fetch": "^2.0.15",
+		"ak-gemini": "^1.1.13",
 		"ak-tools": "^1.1.12",
 		"chance": "^1.1.11",
 		"dayjs": "^1.11.11",
 		"dotenv": "^16.4.5",
-		"google-auth-library": "^9.15.0",
 		"hyparquet-writer": "^0.6.1",
-		"mixpanel-import": "^3.0.0",
+		"mixpanel": "^0.18.0",
+		"mixpanel-import": "^3.2.8",
 		"p-limit": "^3.1.0",
+		"pino": "^9.0.0",
+		"pino-pretty": "^11.0.0",
 		"seedrandom": "^3.0.5",
 		"sentiment": "^5.0.2",
 		"tracery-grammar": "^2.8.4",

package/types.d.ts

@@ -48,7 +48,10 @@ export interface Dungeon {
     hasAnonIds?: boolean;
     hasSessionIds?: boolean;
     alsoInferFunnels?: boolean;
+    makeChart?: boolean | string;
     singleCountry?: string;
+    strictEventCount?: boolean;
+    isUIJob?: boolean;
 
     //models
     events?: EventConfig[]; //| string[]; //can also be a array of strings
@@ -69,13 +72,15 @@ export interface Dungeon {
 
     //probabilities
     percentUsersBornInDataset?: number;
+    /** Bias toward recent birth dates for users born in dataset (0 = uniform, 1 = heavily recent). Default: 0.3 */
+    bornRecentBias?: number;
 }
 
 export type SCDProp = {
     type?: string | "user" | "company_id" | "team_id" | "department_id";
-    frequency: "day" | "week" | "month" | "year";
+    frequency?: "day" | "week" | "month" | "year";
     values: ValueValid;
-    timing: "fixed" | "fuzzy";
+    timing?: "fixed" | "fuzzy";
     max?: number;
 };
 
@@ -164,7 +169,6 @@ export interface RuntimeState {
     userCount: number;
     isBatchMode: boolean;
     verbose: boolean;
-    isCLI: boolean;
 }
 
 /**
@@ -209,7 +213,6 @@ export interface Context {
     incrementUserCount(): void;
     incrementEventCount(): void;
     isBatchMode(): boolean;
-    isCLI(): boolean;
 
     // Time helper methods
     getTimeShift(): number;
@@ -227,6 +230,7 @@ export interface EventConfig {
     isChurnEvent?: boolean;
     isSessionStartEvent?: boolean;
     relativeTimeMs?: number;
+	isStrictEvent?: boolean;
 }
 
 export interface GroupEventConfig extends EventConfig {
@@ -322,6 +326,10 @@ export interface Funnel {
 	 *
 	 */
 	experiment?: boolean;
+	/**
+	 * optional: if set, in sequential funnels, this will determine WHEN the property is bound to the rest of the events in the funnel
+	 */
+	bindPropsIndex?: number;
 }
 
 /**
@@ -424,6 +432,8 @@ export type Result = {
     operations?: number;
     eventCount?: number;
     userCount?: number;
+    groupCount?: number;
+    avgEPS?: number;
 };
 
 /**
@@ -698,7 +708,7 @@ export interface TextGenerator {
  * @param config - Configuration options for the generator
  * @returns Text generator instance
  */
-export declare function createGenerator(config?: TextGeneratorConfig): TextGenerator;
+export declare function createTextGenerator(config?: TextGeneratorConfig): TextGenerator;
 
 /**
  * Generate a batch of text items directly (standalone function)