make-mp-data 2.1.0 → 2.1.2

package/index.js

@@ -188,22 +188,23 @@ async function main(config) {
 
 		// ! DATA GENERATION ENDS HERE
 
-		// Step 10:  flush lookup tables to disk (always as CSVs)
+		// Step 10: Send to Mixpanel (if token provided)
+		// IMPORTANT: Must happen BEFORE flushing to disk, because flush() clears the arrays
+		let importResults;
+		if (validatedConfig.token) {
+			importResults = await sendToMixpanel(context);
+		}
+
+		// Step 11: Flush lookup tables to disk (always as CSVs)
 		if (validatedConfig.writeToDisk) {
 			await flushLookupTablesToDisk(storage, validatedConfig);
 		}
 
-		// Step 11: Flush other storage containers to disk (if writeToDisk enabled)
+		// Step 12: Flush other storage containers to disk (if writeToDisk enabled)
 		if (validatedConfig.writeToDisk) {
 			await flushStorageToDisk(storage, validatedConfig);
 		}
 
-		// Step 12: Send to Mixpanel (if token provided)
-		let importResults;
-		if (validatedConfig.token) {
-			importResults = await sendToMixpanel(context);
-		}
-
 		// Step 13: Compile results
 		jobTimer.stop(false);
 		const { start, end, delta, human } = jobTimer.report(false);

package/lib/orchestrators/mixpanel-sender.js

@@ -28,22 +28,28 @@ export async function sendToMixpanel(context) {
         userProfilesData,
         groupEventData
     } = storage;
-    
-    const { 
-        token, 
-        region, 
+
+    const {
+        token,
+        region,
         writeToDisk = true,
         format,
         serviceAccount,
         projectId,
-        serviceSecret
+        serviceSecret,
+        verbose = false
     } = config;
-    
+
     const importResults = { events: {}, users: {}, groups: [] };
     const isBATCH_MODE = context.isBatchMode();
     const isCLI = context.isCLI();
     const NODE_ENV = process.env.NODE_ENV || "unknown";
 
+    // Create verbose-aware log function
+    const log = (message) => {
+        if (verbose) console.log(message);
+    };
+
     /** @type {import('mixpanel-import').Creds} */
     const creds = { token };
     const mpImportFormat = format === "json" ? "jsonl" : "csv";
@@ -59,13 +65,11 @@ export async function sendToMixpanel(context) {
         dryRun: false,
         abridged: false,
         fixJson: false,
-        showProgress: NODE_ENV === "dev" ? true : false,
+        showProgress: verbose,
         streamFormat: mpImportFormat,
-		workers: 35
+        workers: 35
     };
 
-    if (isCLI) commonOpts.showProgress = true;
-
     // Import events
     if (eventData || isBATCH_MODE) {
         log(`importing events to mixpanel...\n`);
@@ -224,10 +228,3 @@ export async function sendToMixpanel(context) {
     return importResults;
 }
 
-/**
- * Simple logging function
- * @param {string} message - Message to log
- */
-function log(message) {
-    console.log(message);
-}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "make-mp-data",
-	"version": "2.1.0",
+	"version": "2.1.2",
 	"description": "builds all mixpanel primitives for a given project",
 	"type": "module",
 	"main": "index.js",
@@ -63,14 +63,14 @@
 		"@google-cloud/functions-framework": "^3.4.2",
 		"@google-cloud/storage": "^7.14.0",
 		"ak-fetch": "^2.0.12",
-		"ak-gemini": "^1.0.59",
+		"ak-gemini": "^1.0.12",
 		"ak-tools": "^1.1.12",
 		"chance": "^1.1.11",
 		"dayjs": "^1.11.11",
 		"dotenv": "^16.4.5",
 		"google-auth-library": "^9.15.0",
 		"hyparquet-writer": "^0.6.1",
-		"mixpanel-import": "^3.0.0",
+		"mixpanel-import": "^3.1.43",
 		"p-limit": "^3.1.0",
 		"seedrandom": "^3.0.5",
 		"sentiment": "^5.0.2",