make-mp-data 2.0.0 → 2.0.12

package/.vscode/launch.json

@@ -1,80 +0,0 @@
-{
-	// Use IntelliSense to learn about possible attributes.
-	// Hover to view descriptions of existing attributes.
-	// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-	"version": "0.2.0",
-	"configurations": [
-		{
-			"command": "npm run dev",
-			"name": "scratch",
-			"request": "launch",
-			"type": "node-terminal"
-		},
-		{
-            "type": "node",
-            "request": "launch",
-            "name": "run dungeon",
-            "runtimeExecutable": "nodemon",            
-            "program": "${workspaceFolder}/index.js",
-			"args": ["${file}"],
-            "restart": true,
-            "console": "integratedTerminal",
-            "internalConsoleOptions": "neverOpen",
-            "skipFiles": ["<node_internals>/**"],
-			"preLaunchTask": "npm: prune",
-			"runtimeArgs": [
-				"--ignore",
-				"./data"
-			]
-			
-        },
-		{
-			"type": "node",
-			"request": "launch",
-			"name": "current file",
-			"runtimeExecutable": "nodemon",
-			"program": "${file}",
-			"restart": true,
-			"console": "integratedTerminal",
-			"internalConsoleOptions": "neverOpen",
-			"env": {
-				"NODE_ENV": "dev"
-			},
-			"runtimeArgs": ["--ignore", "./data"],
-		},
-		{
-			"command": "npm run func:local",
-			"name": "cloud local",
-			"request": "launch",
-			"type": "node-terminal",
-			"env": {
-				"NODE_ENV": "dev"
-			}
-		},
-		{
-            "type": "node",
-            "request": "launch",
-            "name": "sanity",
-            "runtimeExecutable": "nodemon",
-            "runtimeArgs": ["--inspect"],
-            "program": "${workspaceFolder}/index.js",
-			"args": ["--ignore", "./data/*", "./schemas/sanity.js"],
-            "restart": true,
-            "console": "integratedTerminal",
-            "internalConsoleOptions": "neverOpen",
-            "skipFiles": ["<node_internals>/**"],
-        },
-		{
-            "type": "node-terminal",
-            "request": "launch",
-            "name": "simple",
-            "command": "node ${workspaceFolder}/index.js --simple"
-        },
-		{
-            "type": "node-terminal",
-            "request": "launch",
-            "name": "complex",
-            "command": "node ${workspaceFolder}/index.js --complex"
-        }
-	]
-}

package/.vscode/settings.json

@@ -1,69 +0,0 @@
-{
-	"cSpell.words": [
-		"adspend",
-		"AVCHD",
-		"chartjs",
-		"chupacabra",
-		"Colour",
-		"crumn",
-		"darr",
-		"datagenerator",
-		"dislikers",
-		"Dont",
-		"durtle",
-		"fonk",
-		"gameplay",
-		"garply",
-		"jackalope",
-		"linny",
-		"mbembe",
-		"megalodon",
-		"Miley",
-		"mokele",
-		"mothman",
-		"nanodegree",
-		"nessie",
-		"nesty",
-		"planthopper",
-		"PROOMPTY",
-		"psyllid",
-		"tatzelwurm",
-		"timesoup",
-		"udacity",
-		"unparse",
-		"Vlogs",
-		"weindgo",
-		"whomacity"
-	],
-	// Vitest settings
-	"vitest.enable": true,
-	"vitest.commandLine": "npx vitest",
-	"vitest.include": ["tests/**/*.test.js"],
-	"vitest.exclude": ["**/node_modules/**", "**/coverage/**", "**/data/**", "**/tmp/**"],
-	
-	// Test Explorer settings
-	"testing.automaticallyOpenPeekView": "never",
-	"testing.openTesting": "neverOpen",
-	"js/ts.implicitProjectConfig.checkJs": true,
-	
-	// ESM and import settings
-	"typescript.preferences.includePackageJsonAutoImports": "auto",
-	"typescript.suggest.autoImports": true,
-	"javascript.suggest.autoImports": true,
-	
-	// File associations
-	"files.associations": {
-		"*.js": "javascript",
-		"*.mjs": "javascript",
-		"*.cjs": "javascript"
-	},
-	
-	// Search settings
-	"search.exclude": {
-		"**/node_modules": true,
-		"**/coverage": true,
-		"**/data": true,
-		"**/tmp": true
-	},
-	"testing.automaticallyOpenTestResults": "neverOpen"
-}

package/.vscode/tasks.json

@@ -1,12 +0,0 @@
-{
-    "version": "2.0.0",
-    "tasks": [
-        {
-            "label": "npm: prune",
-            "type": "npm",
-            "script": "prune",
-            "group": "build",
-            "problemMatcher": []
-        }
-    ]
-}

package/env.yaml

@@ -1 +0,0 @@
-NODE_ENV: prod

package/scratch.mjs

@@ -1,62 +0,0 @@
-/*
-----
-TO DOs
-----
-*/
-
-//!feature: fixedTimeFunnel? if set this funnel will occur for all users at the same time ['cards charged', 'charge complete']
-//!feature: churn ... is churnFunnel, possible to return, etc
-//!feature: send SCD data to mixpanel (blocked on dev)
-//!feature: send and map lookup tables to mixpanel (also blocked on dev)
-//!bug: using --mc flag reverts to --complex for some reason
-
-
-import main from "./index.js";
-
-import simple from './dungeons/simple.js';
-import funnels from './dungeons/funnels.js';
-import foobar from './dungeons/foobar.js';
-import complex from './dungeons/complex.js';
-import adspend from './dungeons/adspend.js'
-
-import anon from './dungeons/anon.js';
-import execSync from 'child_process';
-import mirror from './dungeons/mirror.js'
-// import mds from './dungeons/modern-data-stack.js'
-import big from './dungeons/big.js'
-
-const numEvents = 1000;
-
-/** @type {main.Config} */
-const spec = {
-	...simple,
-	numEvents: 10_000,
-	batchSize: 1_000,	
-	writeToDisk: "gs://dungeon_master_4/scratch",
-	verbose: true,
-	makeChart: false,
-	hasAnonIds: true,
-	hasSessionIds: true
-	// format: "csv",
-	// numEvents,
-	// numUsers: numEvents / 100,
-	
-};
-
-
-execSync.execSync('npm run prune');
-const RESULT = await main(spec);
-const { 
-	eventData,
-	groupProfilesData,
-	lookupTableData,
-	mirrorEventData,
-	scdTableData,
-	userProfilesData,
-	importResults,
-	files,
-	adSpendData
-} = RESULT;
-
-
-debugger;

package/scripts/dana.mjs

@@ -1,137 +0,0 @@
-#!/usr/bin/env node
-import { fork } from 'child_process';
-import path from 'path';
-import dotenv from "dotenv";
-dotenv.config();
-const { NODE_ENV = "unknown" } = process.env;
-
-import main from "../index.js";
-import simple from '../dungeons/simple.js';
-import complex from '../dungeons/complex.js';
-import gaming from '../dungeons/customers/gaming.js';
-import big from '../dungeons/big.js';
-
-import Chance from 'chance';
-const chance = new Chance();
-const twentyFivePercent = () => chance.bool({ likelihood: 25 });
-
-// --- shared options ---
-let divisor = 1;
-if (NODE_ENV === "dev") divisor = 1000;
-
-const commonOptions = {
-	numEvents: 10_000_000 / divisor,
-	numUsers: 10_000_000 / 100 / divisor,
-	batchSize: 200_000,
-	hasAdSpend: false,
-	hasAnonIds: true,
-	hasSessionIds: true,
-	format: "json",
-	hasDesktopDevices: true,
-	hasIOSDevices: true,
-	hasAndroidDevices: true,
-	hasBrowser: true,
-	hasCampaigns: true,
-	verbose: true,
-	makeChart: false,
-	writeToDisk: true
-};
-
-
-
-
-const allSpecs = {
-	"simple-spec": { ...simple, ...commonOptions, name: "simple-spec" },
-	"complex-spec": { ...complex, ...commonOptions, name: "complex-spec" },
-	"meta-spec": { ...big, ...commonOptions, name: "meta-spec" },
-	"fun-spec": { ...gaming, ...commonOptions, name: "fun-spec" },
-	"bad-spec": {
-		...big, ...commonOptions, name: "bad-data-spec",
-		hook: function (record, type, meta) {
-			if (type === "event") {
-				if (twentyFivePercent()) delete record.event;
-				if (twentyFivePercent()) delete record.user_id;
-				if (twentyFivePercent()) delete record.session_id;
-				if (twentyFivePercent()) delete record.device_id;
-				if (twentyFivePercent()) delete record.time;
-				if (twentyFivePercent()) delete record.insert_id;
-			}
-			return record;
-		}
-	}
-};
-
-for (const [name, spec] of Object.entries(allSpecs)) {
-	delete spec.groupEvents;
-	delete spec.scdProps;
-	delete spec.lookupTables;
-	delete spec.groupKeys;
-	delete spec.groupProps;
-	delete spec.mirrorProps;
-	delete spec.token;
-	delete spec.secret;
-}
-
-const simulations = Object.entries(allSpecs).map(([name, spec]) => ({ name, spec }));
-
-// 1. If this process is a CHILD, just run the spec!
-if (process.env.__SIM_CHILD) {
-	const name = process.env.__SIM_NAME;
-	const spec = allSpecs[name];
-	(async () => {
-		const result = await main(spec);
-		// Send result to parent
-		if (process.send) process.send(result);
-		else console.log(JSON.stringify(result));
-		process.exit(0);
-	})();
-} else {
-	// 2. PARENT: fork a child for each simulation
-	const { promisify } = await import('util');
-	const limit = 5; // parallelism limit
-	let running = 0, idx = 0, results = [];
-
-	function runOne(sim) {
-		return new Promise((resolve, reject) => {
-			const child = fork(process.argv[1], [], {
-				env: { ...process.env, __SIM_CHILD: "1", __SIM_NAME: sim.name },
-				stdio: ['inherit', 'inherit', 'inherit', 'ipc']
-			});
-			child.on('message', (result) => resolve({ name: sim.name, result }));
-			child.on('error', reject);
-			child.on('exit', (code) => {
-				if (code !== 0) reject(new Error(`Child ${sim.name} exited code ${code}`));
-			});
-		});
-	}
-
-	const queue = [...simulations];
-	const resultsList = [];
-	async function runNext() {
-		while (running < limit && queue.length > 0) {
-			const sim = queue.shift();
-			running++;
-			runOne(sim)
-				.then(res => resultsList.push(res))
-				.catch(e => console.error("Sim error:", e))
-				.finally(() => {
-					running--;
-					runNext();
-				});
-		}
-	}
-	runNext();
-
-	// Wait until all are done
-	const wait = () => new Promise(resolve => {
-		const check = () => {
-			if (resultsList.length === simulations.length) resolve(resultsList);
-			else setTimeout(check, 100);
-		};
-		check();
-	});
-	(await wait()).forEach(res => {
-		console.log(`Finished simulation: ${res.name}`);
-		// optionally: console.log(res.result);
-	});
-}

package/scripts/deploy.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-gcloud alpha functions deploy dm4 \
-  --runtime nodejs20 \
-  --source . \
-  --gen2 \
-  --trigger-http \
-  --no-allow-unauthenticated \
-  --entry-point entry \
-  --env-vars-file env.yaml \
-  --timeout=3600 \
-  --region=us-central1 \
-  --max-instances=1000 \
-  --min-instances=0 \
-  --concurrency=1 \
-  --memory=4G

package/scripts/jsdoctest.js

@@ -1,5 +0,0 @@
-/* eslint-disable no-unused-vars */
-// @ts-check
-const module = require('../index.js');
-
-module({})

package/scripts/new-dungeon.sh

@@ -1,98 +0,0 @@
-#!/bin/bash
-
-# Ensure the ./path directory exists
-mkdir -p ./dungeons
-
-# Generate a random file name with .js extension
-random_file_name=$(mktemp ./dungeons/my-file-XXXXXXXX.js)
-
-# Initial text to write to the file
-initial_text='
-import dayjs from "dayjs";
-import utc from "dayjs/plugin/utc.js";
-import "dotenv/config";
-import * as u from "../lib/utils/utils.js";
-import * as v from "ak-tools";
-
-const SEED = "my-seed";
-dayjs.extend(utc);
-const chance = u.initChance(SEED);
-const num_users = 25_000
-const days = 100
-
-/** @typedef  {import("../types.d.ts").Dungeon} Config */
-
-/** @type {Config} */
-const config = {
-    token: "",
-    seed: SEED,
-    numDays: days, 
-    numEvents: num_users * 100, 
-    numUsers: num_users, 
-	hasAnonIds: false, 
-	hasSessionIds: false, 
-	format: "json",
-	alsoInferFunnels: true,
-	hasLocation: true,
-	hasAndroidDevices: true,
-	hasIOSDevices: true,
-	hasDesktopDevices: true,
-	hasBrowser: true,
-	hasCampaigns: true,
-	isAnonymous: false,
-	hasAdSpend: true,
-	
-	hasAvatar: true,
-	makeChart: false,
-
-	batchSize: 1_500_000,
-	concurrency: 1,
-	writeToDisk: false,
-	
-	funnels: [],
-    events: [],
-    superProps: {},
-    userProps: {},
-    scdProps: {},
-    mirrorProps: {},
-    groupKeys: [],
-    groupProps: {},
-    lookupTables: [],
-	hook: function (record, type, meta) {
-		const NOW = dayjs();
-
-		if (type === "event") {
-			const EVENT_TIME = dayjs(record.time);
-		}
-
-		if (type === "user") {
-
-		}
-
-		if (type === "funnel-post") {
-			
-		}
-
-		if (type === "funnel-pre") {
-
-		}
-
-		if (type === "scd") {
-
-		}
-
-		if (type === "everything") {
-		
-		}
-
-		return record;
-	}
-};
-
-export default config;'
-
-# Write the initial text to the new file
-echo "$initial_text" > "$random_file_name"
-
-# Output the name of the created file
-echo "File created: $random_file_name"

package/scripts/new-project.mjs

@@ -1,14 +0,0 @@
-import project from "../components/project.js";
-import dotenv from "dotenv";
-
-dotenv.config();
-const OAUTH_TOKEN = process.env.OAUTH_TOKEN || "";
-
-if (!OAUTH_TOKEN) throw new Error('No OAUTH_TOKEN in .env');
-
-// @ts-ignore
-const createdProject = await project({
-	oauth: OAUTH_TOKEN	
-})
-
-console.log(createdProject);

package/scripts/run-index.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-RUNTIME=local nodemon --inspect index.js --writeToDisk false -u 100 -e 10000

package/scripts/update-deps.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-npm i mixpanel-import@latest --save
-npm i ak-tools@latest --save
-npm i ak-fetch@latest --save
-npm i ak-gemini@latest --save

package/tests/benchmark/concurrency.mjs

@@ -1,52 +0,0 @@
-/*
-----
-TO DOs
-----
-*/
-
-//!feature: fixedTimeFunnel? if set this funnel will occur for all users at the same time ['cards charged', 'charge complete']
-//!feature: churn ... is churnFunnel, possible to return, etc
-//!feature: send SCD data to mixpanel (blocked on dev)
-//!feature: send and map lookup tables to mixpanel (also blocked on dev)
-//!bug: using --mc flag reverts to --complex for some reason
-
-
-import main from "../../index.js";
-import simple from '../../dungeons/simple.js';
-
-/** @typedef {import('../../types').Dungeon} Config */
-
-/** @type {Config} */
-const noWrites = {
-	...simple,
-	numUsers: 10_000,
-	numEvents: 250_000,
-	writeToDisk: false,
-};
-
-/** @type {Config} */
-const yesWrites = {
-	...noWrites,
-	writeToDisk: true
-};
-
-console.log('concurrency benchmarking');
-
-const concurrency = [1, 2, 3, 4, 5];
-
-const results = [];
-for (const concurrent of concurrency) {
-	console.log(`concurrency: ${concurrent}`);
-	// @ts-ignore
-	const test = await main({ ...noWrites, concurrency: concurrent });
-	results.push({ human: test.time.human, concurrency: concurrent });
-	console.log(`\t\tdone: ${test.time.human}\n\n`);
-}
-
-const display = results.map((r) => {
-	return `concurrency: ${r.concurrency} | duration: ${r.human}`;
-});
-
-console.log(display.join('\n\n'));
-
-debugger;

package/tests/cli.test.js

@@ -1,124 +0,0 @@
-//@ts-nocheck
-import generate from '../index.js';
-import 'dotenv/config';
-import { execSync } from "child_process";
-import * as u from 'ak-tools';
-import Papa from 'papaparse';
-
-import simple from '../dungeons/simple.js';
-import complex from '../dungeons/complex.js';
-import anon from '../dungeons/anon.js';
-import funnels from '../dungeons/funnels.js';
-import foobar from '../dungeons/foobar.js';
-import mirror from '../dungeons/mirror.js';
-import adspend from '../dungeons/adspend.js';
-import scd from '../dungeons/scd.js';
-
-const timeout = 600000;
-const testToken = process.env.TEST_TOKEN || "hello token!";
-
-// Use sequential execution to prevent CLI tests from interfering with each other
-// since they use execSync and create/modify files in the same directories
-describe.sequential('cli', () => {
-
-	test('sanity check', async () => {
-		console.log('SANITY TEST');
-		const run = execSync(`node ./index.js --numEvents 100 --numUsers 10`);
-		const ending = `completed successfully!`;
-		expect(run.toString().trim().endsWith(ending)).toBe(true);
-		const files = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(files.length).toBe(2);
-		const users = files.filter(a => a.includes('USERS'));
-		const events = files.filter(a => a.includes('EVENTS'));
-		expect(users.length).toBe(1);
-		expect(events.length).toBe(1);
-		const eventData = (await u.load(events[0])).trim();
-		const userProfilesData = (await u.load(users[0])).trim();
-		const parsedEvents = Papa.parse(eventData, { header: true }).data;
-		const parsedUsers = Papa.parse(userProfilesData, { header: true }).data;
-		expect(parsedEvents.length).toBeGreaterThan(10);
-		expect(parsedUsers.length).toBeGreaterThan(5);
-		expect(parsedUsers.every(u => u.distinct_id)).toBe(true);
-		expect(parsedEvents.every(e => e.event)).toBe(true);
-		expect(parsedEvents.every(e => e.time)).toBe(true);
-		expect(parsedEvents.every(e => e.insert_id)).toBe(true);
-		expect(parsedEvents.every(e => e.device_id || e.user_id)).toBe(true);
-		expect(parsedUsers.every(u => u.name)).toBe(true);
-		expect(parsedUsers.every(u => u.email)).toBe(true);
-		// expect(parsedUsers.every(u => u.created)).toBe(true);
-		expect(parsedUsers.every(u => u.avatar)).toBe(false);
-		expect(parsedEvents.every(e => validateEvent(e))).toBe(true);
-		expect(parsedUsers.every(u => validateUser(u))).toBe(true);
-	}, timeout);
-
-	test('no args', async () => {
-		console.log('BARE CLI TEST');
-		const run = execSync(`node ./index.js --numEvents 100 --numUsers 10`);
-		expect(run.toString().trim().includes('completed successfully!')).toBe(true);
-		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(2);
-
-	}, timeout);
-
-	test('--complex', async () => {
-		console.log('COMPLEX CLI TEST');
-		const run = execSync(`node ./index.js --numEvents 100 --numUsers 10 --seed "deal with it" --complex`, { stdio: "ignore" });
-		const csvs = (await u.ls('./data')).filter(a => a.includes('.json'));
-		expect(csvs.length).toBeGreaterThan(7);
-
-	}, timeout);
-
-	test('--simple', async () => {
-		console.log('SIMPLE CLI TEST');
-		const run = execSync(`node ./index.js --numEvents 100 --numUsers 10 --seed "deal with it" --simple`);
-		expect(run.toString().trim().includes('completed successfully!')).toBe(true);
-		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(2);
-
-	}, timeout);
-
-
-});
-
-beforeEach(() => {
-	clearData();
-});
-
-afterEach(() => {
-	clearData();
-});
-function clearData() {
-	try {
-		console.log('clearing...');
-		execSync(`npm run prune`);
-		console.log('...files cleared 👍');
-	}
-	catch (err) {
-		console.log('error clearing files');
-	}
-}
-
-function validateEvent(event) {
-	if (!event.event) return false;
-	if (!event.device_id && !event.user_id) return false;
-	if (!event.time) return false;
-	if (!event.insert_id) return false;
-	return true;
-}
-
-
-function validateUser(user) {
-	if (!user.distinct_id) return false;
-	if (!user.name) return false;
-	if (!user.email) return false;
-	// if (!user.created) return false;
-	return true;
-}
-
-
-function validTime(str) {
-	if (!str) return false;
-	if (str.startsWith('-')) return false;
-	if (!str.startsWith('20')) return false;
-	return true;
-}