make-mp-data 1.5.1 → 1.5.3

package/types.d.ts

@@ -12,7 +12,7 @@ declare namespace main {
   /**
    * main config object for the entire data generation
    */
-  export interface Config {
+  export interface Dungeon {
     // constants
     token?: string;
     seed?: string;
@@ -23,9 +23,13 @@ declare namespace main {
     numUsers?: number;
     format?: "csv" | "json" | string;
     region?: "US" | "EU";
-	concurrency?: number;
-	batchSize?: number;
+    concurrency?: number;
+    batchSize?: number;
 
+	serviceAccount?: string;
+	serviceSecret?: string;
+	projectId?: string;
+	
     // ids
     simulationName?: string;
     name?: string;
@@ -44,7 +48,7 @@ declare namespace main {
     verbose?: boolean;
     hasAnonIds?: boolean;
     hasSessionIds?: boolean;
-	alsoInferFunnels?: boolean;
+    alsoInferFunnels?: boolean;
     makeChart?: boolean | string;
 
     //models
@@ -52,18 +56,34 @@ declare namespace main {
     superProps?: Record<string, ValueValid>;
     funnels?: Funnel[];
     userProps?: Record<string, ValueValid>;
-    scdProps?: Record<string, ValueValid>;
+    scdProps?: Record<string, SCDProp>;
     mirrorProps?: Record<string, MirrorProps>;
     groupKeys?: [string, number][] | [string, number, string[]][]; // [key, numGroups, [events]]
     groupProps?: Record<string, Record<string, ValueValid>>;
+    groupEvents?: GroupEventConfig[];
     lookupTables?: LookupTableSchema[];
     soup?: soup;
     hook?: Hook<any>;
 
     //allow anything to be on the config
     [key: string]: any;
+
+	//probabilities
+	percentUsersBornInDataset?: number;
   }
 
+  export type complexSCDProp = {
+    type: string;
+    frequency: "day" | "week" | "month" | "year";
+    values: ValueValid;
+    timing: "fixed" | "fuzzy";
+    max?: number;
+  };
+
+  export type SimpleSCDProp = string[];
+
+  export type SCDProp = complexSCDProp | SimpleSCDProp;
+
   /**
    * the soup is a set of parameters that determine the distribution of events over time
    */
@@ -82,11 +102,14 @@ declare namespace main {
     | "group"
     | "lookup"
     | "scd"
+    | "scd-pre"
     | "mirror"
     | "funnel-pre"
     | "funnel-post"
     | "ad-spend"
     | "churn"
+    | "group-event"
+    | "everything"
     | "";
 
   /**
@@ -99,7 +122,7 @@ declare namespace main {
     type?: hookTypes;
     filename?: string;
     format?: "csv" | "json" | string;
-	concurrency?: number;
+    concurrency?: number;
     [key: string]: any;
   }
 
@@ -107,10 +130,10 @@ declare namespace main {
    * an enriched array is an array that has a hookPush method that can be used to transform-then-push items into the array
    */
   export interface HookedArray<T> extends Array<T> {
-    hookPush: (item: T | T[]) => any;
+    hookPush: (item: T | T[], ...meta) => any;
     flush: () => void;
     getWriteDir: () => string;
-	getWritePath: () => string;
+    getWritePath: () => string;
     [key: string]: any;
   }
 
@@ -133,6 +156,7 @@ declare namespace main {
     groupProfilesData?: HookedArray<GroupProfileSchema>[];
     lookupTableData?: HookedArray<LookupTableSchema>[];
     scdTableData?: HookedArray<SCDSchema>[];
+    groupEventData?: HookedArray<EventSchema>;
   }
 
   /**
@@ -147,6 +171,13 @@ declare namespace main {
     relativeTimeMs?: number;
   }
 
+  export interface GroupEventConfig extends EventConfig {
+    frequency: number; //how often the event occurs (in days)
+    group_key: string; //the key that the group is based on
+    attribute_to_user: boolean; //if true, the event also goes to a user
+    group_size: number; //the number of users in the group
+  }
+
   /**
    * the generated event data
    */
@@ -324,6 +355,6 @@ declare namespace main {
  * const gen = require('make-mp-data')
  * const dta = gen({writeToDisk: false})
  */
-declare function main(config: main.Config): Promise<main.Result>;
+declare function main(config: main.Dungeon): Promise<main.Result>;
 
 export = main;

package/schemas/foobar.js

@@ -1,125 +0,0 @@
-/**
- * This is the default configuration file for the data generator in SIMPLE mode
- * notice how the config object is structured, and see it's type definition in ./types.d.ts
- * feel free to modify this file to customize the data you generate
- * see helper functions in utils.js for more ways to generate data
- */
-
-
-
-
-const Chance = require('chance');
-const chance = new Chance();
-const dayjs = require("dayjs");
-const utc = require("dayjs/plugin/utc");
-dayjs.extend(utc);
-const { uid, comma } = require('ak-tools');
-const { pickAWinner, weighNumRange, date, integer } = require('../components/utils');
-
-const itemCategories = ["Books", "Movies", "Music", "Games", "Electronics", "Computers", "Smart Home", "Home", "Garden", "Pet", "Beauty", "Health", "Toys", "Kids", "Baby", "Handmade", "Sports", "Outdoors", "Automotive", "Industrial", "Entertainment", "Art", "Food", "Appliances", "Office", "Wedding", "Software"];
-
-const videoCategories = ["funny", "educational", "inspirational", "music", "news", "sports", "cooking", "DIY", "travel", "gaming"];
-
-/** @type {import('../types').Config} */
-const config = {
-	token: "",
-	seed: "foo bar",
-	numDays: 365, //how many days worth of data
-	numEvents: 10000000, //how many events
-	numUsers: 25000, //how many users	
-	format: 'json', //csv or json
-	region: "US",
-	hasAnonIds: true, //if true, anonymousIds are created for each user
-	hasSessionIds: false, //if true, hasSessionIds are created for each user
-
-	events: [
-		{
-			event: "foo",
-			weight: 10,
-			properties: {}
-		},
-		{
-			event: "bar",
-			weight: 9,
-			properties: {}
-		},
-		{
-			event: "baz",
-			weight: 8,
-			properties: {}
-		},
-		{
-			event: "qux",
-			weight: 7,
-			properties: {}
-		},
-		{
-			event: "garply",
-			weight: 6,
-			properties: {}
-		},
-		{
-			event: "durtle",
-			weight: 5,
-			properties: {}
-		},
-		{
-			event: "linny",
-			weight: 4,
-			properties: {}
-		},
-		{
-			event: "fonk",
-			weight: 3,
-			properties: {}
-		},
-		{
-			event: "crumn",
-			weight: 2,
-			properties: {}
-		},
-		{
-			event: "yak",
-			weight: 1,
-			properties: {}
-		}
-	],
-	superProps: {
-		color: ["red", "orange", "yellow", "green", "blue", "indigo", "violet"],
-		number: integer,
-
-	},
-	userProps: {
-		title: chance.profession.bind(chance),
-		luckyNumber: weighNumRange(42, 420),
-		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
-	},
-
-	scdProps: {},
-	mirrorProps: {},
-	lookupTables: [],
-	groupKeys: [
-		["company_id", 1000],
-		["product_id", 10000]
-	],
-	groupProps: {
-		company_id: {
-			name: chance.company.bind(chance),
-			industry: chance.pickone.bind(chance, ["tech", "retail", "finance", "healthcare", "manufacturing", "media", "entertainment", "education", "government", "nonprofit", "other"])
-		},
-		product_id: {
-			name: chance.sentence.bind(chance, { words: 3 }),
-			category: chance.pickone.bind(chance, itemCategories),
-			price: chance.floating.bind(chance, { min: 0, max: 1000, fixed: 2 }),
-			quantity: integer
-		}
-	},
-
-	hook: function (record, type, meta) {
-		return record;
-	}
-};
-
-
-
-module.exports = config;

package/schemas/session-replay.js

@@ -1,136 +0,0 @@
-
-const SEED = "my-seed";
-const dayjs = require("dayjs");
-const utc = require("dayjs/plugin/utc");
-dayjs.extend(utc);
-require("dotenv").config();
-const u = require("../components/utils");
-const v = require("ak-tools");
-const chance = u.initChance(SEED);
-const num_users = 1000;
-const days = 90;
-
-/** @type {import("../types").Config} */
-const config = {
-	token: "0135b13525b21294e9e599a483eb35b2",
-	seed: SEED,
-	numDays: days,
-	numEvents: num_users * 100,
-	numUsers: num_users,
-	hasAnonIds: false,
-	hasSessionIds: false,
-	format: "json",
-	alsoInferFunnels: false,
-	hasLocation: true,
-	hasAndroidDevices: false,
-	hasIOSDevices: false,
-	hasDesktopDevices: true,
-	hasBrowser: true,
-	hasCampaigns: true,
-	isAnonymous: false,
-	hasAdSpend: true,
-
-	hasAvatar: true,
-	makeChart: true,
-
-	batchSize: 500_000,
-	concurrency: 500,
-	writeToDisk: false,
-
-	funnels: [
-		{
-			sequence: ["view page", "watch video", "like video"],
-			conversionRate: 25,
-			timeToConvert: 1
-		},
-		{
-			sequence: ["view page", "watch video", "dislike video"],
-			conversionRate: 20,
-			timeToConvert: 1
-		},
-		{
-			sequence: ["view page", "watch video", "comment on video"],
-			conversionRate: 15,
-			timeToConvert: 1
-		},
-		{
-			sequence: ["view page", "sign up"],
-			conversionRate: 70,
-			isFirstFunnel: true,
-
-		}
-	],
-	events: [
-		{
-			"event": "view page",
-			weight: 100,
-
-		},
-		{
-			"event": "watch video",
-			weight: 150,
-			"properties": {
-				"watch time (mins)": u.weighNumRange(1, 60, .34)
-			}
-		},
-		{
-			"event": "like video",
-			weight: 50
-		},
-		{
-			"event": "dislike video",
-			weight: 40
-		},
-		{
-			"event": "comment on video",
-			weight: 30
-		},
-		{
-			"event": "new feature: AI Filters",
-			weight: 10
-		},
-		{
-			"event": "sign up",
-			weight: 10,
-			"isFirstEvent": true,
-		}
-	],
-	superProps: {},
-	userProps: {},
-	scdProps: {},
-	mirrorProps: {},
-	groupKeys: [],
-	groupProps: {},
-	lookupTables: [],
-	hook: function (record, type, meta) {
-		const NOW = dayjs();	
-		const bugStartDate = NOW.subtract(14, 'days');
-		const bugFixedDate = NOW.subtract(3, 'days');
-		const featureReleaseData = NOW.subtract(7, 'days');
-
-		if (type === "event") {
-			const EVENT_TIME = dayjs(record.time);
-			if (record.event === "new feature: AI Filters" && EVENT_TIME.isBefore(featureReleaseData)) {
-				return null;
-			}
-		}
-
-		if (type === "funnel-post") {
-			// there's a bug in Chrome where users can't complete the funnel
-			if (dayjs(record[0].time).isAfter(bugStartDate) && dayjs(record[0].time).isBefore(bugFixedDate)) {
-				if (record[0].browser === "Chrome") {
-					if (record.length > 2) {						
-						record.pop(); // remove the last event						
-					}
-				}
-			}
-			
-		}
-
-		return record;
-	}
-
-
-};
-
-module.exports = config;