@ak--47/dungeon-master 1.5.4 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +21 -12
- package/.claude/skills/create-dungeon/SKILL.md +65 -5
- package/.claude/skills/create-project/SKILL.md +1 -1
- package/.claude/skills/create-project/provision.mjs +57 -10
- package/.claude/skills/verify-dungeon/SKILL.md +56 -25
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +26 -6
- package/.claude/skills/verify-dungeon/references/report-format.md +14 -8
- package/.claude/skills/write-hooks/SKILL.md +156 -18
- package/CHANGELOG.md +183 -0
- package/HOOKS.md +566 -112
- package/README.md +28 -9
- package/docs/guides/1.3.0-upgrade-guide.md +262 -0
- package/docs/guides/1.3.1-upgrade-guide.md +328 -0
- package/docs/guides/1.4.0-upgrade-guide.md +130 -0
- package/docs/guides/1.4.1-upgrade-guide.md +125 -0
- package/docs/guides/1.4.3-upgrade-guide.md +200 -0
- package/docs/guides/1.4.4-upgrade-guide.md +95 -0
- package/docs/guides/1.5.0-upgrade-guide.md +426 -0
- package/docs/guides/1.5.1-upgrade-guide.md +178 -0
- package/docs/guides/1.5.3-upgrade-guide.md +117 -0
- package/docs/guides/1.6.0-upgrade-guide.md +204 -0
- package/dungeons/technical/stories-verify.js +152 -0
- package/dungeons/vertical/README.md +154 -0
- package/dungeons/vertical/ai-platform/ai-platform.js +1424 -0
- package/dungeons/vertical/ai-platform/ai-platform.sql +184 -0
- package/dungeons/vertical/ai-platform/ai-platform.verify.mjs +71 -0
- package/dungeons/vertical/community/community.js +1589 -0
- package/dungeons/vertical/community/community.sql +216 -0
- package/dungeons/vertical/community/community.verify.mjs +72 -0
- package/dungeons/vertical/crypto/crypto.js +1483 -0
- package/dungeons/vertical/crypto/crypto.sql +194 -0
- package/dungeons/vertical/crypto/crypto.verify.mjs +78 -0
- package/dungeons/vertical/dating/dating.js +1540 -0
- package/dungeons/vertical/dating/dating.sql +288 -0
- package/dungeons/vertical/dating/dating.verify.mjs +72 -0
- package/dungeons/vertical/devtools/devtools.js +1524 -0
- package/dungeons/vertical/devtools/devtools.sql +161 -0
- package/dungeons/vertical/devtools/devtools.verify.mjs +73 -0
- package/dungeons/vertical/ecommerce/ecommerce.js +1480 -0
- package/dungeons/vertical/ecommerce/ecommerce.sql +218 -0
- package/dungeons/vertical/ecommerce/ecommerce.verify.mjs +70 -0
- package/dungeons/vertical/education/education.js +1763 -0
- package/dungeons/vertical/education/education.sql +243 -0
- package/dungeons/vertical/education/education.verify.mjs +75 -0
- package/dungeons/vertical/fintech/fintech.js +1410 -0
- package/dungeons/vertical/fintech/fintech.sql +175 -0
- package/dungeons/vertical/fintech/fintech.verify.mjs +70 -0
- package/dungeons/vertical/fitness/fitness.js +1495 -0
- package/dungeons/vertical/fitness/fitness.sql +206 -0
- package/dungeons/vertical/fitness/fitness.verify.mjs +71 -0
- package/dungeons/vertical/food-delivery/food-delivery.js +1641 -0
- package/dungeons/vertical/food-delivery/food-delivery.sql +183 -0
- package/dungeons/vertical/food-delivery/food-delivery.verify.mjs +75 -0
- package/dungeons/vertical/gaming/gaming.js +1741 -0
- package/dungeons/vertical/gaming/gaming.sql +249 -0
- package/dungeons/vertical/gaming/gaming.verify.mjs +70 -0
- package/dungeons/vertical/healthcare/healthcare.js +1488 -0
- package/dungeons/vertical/healthcare/healthcare.sql +193 -0
- package/dungeons/vertical/healthcare/healthcare.verify.mjs +72 -0
- package/dungeons/vertical/insurance-application/insurance-application.js +1852 -0
- package/dungeons/vertical/insurance-application/insurance-application.sql +174 -0
- package/dungeons/vertical/insurance-application/insurance-application.verify.mjs +74 -0
- package/dungeons/vertical/logistics/logistics.js +1541 -0
- package/dungeons/vertical/logistics/logistics.sql +205 -0
- package/dungeons/vertical/logistics/logistics.verify.mjs +73 -0
- package/dungeons/vertical/marketplace/marketplace.js +1438 -0
- package/dungeons/vertical/marketplace/marketplace.sql +148 -0
- package/dungeons/vertical/marketplace/marketplace.verify.mjs +74 -0
- package/dungeons/vertical/{media.js → media/media.js} +557 -58
- package/dungeons/vertical/media/media.sql +173 -0
- package/dungeons/vertical/media/media.verify.mjs +74 -0
- package/dungeons/vertical/real-estate/real-estate.js +1629 -0
- package/dungeons/vertical/real-estate/real-estate.sql +167 -0
- package/dungeons/vertical/real-estate/real-estate.verify.mjs +75 -0
- package/dungeons/vertical/sass/sass.js +1733 -0
- package/dungeons/vertical/sass/sass.sql +173 -0
- package/dungeons/vertical/sass/sass.verify.mjs +74 -0
- package/dungeons/vertical/social/social.js +1456 -0
- package/dungeons/vertical/social/social.sql +160 -0
- package/dungeons/vertical/social/social.verify.mjs +76 -0
- package/dungeons/vertical/streaming/streaming.js +677 -0
- package/dungeons/vertical/streaming/streaming.sql +101 -0
- package/dungeons/vertical/streaming/streaming.verify.mjs +75 -0
- package/dungeons/vertical/support-desk/support-desk.js +741 -0
- package/dungeons/vertical/support-desk/support-desk.sql +137 -0
- package/dungeons/vertical/support-desk/support-desk.verify.mjs +75 -0
- package/dungeons/vertical/travel/travel.js +1603 -0
- package/dungeons/vertical/travel/travel.sql +236 -0
- package/dungeons/vertical/travel/travel.verify.mjs +72 -0
- package/index.js +2 -2
- package/lib/core/config-validator.js +37 -2
- package/lib/core/context.js +4 -2
- package/lib/core/dungeon-loader.js +20 -15
- package/lib/core/dungeon-to-json.js +22 -4
- package/lib/core/storage.js +3 -3
- package/lib/generators/funnels.js +36 -9
- package/lib/hook-helpers/cohort.js +53 -0
- package/lib/hook-helpers/index.js +8 -0
- package/lib/hook-helpers/mutate.js +1 -1
- package/lib/hook-helpers/shape.js +319 -0
- package/lib/hook-patterns/aggregate-per-user-by-bin.js +13 -5
- package/lib/hook-patterns/attributed-by-source.js +60 -58
- package/lib/hook-patterns/frequency-by-frequency.js +14 -4
- package/lib/hook-patterns/funnel-frequency-breakdown.js +24 -12
- package/lib/hook-patterns/index.js +1 -1
- package/lib/hook-patterns/time-to-convert-by-segment.js +75 -13
- package/lib/orchestrators/user-loop.js +41 -0
- package/lib/templates/story-spec.schema.json +175 -0
- package/lib/utils/json-evaluator.js +1 -1
- package/lib/utils/utils.js +5 -4
- package/lib/verify/coerce.js +186 -0
- package/lib/verify/counting.js +270 -32
- package/lib/verify/emulate-breakdown.js +1285 -203
- package/lib/verify/first-time.js +89 -0
- package/lib/verify/flows.js +679 -0
- package/lib/verify/formula.js +259 -0
- package/lib/verify/funnel-engine.js +697 -113
- package/lib/verify/index.js +32 -4
- package/lib/verify/sessionize.js +275 -0
- package/lib/verify/story-runner.js +483 -0
- package/lib/verify/verify-dungeon.js +59 -37
- package/package.json +2 -1
- package/scripts/extract-dungeon-schema.mjs +2 -2
- package/scripts/run-many.mjs +2 -2
- package/scripts/verify-runner.mjs +2 -2
- package/scripts/verify-stories.mjs +244 -0
- package/types.d.ts +296 -30
- package/dungeons/vertical/ai-platform.js +0 -818
- package/dungeons/vertical/community.js +0 -966
- package/dungeons/vertical/crypto.js +0 -895
- package/dungeons/vertical/dating.js +0 -830
- package/dungeons/vertical/devtools.js +0 -1050
- package/dungeons/vertical/ecommerce.js +0 -811
- package/dungeons/vertical/education.js +0 -964
- package/dungeons/vertical/fintech.js +0 -946
- package/dungeons/vertical/fitness.js +0 -935
- package/dungeons/vertical/food-delivery.js +0 -934
- package/dungeons/vertical/gaming.js +0 -1120
- package/dungeons/vertical/healthcare.js +0 -945
- package/dungeons/vertical/insurance-application.js +0 -1039
- package/dungeons/vertical/logistics.js +0 -986
- package/dungeons/vertical/marketplace.js +0 -1019
- package/dungeons/vertical/real-estate.js +0 -872
- package/dungeons/vertical/sass.js +0 -995
- package/dungeons/vertical/social.js +0 -904
- package/dungeons/vertical/travel.js +0 -930
|
@@ -1,818 +0,0 @@
|
|
|
1
|
-
// ── IMPORTS ──
|
|
2
|
-
import dayjs from "dayjs";
|
|
3
|
-
import utc from "dayjs/plugin/utc.js";
|
|
4
|
-
dayjs.extend(utc);
|
|
5
|
-
import "dotenv/config";
|
|
6
|
-
import * as u from "../../lib/utils/utils.js";
|
|
7
|
-
/** @typedef {import("../../types").Dungeon} Config */
|
|
8
|
-
|
|
9
|
-
// ── OVERVIEW ──
|
|
10
|
-
/*
|
|
11
|
-
* NAME: PromptForge
|
|
12
|
-
* APP: LLM API platform (Anthropic/OpenAI-style). Customers send API
|
|
13
|
-
* requests for chat completions, embeddings, evaluations, and tool
|
|
14
|
-
* use. Billing is per input/output token. Features: prompt caching,
|
|
15
|
-
* tool use, multi-turn conversations, batch API, model selection,
|
|
16
|
-
* eval pipelines.
|
|
17
|
-
* SCALE: 10,000 users, ~800K events, 121 days (2026-01-01 → 2026-05-01)
|
|
18
|
-
* CORE LOOP: organization created → api key created → api call → iterate
|
|
19
|
-
*
|
|
20
|
-
* EVENTS (18):
|
|
21
|
-
* api call (10) > dashboard viewed (5) > tool use call (4) > docs searched (4)
|
|
22
|
-
* > playground session (4) > eval job (3) > eval result (3) > rate limit error (3)
|
|
23
|
-
* > model selected (3) > api key created (2) > batch job submitted (2)
|
|
24
|
-
* > batch job completed (2) > billing payment (2) > member invited (2)
|
|
25
|
-
* > organization created (1) > api key rotated (1) > webhook configured (1)
|
|
26
|
-
* > account deactivated (1)
|
|
27
|
-
*
|
|
28
|
-
* FUNNELS (3):
|
|
29
|
-
* - Onboarding: organization created → api key created → api call (70%)
|
|
30
|
-
* - API to Eval Pipeline: api call → tool use call → eval job (45%)
|
|
31
|
-
* - Usage to Billing: api call → billing payment (30%)
|
|
32
|
-
*
|
|
33
|
-
* USER PROPS: api_tier, primary_use_case, sdk_language, monthly_spend, total_api_calls, preferred_model
|
|
34
|
-
* SUPER PROPS: api_tier, primary_use_case, sdk_language
|
|
35
|
-
* SCD PROPS: monthly_api_usage (weekly fuzzy, max 20), api_tier_history (Free/Build/Enterprise, monthly fixed, max 6)
|
|
36
|
-
* GROUPS: none
|
|
37
|
-
*/
|
|
38
|
-
|
|
39
|
-
// ── HOOK STORIES ──
|
|
40
|
-
/*
|
|
41
|
-
* ---------------------------------------------------------------
|
|
42
|
-
* 1. PROMPT CACHING ADOPTION (CONVERSION — everything)
|
|
43
|
-
* ---------------------------------------------------------------
|
|
44
|
-
*
|
|
45
|
-
* PATTERN: Customers who enable prompt caching see 70% lower
|
|
46
|
-
* cost_per_call. Once any api call has cache_enabled=true, all
|
|
47
|
-
* subsequent calls for that user get cost_usd reduced by 70%.
|
|
48
|
-
*
|
|
49
|
-
* HOW TO FIND IT IN MIXPANEL:
|
|
50
|
-
*
|
|
51
|
-
* Report 1: Cost Per Call by Cache Status
|
|
52
|
-
* - Report type: Insights
|
|
53
|
-
* - Event: "api call"
|
|
54
|
-
* - Measure: Average of "cost_usd"
|
|
55
|
-
* - Breakdown: "cache_enabled"
|
|
56
|
-
* - Expected: cache_enabled=true ~ $0.003, false ~ $0.01 (70% cheaper)
|
|
57
|
-
*
|
|
58
|
-
* Report 2: Cache Adoption Over Time
|
|
59
|
-
* - Report type: Insights
|
|
60
|
-
* - Event: "api call"
|
|
61
|
-
* - Measure: Total
|
|
62
|
-
* - Filter: cache_enabled = true
|
|
63
|
-
* - Line chart by week
|
|
64
|
-
* - Expected: steady growth in cached calls over the dataset
|
|
65
|
-
*
|
|
66
|
-
* REAL-WORLD ANALOGUE: Prompt caching avoids re-processing long
|
|
67
|
-
* system prompts on every call, dramatically reducing cost and latency.
|
|
68
|
-
*
|
|
69
|
-
* ---------------------------------------------------------------
|
|
70
|
-
* 2. MODEL MIGRATION WAVE (TIMED RELEASE — event)
|
|
71
|
-
* ---------------------------------------------------------------
|
|
72
|
-
*
|
|
73
|
-
* PATTERN: At day 60, new model "opus-4-7" releases. After day 60,
|
|
74
|
-
* 35% of api calls from Build/Enterprise users switch model to
|
|
75
|
-
* "opus-4-7". These calls use 1.5x tokens (smarter model, longer
|
|
76
|
-
* responses).
|
|
77
|
-
*
|
|
78
|
-
* HOW TO FIND IT IN MIXPANEL:
|
|
79
|
-
*
|
|
80
|
-
* Report 1: Model Distribution Over Time
|
|
81
|
-
* - Report type: Insights
|
|
82
|
-
* - Event: "api call"
|
|
83
|
-
* - Measure: Total
|
|
84
|
-
* - Breakdown: "model"
|
|
85
|
-
* - Line chart by week
|
|
86
|
-
* - Expected: opus-4-7 appears at day 60, ramps to ~35% of paid calls
|
|
87
|
-
*
|
|
88
|
-
* Report 2: Tokens Per Model
|
|
89
|
-
* - Report type: Insights
|
|
90
|
-
* - Event: "api call"
|
|
91
|
-
* - Measure: Average of "tokens_used"
|
|
92
|
-
* - Breakdown: "model"
|
|
93
|
-
* - Expected: opus-4-7 ~ 1.5x tokens vs other models
|
|
94
|
-
*
|
|
95
|
-
* REAL-WORLD ANALOGUE: New flagship model launches cause migration
|
|
96
|
-
* waves among power users who want improved capabilities.
|
|
97
|
-
*
|
|
98
|
-
* ---------------------------------------------------------------
|
|
99
|
-
* 3. AGENTIC LOOP POWER USERS (everything)
|
|
100
|
-
* ---------------------------------------------------------------
|
|
101
|
-
*
|
|
102
|
-
* PATTERN: Users with 3+ "tool use call" AND 3+ api-call events with
|
|
103
|
-
* multi_turn=true get 8x tokens_used on api calls plus 2 extra cloned
|
|
104
|
-
* api-call events per existing (3x rate). Cloned events with unique
|
|
105
|
-
* offset timestamps. No flag — discover via cohort builder.
|
|
106
|
-
*
|
|
107
|
-
* HOW TO FIND IT IN MIXPANEL:
|
|
108
|
-
*
|
|
109
|
-
* Report 1: Tokens per User — Agentic Cohort
|
|
110
|
-
* - Report type: Insights (with cohort)
|
|
111
|
-
* - Cohort A: users with >= 3 "tool use call" AND >= 3 api-call with multi_turn=true
|
|
112
|
-
* - Cohort B: rest
|
|
113
|
-
* - Event: "api call"
|
|
114
|
-
* - Measure: Average of "tokens_used"
|
|
115
|
-
* - Expected: A ~ 8x B
|
|
116
|
-
*
|
|
117
|
-
* REAL-WORLD ANALOGUE: Agentic workloads consume dramatically more
|
|
118
|
-
* tokens via extended tool-use loops.
|
|
119
|
-
*
|
|
120
|
-
* ---------------------------------------------------------------
|
|
121
|
-
* 4. RATE LIMIT CHURN (everything)
|
|
122
|
-
* ---------------------------------------------------------------
|
|
123
|
-
*
|
|
124
|
-
* PATTERN: Users with >= 2 "rate limit error" events in first 7 days
|
|
125
|
-
* lose 60% of events after week 1. No flag — discover via cohort.
|
|
126
|
-
*
|
|
127
|
-
* HOW TO FIND IT IN MIXPANEL:
|
|
128
|
-
*
|
|
129
|
-
* Report 1: Retention by Early Rate-Limit Cohort
|
|
130
|
-
* - Report type: Retention
|
|
131
|
-
* - Cohort A: users with >= 2 "rate limit error" in first 7 days
|
|
132
|
-
* - Cohort B: rest
|
|
133
|
-
* - Expected: A retention ~ 40% vs B ~ 80%
|
|
134
|
-
*
|
|
135
|
-
* REAL-WORLD ANALOGUE: Developers who get rate-limited early often
|
|
136
|
-
* switch to a competitor.
|
|
137
|
-
*
|
|
138
|
-
* ---------------------------------------------------------------
|
|
139
|
-
* 5. TIER-BASED CONTEXT WINDOW (SUBSCRIPTION TIER — everything)
|
|
140
|
-
* ---------------------------------------------------------------
|
|
141
|
-
*
|
|
142
|
-
* PATTERN: Free users have context_window=200000, Build=1000000,
|
|
143
|
-
* Enterprise=2000000. Enterprise users send 4x larger input_tokens.
|
|
144
|
-
* Context window and input tokens are scaled by tier.
|
|
145
|
-
*
|
|
146
|
-
* HOW TO FIND IT IN MIXPANEL:
|
|
147
|
-
*
|
|
148
|
-
* Report 1: Input Tokens by Tier
|
|
149
|
-
* - Report type: Insights
|
|
150
|
-
* - Event: "api call"
|
|
151
|
-
* - Measure: Average of "input_tokens"
|
|
152
|
-
* - Breakdown: "api_tier" (superProp)
|
|
153
|
-
* - Expected: Enterprise ~ 4x Free (Enterprise ~ 8K, Free ~ 2K)
|
|
154
|
-
*
|
|
155
|
-
* Report 2: Context Window by Tier
|
|
156
|
-
* - Report type: Insights
|
|
157
|
-
* - Event: "api call"
|
|
158
|
-
* - Measure: Average of "context_window"
|
|
159
|
-
* - Breakdown: "api_tier"
|
|
160
|
-
* - Expected: Free=200K, Build=1M, Enterprise=2M
|
|
161
|
-
*
|
|
162
|
-
* REAL-WORLD ANALOGUE: Enterprise customers pay for larger context
|
|
163
|
-
* windows and use them for long-document analysis and code review.
|
|
164
|
-
*
|
|
165
|
-
* ---------------------------------------------------------------
|
|
166
|
-
* 6. OUTAGE DAY (TIME-BASED — event)
|
|
167
|
-
* ---------------------------------------------------------------
|
|
168
|
-
*
|
|
169
|
-
* PATTERN: Days 40-41, is_error is set to true on 40% of api call
|
|
170
|
-
* events. error_type is set to service errors. Simulates a major
|
|
171
|
-
* platform outage.
|
|
172
|
-
*
|
|
173
|
-
* HOW TO FIND IT IN MIXPANEL:
|
|
174
|
-
*
|
|
175
|
-
* Report 1: Error Rate Over Time
|
|
176
|
-
* - Report type: Insights
|
|
177
|
-
* - Event: "api call"
|
|
178
|
-
* - Measure: Total
|
|
179
|
-
* - Filter: is_error = true
|
|
180
|
-
* - Line chart by day
|
|
181
|
-
* - Expected: massive spike on days 40-41 (8x baseline error rate)
|
|
182
|
-
*
|
|
183
|
-
* Report 2: Error Types During Outage
|
|
184
|
-
* - Report type: Insights
|
|
185
|
-
* - Event: "api call"
|
|
186
|
-
* - Filter: is_error = true
|
|
187
|
-
* - Breakdown: "error_type"
|
|
188
|
-
* - Date range: days 40-41
|
|
189
|
-
* - Expected: service_overloaded and internal_server_error dominate
|
|
190
|
-
*
|
|
191
|
-
* REAL-WORLD ANALOGUE: API platforms experience periodic outages
|
|
192
|
-
* that spike error rates across all customers.
|
|
193
|
-
*
|
|
194
|
-
* ---------------------------------------------------------------
|
|
195
|
-
* 7. BATCH API DISCOUNT (everything)
|
|
196
|
-
* ---------------------------------------------------------------
|
|
197
|
-
*
|
|
198
|
-
* PATTERN: Users with any "batch job submitted" event get 50% lower
|
|
199
|
-
* cost_per_token on api calls + 2x tokens_used. Mutates raw props.
|
|
200
|
-
* No flag — discover via cohort builder.
|
|
201
|
-
*
|
|
202
|
-
* HOW TO FIND IT IN MIXPANEL:
|
|
203
|
-
*
|
|
204
|
-
* Report 1: Cost per Token by Batch Cohort
|
|
205
|
-
* - Report type: Insights (with cohort)
|
|
206
|
-
* - Cohort A: users with >= 1 "batch job submitted"
|
|
207
|
-
* - Cohort B: rest
|
|
208
|
-
* - Event: "api call"
|
|
209
|
-
* - Measure: Average of "cost_per_token"
|
|
210
|
-
* - Expected: A ~ 0.5x B
|
|
211
|
-
*
|
|
212
|
-
* REAL-WORLD ANALOGUE: Batch API pricing rewards high-volume workloads.
|
|
213
|
-
*
|
|
214
|
-
* ---------------------------------------------------------------
|
|
215
|
-
* 8. EVAL-DRIVEN RETENTION (everything)
|
|
216
|
-
* ---------------------------------------------------------------
|
|
217
|
-
*
|
|
218
|
-
* PATTERN: Users with any "eval job" in first 7 days keep all events.
|
|
219
|
-
* Non-eval users lose 75% of post-day-30 events. No flag — discover
|
|
220
|
-
* via retention cohort.
|
|
221
|
-
*
|
|
222
|
-
* HOW TO FIND IT IN MIXPANEL:
|
|
223
|
-
*
|
|
224
|
-
* Report 1: Retention by Early Eval Cohort
|
|
225
|
-
* - Report type: Retention
|
|
226
|
-
* - Cohort A: users with >= 1 "eval job" in first 7 days
|
|
227
|
-
* - Cohort B: rest
|
|
228
|
-
* - Expected: A ~ 75% D30 vs B ~ 25%
|
|
229
|
-
*
|
|
230
|
-
* REAL-WORLD ANALOGUE: Teams that set up eval pipelines stick around.
|
|
231
|
-
*
|
|
232
|
-
* ---------------------------------------------------------------
|
|
233
|
-
* 9. API-TO-EVAL TIME-TO-CONVERT (funnel-post)
|
|
234
|
-
* ---------------------------------------------------------------
|
|
235
|
-
*
|
|
236
|
-
* PATTERN: Enterprise users complete the "API to Eval Pipeline" funnel
|
|
237
|
-
* 1.5x faster than baseline (factor 0.67 on inter-event gaps); Free
|
|
238
|
-
* users 1.4x slower (factor 1.4). Mutates funnel event timestamps.
|
|
239
|
-
*
|
|
240
|
-
* HOW TO FIND IT IN MIXPANEL:
|
|
241
|
-
*
|
|
242
|
-
* Report 1: API to Eval — Median Time-to-Convert by Tier
|
|
243
|
-
* - Report type: Funnels
|
|
244
|
-
* - Steps: "api call" -> "tool use call" -> "eval job"
|
|
245
|
-
* - Measure: Median time to convert
|
|
246
|
-
* - Breakdown: "api_tier"
|
|
247
|
-
* - Expected: Enterprise ~ 0.67x Build; Free ~ 1.4x Build
|
|
248
|
-
*
|
|
249
|
-
* NOTE (funnel-post measurement): visible only via Mixpanel funnel
|
|
250
|
-
* median TTC. Cross-event MIN→MIN SQL queries on raw events do NOT
|
|
251
|
-
* show this — funnel-post adjusts gaps within funnel instances, not
|
|
252
|
-
* across the user's full event history.
|
|
253
|
-
*
|
|
254
|
-
* REAL-WORLD ANALOGUE: Enterprise teams have dedicated platform engineers
|
|
255
|
-
* who execute end-to-end pipelines faster.
|
|
256
|
-
*
|
|
257
|
-
* ---------------------------------------------------------------
|
|
258
|
-
* 10. DOCS-SEARCHED MAGIC NUMBER (in-funnel, everything)
|
|
259
|
-
* ---------------------------------------------------------------
|
|
260
|
-
*
|
|
261
|
-
* PATTERN: Count "docs searched" events between organization-created and
|
|
262
|
-
* first billing-payment. Sweet 2-4 → +35% on amount_usd of billing
|
|
263
|
-
* payment events. Over 5+ → drop 30% of billing payment events. No flag.
|
|
264
|
-
*
|
|
265
|
-
* HOW TO FIND IT IN MIXPANEL:
|
|
266
|
-
*
|
|
267
|
-
* Report 1: Avg Billing Amount by Docs-Searched Bucket
|
|
268
|
-
* - Report type: Insights (with cohort)
|
|
269
|
-
* - Cohort A: users with 2-4 "docs searched" between sign-up and first billing
|
|
270
|
-
* - Cohort B: users with 0-1
|
|
271
|
-
* - Event: "billing payment"
|
|
272
|
-
* - Measure: Average of "amount_usd"
|
|
273
|
-
* - Expected: A ~ 1.35x B
|
|
274
|
-
*
|
|
275
|
-
* Report 2: Billing Payments per User on Heavy Searchers
|
|
276
|
-
* - Report type: Insights (with cohort)
|
|
277
|
-
* - Cohort C: users with >= 5 "docs searched" between sign-up and billing
|
|
278
|
-
* - Cohort A: users with 2-4
|
|
279
|
-
* - Event: "billing payment"
|
|
280
|
-
* - Measure: Total per user
|
|
281
|
-
* - Expected: C ~ 30% fewer billing payments per user
|
|
282
|
-
*
|
|
283
|
-
* REAL-WORLD ANALOGUE: Reading the docs lifts willingness to pay; doc
|
|
284
|
-
* obsession signals stuck on integration and never paying.
|
|
285
|
-
*
|
|
286
|
-
* ===============================================================
|
|
287
|
-
* EXPECTED METRICS SUMMARY
|
|
288
|
-
* ===============================================================
|
|
289
|
-
*
|
|
290
|
-
* Hook | Metric | Baseline | Effect | Ratio
|
|
291
|
-
* ----------------------------|----------------------|------------|--------------|------
|
|
292
|
-
* Prompt Caching Adoption | cost_usd | $0.01 | $0.003 | 0.3x
|
|
293
|
-
* Model Migration Wave | opus-4-7 share paid | 0% | ~ 35% | new
|
|
294
|
-
* Agentic Loop Power Users | tokens/api-call | 1x | 8x | 8x
|
|
295
|
-
* Rate Limit Churn | D30 retention | 80% | 40% | 0.5x
|
|
296
|
-
* Tier-Based Context Window | input_tokens | 2K (Free) | 8K (Ent) | 4x
|
|
297
|
-
* Outage Day | error rate days 40-41| 5% | 40% | 8x
|
|
298
|
-
* Batch API Discount | cost_per_token | 1x | 0.5x | -50%
|
|
299
|
-
* Eval-Driven Retention | D30 retention | 25% | 75% | 3x
|
|
300
|
-
* API-to-Eval T2C | median min by tier | 1x (Build) | 0.67x / 1.4x | 1.5x range
|
|
301
|
-
* Docs Magic Number | sweet billing amount | 1x | 1.35x | 1.35x
|
|
302
|
-
* Docs Magic Number | over billing/user | 1x | 0.7x | -30%
|
|
303
|
-
*/
|
|
304
|
-
|
|
305
|
-
// ── SCALE ──
|
|
306
|
-
const SEED = "promptforge";
|
|
307
|
-
const NUM_USERS = 10_000;
|
|
308
|
-
const DATASET_START = "2026-01-01T00:00:00Z";
|
|
309
|
-
const DATASET_END = "2026-05-01T23:59:59Z";
|
|
310
|
-
const EVENTS_PER_DAY = 0.83;
|
|
311
|
-
const token = process.env.MP_TOKEN || "your-mixpanel-token";
|
|
312
|
-
|
|
313
|
-
const chance = u.initChance(SEED);
|
|
314
|
-
|
|
315
|
-
// ── KNOBS (tweak these to reshape stories) ──
|
|
316
|
-
const OUTAGE_START_DAY = 40;
|
|
317
|
-
const OUTAGE_END_DAY = 42;
|
|
318
|
-
const OUTAGE_ERROR_LIKELIHOOD = 40;
|
|
319
|
-
const OUTAGE_LATENCY_MULT = 3;
|
|
320
|
-
|
|
321
|
-
const TIER_CONTEXT_WINDOW = { Free: 200000, Build: 1000000, Enterprise: 2000000 };
|
|
322
|
-
const TIER_INPUT_MULT = { Free: 1, Build: 2, Enterprise: 4 };
|
|
323
|
-
|
|
324
|
-
const CACHE_USER_HASH_MOD = 4;
|
|
325
|
-
const CACHE_COST_FACTOR = 0.3;
|
|
326
|
-
const CACHE_ACTIVATION_PCT = 0.2;
|
|
327
|
-
|
|
328
|
-
const MODEL_MIGRATION_DAY = 60;
|
|
329
|
-
const MODEL_MIGRATION_LIKELIHOOD = 35;
|
|
330
|
-
const MODEL_MIGRATION_TOKEN_MULT = 1.5;
|
|
331
|
-
|
|
332
|
-
const AGENTIC_TOOL_THRESHOLD = 3;
|
|
333
|
-
const AGENTIC_MULTITURN_THRESHOLD = 3;
|
|
334
|
-
const AGENTIC_TOKEN_MULT = 8;
|
|
335
|
-
const AGENTIC_CLONE_MULT = 2;
|
|
336
|
-
|
|
337
|
-
const RATE_LIMIT_THRESHOLD = 2;
|
|
338
|
-
const RATE_LIMIT_KEEP_LIKELIHOOD = 40;
|
|
339
|
-
|
|
340
|
-
const BATCH_COST_FACTOR = 0.5;
|
|
341
|
-
const BATCH_TOKEN_MULT = 2;
|
|
342
|
-
|
|
343
|
-
const EVAL_NON_USER_KEEP_LIKELIHOOD = 25;
|
|
344
|
-
const EVAL_CUTOFF_DAYS = 30;
|
|
345
|
-
|
|
346
|
-
const DOCS_SWEET_MIN = 2;
|
|
347
|
-
const DOCS_SWEET_MAX = 4;
|
|
348
|
-
const DOCS_OVER_THRESHOLD = 5;
|
|
349
|
-
const DOCS_BILLING_BOOST = 1.35;
|
|
350
|
-
const DOCS_BILLING_DROP_LIKELIHOOD = 30;
|
|
351
|
-
|
|
352
|
-
const FUNNEL_TTC_ENTERPRISE = 0.67;
|
|
353
|
-
const FUNNEL_TTC_FREE = 1.4;
|
|
354
|
-
|
|
355
|
-
// ── HELPER FUNCTIONS ──
|
|
356
|
-
function handleFunnelPostHooks(record, meta) {
|
|
357
|
-
// H9: API-to-Eval TTC scaled by tier
|
|
358
|
-
const segment = meta?.profile?.api_tier;
|
|
359
|
-
if (Array.isArray(record) && record.length > 1) {
|
|
360
|
-
const factor = (
|
|
361
|
-
segment === "Enterprise" ? FUNNEL_TTC_ENTERPRISE :
|
|
362
|
-
segment === "Free" ? FUNNEL_TTC_FREE :
|
|
363
|
-
1.0
|
|
364
|
-
);
|
|
365
|
-
if (factor !== 1.0) {
|
|
366
|
-
for (let i = 1; i < record.length; i++) {
|
|
367
|
-
const prev = dayjs(record[i - 1].time);
|
|
368
|
-
const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
|
|
369
|
-
record[i].time = prev.add(newGap, "milliseconds").toISOString();
|
|
370
|
-
}
|
|
371
|
-
}
|
|
372
|
-
}
|
|
373
|
-
return record;
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
function handleEverythingHooks(record, meta) {
|
|
377
|
-
const datasetStart = dayjs.unix(meta.datasetStart);
|
|
378
|
-
let events = record;
|
|
379
|
-
if (!events.length) return record;
|
|
380
|
-
const profile = meta && meta.profile ? meta.profile : {};
|
|
381
|
-
|
|
382
|
-
events.forEach(e => {
|
|
383
|
-
if (profile.api_tier) e.api_tier = profile.api_tier;
|
|
384
|
-
if (profile.primary_use_case) e.primary_use_case = profile.primary_use_case;
|
|
385
|
-
if (profile.sdk_language) e.sdk_language = profile.sdk_language;
|
|
386
|
-
});
|
|
387
|
-
|
|
388
|
-
// H6: Outage day (days 40-41) — 40% of api calls flipped to errors
|
|
389
|
-
events.forEach(e => {
|
|
390
|
-
if (e.event !== "api call") return;
|
|
391
|
-
const dayInDataset = dayjs(e.time).diff(datasetStart, "days", true);
|
|
392
|
-
if (dayInDataset >= OUTAGE_START_DAY && dayInDataset < OUTAGE_END_DAY) {
|
|
393
|
-
if (chance.bool({ likelihood: OUTAGE_ERROR_LIKELIHOOD })) {
|
|
394
|
-
e.is_error = true;
|
|
395
|
-
e.error_type = chance.pickone(["service_overloaded", "internal_server_error", "gateway_timeout"]);
|
|
396
|
-
e.latency_ms = Math.floor((e.latency_ms || 1500) * OUTAGE_LATENCY_MULT);
|
|
397
|
-
}
|
|
398
|
-
}
|
|
399
|
-
});
|
|
400
|
-
|
|
401
|
-
const sortedByTime = [...events].sort((a, b) => dayjs(a.time).valueOf() - dayjs(b.time).valueOf());
|
|
402
|
-
const firstEventTime = sortedByTime.length > 0 ? dayjs(sortedByTime[0].time) : datasetStart;
|
|
403
|
-
|
|
404
|
-
// H5: Tier-based context window & input tokens
|
|
405
|
-
const tier = profile.api_tier || "Free";
|
|
406
|
-
const contextWindow = TIER_CONTEXT_WINDOW[tier] ?? TIER_CONTEXT_WINDOW.Free;
|
|
407
|
-
const inputMultiplier = TIER_INPUT_MULT[tier] ?? TIER_INPUT_MULT.Free;
|
|
408
|
-
events.forEach(e => {
|
|
409
|
-
if (e.event === "api call") {
|
|
410
|
-
e.context_window = contextWindow;
|
|
411
|
-
e.input_tokens = Math.floor((e.input_tokens || 2000) * inputMultiplier);
|
|
412
|
-
}
|
|
413
|
-
});
|
|
414
|
-
|
|
415
|
-
// H1: Prompt caching adoption — ~25% of users; activates 20% into stream
|
|
416
|
-
const userId = events[0] && events[0].user_id;
|
|
417
|
-
const idHash = String(userId || "").split("").reduce((acc, c) => acc + c.charCodeAt(0), 0);
|
|
418
|
-
const isCacheUser = (idHash % CACHE_USER_HASH_MOD) === 0;
|
|
419
|
-
if (isCacheUser) {
|
|
420
|
-
let cacheActivated = false;
|
|
421
|
-
const activationPoint = Math.floor(events.length * CACHE_ACTIVATION_PCT);
|
|
422
|
-
events.forEach((e, idx) => {
|
|
423
|
-
if (e.event === "api call") {
|
|
424
|
-
if (idx >= activationPoint) cacheActivated = true;
|
|
425
|
-
if (cacheActivated) {
|
|
426
|
-
e.cache_enabled = true;
|
|
427
|
-
e.cost_usd = Math.round((e.cost_usd || 0.01) * CACHE_COST_FACTOR * 10000) / 10000;
|
|
428
|
-
}
|
|
429
|
-
}
|
|
430
|
-
});
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
// H2: Model migration wave — post-day-60 Build/Enterprise calls migrate to opus-4-7
|
|
434
|
-
const migrationCutoff = datasetStart.add(MODEL_MIGRATION_DAY, "days");
|
|
435
|
-
if (tier === "Build" || tier === "Enterprise") {
|
|
436
|
-
events.forEach(e => {
|
|
437
|
-
if (e.event === "api call" && dayjs(e.time).isAfter(migrationCutoff)) {
|
|
438
|
-
if (chance.bool({ likelihood: MODEL_MIGRATION_LIKELIHOOD })) {
|
|
439
|
-
e.model = "opus-4-7";
|
|
440
|
-
e.tokens_used = Math.floor((e.tokens_used || 2500) * MODEL_MIGRATION_TOKEN_MULT);
|
|
441
|
-
}
|
|
442
|
-
}
|
|
443
|
-
});
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
// H3: Agentic loop power users — 3+ tool calls + 3+ multi_turn → 8x tokens, 3x events
|
|
447
|
-
const toolUseCount = events.filter(e => e.event === "tool use call").length;
|
|
448
|
-
const multiTurnCount = events.filter(e => e.event === "api call" && e.multi_turn === true).length;
|
|
449
|
-
const isAgenticUser = toolUseCount >= AGENTIC_TOOL_THRESHOLD && multiTurnCount >= AGENTIC_MULTITURN_THRESHOLD;
|
|
450
|
-
if (isAgenticUser) {
|
|
451
|
-
events.forEach(e => {
|
|
452
|
-
if (e.event === "api call") {
|
|
453
|
-
e.tokens_used = Math.floor((e.tokens_used || 2500) * AGENTIC_TOKEN_MULT);
|
|
454
|
-
}
|
|
455
|
-
});
|
|
456
|
-
const apiCalls = events.filter(e => e.event === "api call");
|
|
457
|
-
const extraCount = apiCalls.length * AGENTIC_CLONE_MULT;
|
|
458
|
-
for (let i = 0; i < extraCount; i++) {
|
|
459
|
-
const template = apiCalls[i % apiCalls.length];
|
|
460
|
-
if (template) {
|
|
461
|
-
events.push({
|
|
462
|
-
...template,
|
|
463
|
-
time: dayjs(template.time).add(chance.integer({ min: 1, max: 120 }), "minutes").toISOString(),
|
|
464
|
-
user_id: template.user_id,
|
|
465
|
-
multi_turn: true,
|
|
466
|
-
});
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
// H4: Rate-limit churn — 2+ early rate-limit errors → drop 60% of post-week-1 events
|
|
472
|
-
const firstWeekEnd = firstEventTime.add(7, "days");
|
|
473
|
-
const earlyRateLimits = events.filter(e =>
|
|
474
|
-
e.event === "rate limit error" && dayjs(e.time).isBefore(firstWeekEnd)
|
|
475
|
-
).length;
|
|
476
|
-
if (earlyRateLimits >= RATE_LIMIT_THRESHOLD) {
|
|
477
|
-
events = events.filter(e => {
|
|
478
|
-
if (dayjs(e.time).isAfter(firstWeekEnd)) {
|
|
479
|
-
return chance.bool({ likelihood: RATE_LIMIT_KEEP_LIKELIHOOD });
|
|
480
|
-
}
|
|
481
|
-
return true;
|
|
482
|
-
});
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
// H7: Batch API discount — any batch job submitted → 50% cost, 2x tokens
|
|
486
|
-
const isBatchUser = events.some(e => e.event === "batch job submitted");
|
|
487
|
-
if (isBatchUser) {
|
|
488
|
-
events.forEach(e => {
|
|
489
|
-
if (e.event === "api call") {
|
|
490
|
-
e.cost_per_token = Math.round((e.cost_per_token || 0.00001) * BATCH_COST_FACTOR * 10000000) / 10000000;
|
|
491
|
-
e.tokens_used = Math.floor((e.tokens_used || 2500) * BATCH_TOKEN_MULT);
|
|
492
|
-
}
|
|
493
|
-
});
|
|
494
|
-
}
|
|
495
|
-
|
|
496
|
-
// H8: Eval-driven retention — non-eval users lose 75% post-day-30
|
|
497
|
-
const hasEarlyEval = events.some(e =>
|
|
498
|
-
e.event === "eval job" && dayjs(e.time).isBefore(firstWeekEnd)
|
|
499
|
-
);
|
|
500
|
-
if (!hasEarlyEval) {
|
|
501
|
-
const cutoff = firstEventTime.add(EVAL_CUTOFF_DAYS, "days");
|
|
502
|
-
events = events.filter(e => {
|
|
503
|
-
if (dayjs(e.time).isAfter(cutoff)) {
|
|
504
|
-
return chance.bool({ likelihood: EVAL_NON_USER_KEEP_LIKELIHOOD });
|
|
505
|
-
}
|
|
506
|
-
return true;
|
|
507
|
-
});
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
// H10: Docs-searched magic number — count between org-created and first billing
|
|
511
|
-
const orgEvent = events.find(e => e.event === "organization created");
|
|
512
|
-
const firstBilling = events.find(e => e.event === "billing payment");
|
|
513
|
-
if (orgEvent && firstBilling) {
|
|
514
|
-
const aTime = dayjs(orgEvent.time);
|
|
515
|
-
const bTime = dayjs(firstBilling.time);
|
|
516
|
-
const docsBetween = events.filter(e =>
|
|
517
|
-
e.event === "docs searched" &&
|
|
518
|
-
dayjs(e.time).isAfter(aTime) &&
|
|
519
|
-
dayjs(e.time).isBefore(bTime)
|
|
520
|
-
).length;
|
|
521
|
-
if (docsBetween >= DOCS_SWEET_MIN && docsBetween <= DOCS_SWEET_MAX) {
|
|
522
|
-
events.forEach(e => {
|
|
523
|
-
if (e.event === "billing payment" && typeof e.amount_usd === "number") {
|
|
524
|
-
e.amount_usd = Math.round(e.amount_usd * DOCS_BILLING_BOOST);
|
|
525
|
-
}
|
|
526
|
-
});
|
|
527
|
-
} else if (docsBetween >= DOCS_OVER_THRESHOLD) {
|
|
528
|
-
for (let i = events.length - 1; i >= 0; i--) {
|
|
529
|
-
if (events[i].event === "billing payment" && chance.bool({ likelihood: DOCS_BILLING_DROP_LIKELIHOOD })) {
|
|
530
|
-
events.splice(i, 1);
|
|
531
|
-
}
|
|
532
|
-
}
|
|
533
|
-
}
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
return events;
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
// ── CONFIG ──
|
|
540
|
-
/** @type {Config} */
|
|
541
|
-
const config = {
|
|
542
|
-
version: 2,
|
|
543
|
-
seed: SEED,
|
|
544
|
-
datasetStart: DATASET_START,
|
|
545
|
-
datasetEnd: DATASET_END,
|
|
546
|
-
avgEventsPerUserPerDay: EVENTS_PER_DAY,
|
|
547
|
-
numUsers: NUM_USERS,
|
|
548
|
-
format: "json",
|
|
549
|
-
gzip: true,
|
|
550
|
-
credentials: {
|
|
551
|
-
token,
|
|
552
|
-
},
|
|
553
|
-
switches: {
|
|
554
|
-
hasSessionIds: true,
|
|
555
|
-
alsoInferFunnels: false,
|
|
556
|
-
hasLocation: true,
|
|
557
|
-
hasAndroidDevices: false,
|
|
558
|
-
hasIOSDevices: false,
|
|
559
|
-
hasDesktopDevices: true,
|
|
560
|
-
hasBrowser: true,
|
|
561
|
-
hasCampaigns: false,
|
|
562
|
-
isAnonymous: false,
|
|
563
|
-
hasAdSpend: false,
|
|
564
|
-
hasAvatar: true,
|
|
565
|
-
},
|
|
566
|
-
identity: {
|
|
567
|
-
avgDevicePerUser: 2,
|
|
568
|
-
},
|
|
569
|
-
concurrency: 1,
|
|
570
|
-
writeToDisk: false,
|
|
571
|
-
soup: "growth",
|
|
572
|
-
scdProps: {
|
|
573
|
-
monthly_api_usage: {
|
|
574
|
-
values: u.weighNumRange(0, 1000000, 0.3, 50),
|
|
575
|
-
frequency: "week",
|
|
576
|
-
timing: "fuzzy",
|
|
577
|
-
max: 20,
|
|
578
|
-
},
|
|
579
|
-
api_tier_history: {
|
|
580
|
-
values: ["Free", "Build", "Enterprise"],
|
|
581
|
-
frequency: "month",
|
|
582
|
-
timing: "fixed",
|
|
583
|
-
max: 6,
|
|
584
|
-
},
|
|
585
|
-
},
|
|
586
|
-
events: [
|
|
587
|
-
{
|
|
588
|
-
event: "organization created",
|
|
589
|
-
weight: 1,
|
|
590
|
-
isFirstEvent: true,
|
|
591
|
-
isAuthEvent: true,
|
|
592
|
-
properties: {
|
|
593
|
-
org_size: ["solo", "startup", "growth", "enterprise"],
|
|
594
|
-
referral_source: ["docs", "blog", "github", "word_of_mouth", "search", "conference"],
|
|
595
|
-
},
|
|
596
|
-
},
|
|
597
|
-
{
|
|
598
|
-
event: "api key created",
|
|
599
|
-
weight: 2,
|
|
600
|
-
properties: {
|
|
601
|
-
key_type: ["development", "production", "staging"],
|
|
602
|
-
key_scope: ["full_access", "read_only", "completions_only"],
|
|
603
|
-
},
|
|
604
|
-
},
|
|
605
|
-
{
|
|
606
|
-
event: "api key rotated",
|
|
607
|
-
weight: 1,
|
|
608
|
-
properties: {
|
|
609
|
-
rotation_reason: ["scheduled", "compromised", "policy", "manual"],
|
|
610
|
-
},
|
|
611
|
-
},
|
|
612
|
-
{
|
|
613
|
-
event: "api call",
|
|
614
|
-
weight: 10,
|
|
615
|
-
isStrictEvent: false,
|
|
616
|
-
properties: {
|
|
617
|
-
model: ["sonnet-4", "sonnet-4", "sonnet-4", "haiku-4", "haiku-4", "opus-4-6"],
|
|
618
|
-
input_tokens: u.weighNumRange(50, 8000, 0.4, 2000),
|
|
619
|
-
output_tokens: u.weighNumRange(10, 4000, 0.4, 500),
|
|
620
|
-
tokens_used: u.weighNumRange(100, 12000, 0.4, 2500),
|
|
621
|
-
cost_usd: [0.001, 0.002, 0.003, 0.003, 0.005, 0.005, 0.005, 0.008, 0.008, 0.01, 0.01, 0.01, 0.01, 0.015, 0.015, 0.02, 0.025, 0.03, 0.04, 0.05],
|
|
622
|
-
cost_per_token: [0.000002, 0.000003, 0.000005, 0.000005, 0.000008, 0.000008, 0.00001, 0.00001, 0.00001, 0.000012, 0.000015, 0.00002, 0.000025, 0.00003],
|
|
623
|
-
latency_ms: u.weighNumRange(100, 15000, 0.4, 1500),
|
|
624
|
-
cache_enabled: [false],
|
|
625
|
-
is_error: [false],
|
|
626
|
-
error_type: ["none"],
|
|
627
|
-
multi_turn: [false, false, false, true],
|
|
628
|
-
context_window: [200000],
|
|
629
|
-
stream: [true, true, true, false],
|
|
630
|
-
stop_reason: ["end_turn", "end_turn", "end_turn", "max_tokens", "tool_use"],
|
|
631
|
-
},
|
|
632
|
-
},
|
|
633
|
-
{
|
|
634
|
-
event: "tool use call",
|
|
635
|
-
weight: 4,
|
|
636
|
-
isStrictEvent: false,
|
|
637
|
-
properties: {
|
|
638
|
-
tool_name: ["web_search", "code_interpreter", "file_reader", "calculator", "database_query", "api_connector"],
|
|
639
|
-
execution_time_ms: u.weighNumRange(50, 10000, 0.4, 800),
|
|
640
|
-
success: [true, true, true, true, false],
|
|
641
|
-
tool_input_tokens: u.weighNumRange(50, 2000, 0.4, 300),
|
|
642
|
-
tool_output_tokens: u.weighNumRange(20, 5000, 0.4, 500),
|
|
643
|
-
},
|
|
644
|
-
},
|
|
645
|
-
{
|
|
646
|
-
event: "batch job submitted",
|
|
647
|
-
weight: 2,
|
|
648
|
-
properties: {
|
|
649
|
-
batch_size: u.weighNumRange(10, 10000, 0.3, 500),
|
|
650
|
-
model: ["sonnet-4", "haiku-4", "opus-4-6"],
|
|
651
|
-
estimated_tokens: u.weighNumRange(10000, 5000000, 0.3, 500000),
|
|
652
|
-
priority: ["standard", "standard", "standard", "express"],
|
|
653
|
-
},
|
|
654
|
-
},
|
|
655
|
-
{
|
|
656
|
-
event: "batch job completed",
|
|
657
|
-
weight: 2,
|
|
658
|
-
properties: {
|
|
659
|
-
batch_size: u.weighNumRange(10, 10000, 0.3, 500),
|
|
660
|
-
processing_time_sec: u.weighNumRange(60, 7200, 0.4, 900),
|
|
661
|
-
total_tokens: u.weighNumRange(10000, 5000000, 0.3, 500000),
|
|
662
|
-
success_rate: u.weighNumRange(90, 100, 0.8, 98),
|
|
663
|
-
},
|
|
664
|
-
},
|
|
665
|
-
{
|
|
666
|
-
event: "eval job",
|
|
667
|
-
weight: 3,
|
|
668
|
-
isStrictEvent: false,
|
|
669
|
-
properties: {
|
|
670
|
-
eval_type: ["accuracy", "relevance", "safety", "latency", "cost", "custom"],
|
|
671
|
-
num_test_cases: u.weighNumRange(10, 1000, 0.3, 100),
|
|
672
|
-
model: ["sonnet-4", "haiku-4", "opus-4-6"],
|
|
673
|
-
dataset_name: ["prod_prompts", "safety_suite", "regression_set", "benchmark_v2", "custom_eval"],
|
|
674
|
-
},
|
|
675
|
-
},
|
|
676
|
-
{
|
|
677
|
-
event: "eval result",
|
|
678
|
-
weight: 3,
|
|
679
|
-
properties: {
|
|
680
|
-
eval_type: ["accuracy", "relevance", "safety", "latency", "cost", "custom"],
|
|
681
|
-
score: u.weighNumRange(0, 100, 0.6, 75),
|
|
682
|
-
pass_rate: u.weighNumRange(50, 100, 0.7, 85),
|
|
683
|
-
model: ["sonnet-4", "haiku-4", "opus-4-6"],
|
|
684
|
-
regression_detected: [false, false, false, false, true],
|
|
685
|
-
},
|
|
686
|
-
},
|
|
687
|
-
{
|
|
688
|
-
event: "rate limit error",
|
|
689
|
-
weight: 3,
|
|
690
|
-
properties: {
|
|
691
|
-
error_code: [429],
|
|
692
|
-
retry_after_ms: u.weighNumRange(1000, 60000, 0.3, 5000),
|
|
693
|
-
requests_per_minute: u.weighNumRange(50, 2000, 0.4, 500),
|
|
694
|
-
tier_limit: ["Free", "Build", "Enterprise"],
|
|
695
|
-
},
|
|
696
|
-
},
|
|
697
|
-
{
|
|
698
|
-
event: "billing payment",
|
|
699
|
-
weight: 2,
|
|
700
|
-
isStrictEvent: false,
|
|
701
|
-
properties: {
|
|
702
|
-
amount_usd: u.weighNumRange(5, 50000, 0.2, 500),
|
|
703
|
-
payment_method: ["credit_card", "credit_card", "credit_card", "invoice", "wire_transfer"],
|
|
704
|
-
billing_period: ["monthly", "monthly", "annual"],
|
|
705
|
-
tokens_consumed: u.weighNumRange(100000, 50000000, 0.3, 5000000),
|
|
706
|
-
},
|
|
707
|
-
},
|
|
708
|
-
{
|
|
709
|
-
event: "model selected",
|
|
710
|
-
weight: 3,
|
|
711
|
-
properties: {
|
|
712
|
-
model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
713
|
-
is_default: [true, true, false],
|
|
714
|
-
selection_context: ["playground", "api_config", "eval_setup", "batch_config"],
|
|
715
|
-
},
|
|
716
|
-
},
|
|
717
|
-
{
|
|
718
|
-
event: "dashboard viewed",
|
|
719
|
-
weight: 5,
|
|
720
|
-
properties: {
|
|
721
|
-
dashboard_section: ["usage", "billing", "api_keys", "models", "evals", "logs"],
|
|
722
|
-
time_range: ["1h", "24h", "7d", "30d"],
|
|
723
|
-
},
|
|
724
|
-
},
|
|
725
|
-
{
|
|
726
|
-
event: "docs searched",
|
|
727
|
-
weight: 4,
|
|
728
|
-
properties: {
|
|
729
|
-
search_query_category: ["api_reference", "quickstart", "pricing", "models", "tool_use", "batch_api", "caching", "errors"],
|
|
730
|
-
results_found: u.weighNumRange(0, 50, 0.5, 8),
|
|
731
|
-
clicked_result: [true, true, true, false],
|
|
732
|
-
},
|
|
733
|
-
},
|
|
734
|
-
{
|
|
735
|
-
event: "member invited",
|
|
736
|
-
weight: 2,
|
|
737
|
-
properties: {
|
|
738
|
-
invite_role: ["admin", "developer", "developer", "billing", "viewer"],
|
|
739
|
-
invite_method: ["email", "email", "sso", "link"],
|
|
740
|
-
},
|
|
741
|
-
},
|
|
742
|
-
{
|
|
743
|
-
event: "webhook configured",
|
|
744
|
-
weight: 1,
|
|
745
|
-
properties: {
|
|
746
|
-
webhook_event: ["usage_alert", "rate_limit", "batch_complete", "eval_complete", "billing_threshold"],
|
|
747
|
-
delivery_method: ["https", "https", "slack", "email"],
|
|
748
|
-
},
|
|
749
|
-
},
|
|
750
|
-
{
|
|
751
|
-
event: "playground session",
|
|
752
|
-
weight: 4,
|
|
753
|
-
properties: {
|
|
754
|
-
model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
755
|
-
turns: u.weighNumRange(1, 30, 0.4, 5),
|
|
756
|
-
shared: [false, false, false, true],
|
|
757
|
-
tokens_used: u.weighNumRange(100, 20000, 0.3, 3000),
|
|
758
|
-
},
|
|
759
|
-
},
|
|
760
|
-
{
|
|
761
|
-
event: "account deactivated",
|
|
762
|
-
weight: 1,
|
|
763
|
-
isChurnEvent: true,
|
|
764
|
-
returnLikelihood: 0.1,
|
|
765
|
-
isStrictEvent: true,
|
|
766
|
-
properties: {
|
|
767
|
-
reason: ["cost", "switched_provider", "project_ended", "rate_limits", "no_longer_needed", "performance"],
|
|
768
|
-
},
|
|
769
|
-
},
|
|
770
|
-
],
|
|
771
|
-
funnels: [
|
|
772
|
-
{
|
|
773
|
-
name: "Onboarding",
|
|
774
|
-
sequence: ["organization created", "api key created", "api call"],
|
|
775
|
-
conversionRate: 70,
|
|
776
|
-
order: "sequential",
|
|
777
|
-
isFirstFunnel: true,
|
|
778
|
-
timeToConvert: 48,
|
|
779
|
-
weight: 3,
|
|
780
|
-
},
|
|
781
|
-
{
|
|
782
|
-
name: "API to Eval Pipeline",
|
|
783
|
-
sequence: ["api call", "tool use call", "eval job"],
|
|
784
|
-
conversionRate: 45,
|
|
785
|
-
order: "sequential",
|
|
786
|
-
timeToConvert: 168,
|
|
787
|
-
weight: 5,
|
|
788
|
-
},
|
|
789
|
-
{
|
|
790
|
-
name: "Usage to Billing",
|
|
791
|
-
sequence: ["api call", "billing payment"],
|
|
792
|
-
conversionRate: 30,
|
|
793
|
-
order: "sequential",
|
|
794
|
-
timeToConvert: 336,
|
|
795
|
-
weight: 2,
|
|
796
|
-
},
|
|
797
|
-
],
|
|
798
|
-
superProps: {
|
|
799
|
-
api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
|
|
800
|
-
primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
|
|
801
|
-
sdk_language: ["python", "typescript", "java", "go", "curl"],
|
|
802
|
-
},
|
|
803
|
-
userProps: {
|
|
804
|
-
api_tier: ["Free", "Free", "Build", "Build", "Enterprise"],
|
|
805
|
-
primary_use_case: ["chatbot", "code_generation", "data_extraction", "content_creation", "agents"],
|
|
806
|
-
sdk_language: ["python", "typescript", "java", "go", "curl"],
|
|
807
|
-
monthly_spend: u.weighNumRange(0, 50000, 0.2, 200),
|
|
808
|
-
total_api_calls: u.weighNumRange(0, 500000, 0.2, 10000),
|
|
809
|
-
preferred_model: ["sonnet-4", "sonnet-4", "haiku-4", "opus-4-6"],
|
|
810
|
-
},
|
|
811
|
-
hook(record, type, meta) {
|
|
812
|
-
if (type === "funnel-post") return handleFunnelPostHooks(record, meta);
|
|
813
|
-
if (type === "everything") return handleEverythingHooks(record, meta);
|
|
814
|
-
return record;
|
|
815
|
-
},
|
|
816
|
-
};
|
|
817
|
-
|
|
818
|
-
export default config;
|