@ak--47/dungeon-master 1.5.4 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/analyze-soup/SKILL.md +21 -12
- package/.claude/skills/create-dungeon/SKILL.md +65 -5
- package/.claude/skills/create-project/SKILL.md +1 -1
- package/.claude/skills/create-project/provision.mjs +57 -10
- package/.claude/skills/powertools/SKILL.md +75 -0
- package/.claude/skills/powertools/pt.mjs +91 -0
- package/.claude/skills/powertools/snapshot-project.mjs +124 -0
- package/.claude/skills/verify-dungeon/SKILL.md +56 -25
- package/.claude/skills/verify-dungeon/references/counting-semantics.md +26 -6
- package/.claude/skills/verify-dungeon/references/report-format.md +14 -8
- package/.claude/skills/write-hooks/SKILL.md +156 -18
- package/CHANGELOG.md +224 -0
- package/HOOKS.md +566 -112
- package/README.md +30 -9
- package/docs/guides/1.3.0-upgrade-guide.md +262 -0
- package/docs/guides/1.3.1-upgrade-guide.md +328 -0
- package/docs/guides/1.4.0-upgrade-guide.md +130 -0
- package/docs/guides/1.4.1-upgrade-guide.md +125 -0
- package/docs/guides/1.4.3-upgrade-guide.md +200 -0
- package/docs/guides/1.4.4-upgrade-guide.md +95 -0
- package/docs/guides/1.5.0-upgrade-guide.md +426 -0
- package/docs/guides/1.5.1-upgrade-guide.md +178 -0
- package/docs/guides/1.5.3-upgrade-guide.md +117 -0
- package/docs/guides/1.6.0-upgrade-guide.md +204 -0
- package/dungeons/technical/stories-verify.js +152 -0
- package/dungeons/vertical/README.md +154 -0
- package/dungeons/vertical/ai-platform/ai-platform.js +1424 -0
- package/dungeons/vertical/ai-platform/ai-platform.sql +184 -0
- package/dungeons/vertical/ai-platform/ai-platform.verify.mjs +71 -0
- package/dungeons/vertical/community/community.js +1589 -0
- package/dungeons/vertical/community/community.sql +216 -0
- package/dungeons/vertical/community/community.verify.mjs +72 -0
- package/dungeons/vertical/crypto/crypto.js +1483 -0
- package/dungeons/vertical/crypto/crypto.sql +194 -0
- package/dungeons/vertical/crypto/crypto.verify.mjs +78 -0
- package/dungeons/vertical/dating/dating.js +1540 -0
- package/dungeons/vertical/dating/dating.sql +288 -0
- package/dungeons/vertical/dating/dating.verify.mjs +72 -0
- package/dungeons/vertical/devtools/devtools.js +1524 -0
- package/dungeons/vertical/devtools/devtools.sql +161 -0
- package/dungeons/vertical/devtools/devtools.verify.mjs +73 -0
- package/dungeons/vertical/ecommerce/ecommerce.js +1480 -0
- package/dungeons/vertical/ecommerce/ecommerce.sql +218 -0
- package/dungeons/vertical/ecommerce/ecommerce.verify.mjs +70 -0
- package/dungeons/vertical/education/education.js +1763 -0
- package/dungeons/vertical/education/education.sql +243 -0
- package/dungeons/vertical/education/education.verify.mjs +75 -0
- package/dungeons/vertical/fintech/fintech.js +1410 -0
- package/dungeons/vertical/fintech/fintech.sql +175 -0
- package/dungeons/vertical/fintech/fintech.verify.mjs +70 -0
- package/dungeons/vertical/fitness/fitness.js +1495 -0
- package/dungeons/vertical/fitness/fitness.sql +206 -0
- package/dungeons/vertical/fitness/fitness.verify.mjs +71 -0
- package/dungeons/vertical/food-delivery/food-delivery.js +1641 -0
- package/dungeons/vertical/food-delivery/food-delivery.sql +183 -0
- package/dungeons/vertical/food-delivery/food-delivery.verify.mjs +75 -0
- package/dungeons/vertical/gaming/gaming.js +1741 -0
- package/dungeons/vertical/gaming/gaming.sql +249 -0
- package/dungeons/vertical/gaming/gaming.verify.mjs +70 -0
- package/dungeons/vertical/healthcare/healthcare.js +1488 -0
- package/dungeons/vertical/healthcare/healthcare.sql +193 -0
- package/dungeons/vertical/healthcare/healthcare.verify.mjs +72 -0
- package/dungeons/vertical/insurance-application/insurance-application.js +1852 -0
- package/dungeons/vertical/insurance-application/insurance-application.sql +174 -0
- package/dungeons/vertical/insurance-application/insurance-application.verify.mjs +74 -0
- package/dungeons/vertical/logistics/logistics.js +1541 -0
- package/dungeons/vertical/logistics/logistics.sql +205 -0
- package/dungeons/vertical/logistics/logistics.verify.mjs +73 -0
- package/dungeons/vertical/marketplace/marketplace.js +1438 -0
- package/dungeons/vertical/marketplace/marketplace.sql +148 -0
- package/dungeons/vertical/marketplace/marketplace.verify.mjs +74 -0
- package/dungeons/vertical/{media.js → media/media.js} +557 -58
- package/dungeons/vertical/media/media.sql +173 -0
- package/dungeons/vertical/media/media.verify.mjs +74 -0
- package/dungeons/vertical/real-estate/real-estate.js +1629 -0
- package/dungeons/vertical/real-estate/real-estate.sql +167 -0
- package/dungeons/vertical/real-estate/real-estate.verify.mjs +75 -0
- package/dungeons/vertical/sass/sass.js +1733 -0
- package/dungeons/vertical/sass/sass.sql +173 -0
- package/dungeons/vertical/sass/sass.verify.mjs +74 -0
- package/dungeons/vertical/social/social.js +1456 -0
- package/dungeons/vertical/social/social.sql +160 -0
- package/dungeons/vertical/social/social.verify.mjs +76 -0
- package/dungeons/vertical/streaming/streaming.js +677 -0
- package/dungeons/vertical/streaming/streaming.sql +101 -0
- package/dungeons/vertical/streaming/streaming.verify.mjs +75 -0
- package/dungeons/vertical/support-desk/support-desk.js +741 -0
- package/dungeons/vertical/support-desk/support-desk.sql +137 -0
- package/dungeons/vertical/support-desk/support-desk.verify.mjs +75 -0
- package/dungeons/vertical/travel/travel.js +1603 -0
- package/dungeons/vertical/travel/travel.sql +236 -0
- package/dungeons/vertical/travel/travel.verify.mjs +72 -0
- package/index.js +7 -3
- package/lib/core/config-validator.js +37 -2
- package/lib/core/context.js +4 -2
- package/lib/core/dungeon-loader.js +20 -15
- package/lib/core/dungeon-to-json.js +22 -4
- package/lib/core/storage.js +3 -3
- package/lib/generators/funnels.js +36 -9
- package/lib/hook-helpers/cohort.js +53 -0
- package/lib/hook-helpers/index.js +8 -0
- package/lib/hook-helpers/mutate.js +1 -1
- package/lib/hook-helpers/shape.js +319 -0
- package/lib/hook-patterns/aggregate-per-user-by-bin.js +13 -5
- package/lib/hook-patterns/attributed-by-source.js +60 -58
- package/lib/hook-patterns/frequency-by-frequency.js +14 -4
- package/lib/hook-patterns/funnel-frequency-breakdown.js +24 -12
- package/lib/hook-patterns/index.js +1 -1
- package/lib/hook-patterns/time-to-convert-by-segment.js +75 -13
- package/lib/orchestrators/user-loop.js +41 -0
- package/lib/templates/story-spec.schema.json +175 -0
- package/lib/utils/json-evaluator.js +1 -1
- package/lib/utils/utils.js +151 -59
- package/lib/verify/coerce.js +186 -0
- package/lib/verify/counting.js +270 -32
- package/lib/verify/emulate-breakdown.js +1285 -203
- package/lib/verify/first-time.js +89 -0
- package/lib/verify/flows.js +679 -0
- package/lib/verify/formula.js +259 -0
- package/lib/verify/funnel-engine.js +697 -113
- package/lib/verify/index.js +32 -4
- package/lib/verify/sessionize.js +275 -0
- package/lib/verify/story-runner.js +483 -0
- package/lib/verify/verify-dungeon.js +59 -37
- package/package.json +2 -1
- package/scripts/extract-dungeon-schema.mjs +2 -2
- package/scripts/run-many.mjs +2 -2
- package/scripts/verify-runner.mjs +2 -2
- package/scripts/verify-stories.mjs +244 -0
- package/types.d.ts +298 -30
- package/dungeons/vertical/ai-platform.js +0 -818
- package/dungeons/vertical/community.js +0 -966
- package/dungeons/vertical/crypto.js +0 -895
- package/dungeons/vertical/dating.js +0 -830
- package/dungeons/vertical/devtools.js +0 -1050
- package/dungeons/vertical/ecommerce.js +0 -811
- package/dungeons/vertical/education.js +0 -964
- package/dungeons/vertical/fintech.js +0 -946
- package/dungeons/vertical/fitness.js +0 -935
- package/dungeons/vertical/food-delivery.js +0 -934
- package/dungeons/vertical/gaming.js +0 -1120
- package/dungeons/vertical/healthcare.js +0 -945
- package/dungeons/vertical/insurance-application.js +0 -1039
- package/dungeons/vertical/logistics.js +0 -986
- package/dungeons/vertical/marketplace.js +0 -1019
- package/dungeons/vertical/real-estate.js +0 -872
- package/dungeons/vertical/sass.js +0 -995
- package/dungeons/vertical/social.js +0 -904
- package/dungeons/vertical/travel.js +0 -930
|
@@ -0,0 +1,1733 @@
|
|
|
1
|
+
// ── IMPORTS ──
|
|
2
|
+
import dayjs from "dayjs";
|
|
3
|
+
import utc from "dayjs/plugin/utc.js";
|
|
4
|
+
dayjs.extend(utc);
|
|
5
|
+
import "dotenv/config";
|
|
6
|
+
import * as u from "@ak--47/dungeon-master/utils";
|
|
7
|
+
import * as v from "ak-tools";
|
|
8
|
+
/** @typedef {import("../../../types").Dungeon} Config */
|
|
9
|
+
|
|
10
|
+
// ── OVERVIEW ──
|
|
11
|
+
/*
|
|
12
|
+
* NAME: CloudForge
|
|
13
|
+
* APP: B2B SaaS that fuses infrastructure monitoring (Datadog-style) with
|
|
14
|
+
* deployment automation (Terraform-style). Engineering teams create
|
|
15
|
+
* workspaces, deploy services across AWS/GCP/Azure, monitor uptime
|
|
16
|
+
* and cost, and respond to alerts via Slack/PagerDuty runbooks.
|
|
17
|
+
* Pricing: Free / Team / Business / Enterprise (seats + usage).
|
|
18
|
+
* SCALE: 10,000 users, ~1.4M events, 121 days (2026-01-01 → 2026-05-01)
|
|
19
|
+
* CORE LOOP: workspace created → service deployed → dashboard viewed → alert/resolve
|
|
20
|
+
*
|
|
21
|
+
* EVENTS (19):
|
|
22
|
+
* dashboard viewed (20) > api call (16) > query executed (15) > alert triggered (12)
|
|
23
|
+
* > service deployed (10) > deployment pipeline run (9) > alert acknowledged (8)
|
|
24
|
+
* > alert resolved (7) > documentation viewed (7) > security scan (6)
|
|
25
|
+
* > infrastructure scaled (5) > cost report generated (4) > integration configured (4)
|
|
26
|
+
* > feature flag toggled (4) > team member invited (3) > runbook executed (3)
|
|
27
|
+
* > billing event (3) > workspace created (1) > incident created (1)
|
|
28
|
+
*
|
|
29
|
+
* FUNNELS (8):
|
|
30
|
+
* - Onboarding: workspace created → service deployed → dashboard viewed (70%)
|
|
31
|
+
* - Daily Monitoring: dashboard viewed → query executed → api call (80%)
|
|
32
|
+
* - Incident Response: alert triggered → alert acknowledged → alert resolved (55%)
|
|
33
|
+
* - Deployment: deployment pipeline run → service deployed → dashboard viewed (65%, Canary A/B)
|
|
34
|
+
* - Infrastructure Mgmt: cost report generated → infrastructure scaled → security scan (50%)
|
|
35
|
+
* - Team & Config: team member invited → integration configured → feature flag toggled (40%)
|
|
36
|
+
* - Docs & Runbooks: documentation viewed → runbook executed → service deployed (45%)
|
|
37
|
+
* - Billing: billing event → dashboard viewed (60%)
|
|
38
|
+
*
|
|
39
|
+
* USER PROPS: company_size, primary_role, team_name, seat_count, annual_contract_value,
|
|
40
|
+
* customer_success_manager, customer_health_score, plan_tier, cloud_provider
|
|
41
|
+
* SUPER PROPS: plan_tier, cloud_provider
|
|
42
|
+
* SCD PROPS: primary_role (viewer/editor/admin/owner, monthly fuzzy, max 6),
|
|
43
|
+
* plan_tier (starter/growth/enterprise/scale, monthly fixed, max 6, company_id-scoped)
|
|
44
|
+
* GROUPS: company_id (300 companies)
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
// ── HOOK STORIES ──
|
|
48
|
+
/*
|
|
49
|
+
* NOTE: All cohort effects are HIDDEN — no flag stamping. Discoverable via
|
|
50
|
+
* behavioral cohorts or raw-prop breakdowns (company_size, day, doc_section).
|
|
51
|
+
*
|
|
52
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
53
|
+
* 1. END-OF-QUARTER SPIKE (everything)
|
|
54
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
55
|
+
*
|
|
56
|
+
* PATTERN: Days 100-110 billing events shift event_type toward "plan_upgraded"
|
|
57
|
+
* 40% of the time and team member invitations are duplicated 50% of the time.
|
|
58
|
+
* No flag — discover via line chart by day.
|
|
59
|
+
*
|
|
60
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
61
|
+
*
|
|
62
|
+
* Report 1: Plan Upgrades Over Time
|
|
63
|
+
* - Report type: Insights
|
|
64
|
+
* - Event: "billing event"
|
|
65
|
+
* - Measure: Total
|
|
66
|
+
* - Filter: event_type = "plan_upgraded"
|
|
67
|
+
* - Line chart by day
|
|
68
|
+
* - Expected: ~3.9x upgrades/day during days 100-110 (2026-04-11 to
|
|
69
|
+
* 2026-04-21). Mechanism: baseline plan_upgraded share is 1/8 of
|
|
70
|
+
* billing events (~0.113 measured); in-window 40% are forced +
|
|
71
|
+
* 60% x baseline = ~0.45 share at flat billing volume.
|
|
72
|
+
*
|
|
73
|
+
* Report 2: Team Expansion Surge
|
|
74
|
+
* - Report type: Insights
|
|
75
|
+
* - Event: "team member invited"
|
|
76
|
+
* - Measure: Total
|
|
77
|
+
* - Line chart by day
|
|
78
|
+
* - Expected: ~1.55x invites/day during days 100-110 (50% clone
|
|
79
|
+
* likelihood -> 1.5x; clones land +1-60min, mostly in-window)
|
|
80
|
+
*
|
|
81
|
+
* REAL-WORLD ANALOGUE: B2B SaaS revenue clusters at quarter-close as sales
|
|
82
|
+
* teams pull deals forward and customers expand seats to lock in pricing.
|
|
83
|
+
*
|
|
84
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
85
|
+
* 2. CHURNED ACCOUNT SILENCING (everything)
|
|
86
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
87
|
+
*
|
|
88
|
+
* PATTERN: ~20% of users (deterministic: charCode-sum of user_id % 5 === 0)
|
|
89
|
+
* go completely silent after day 30. All post-d30 events are removed via
|
|
90
|
+
* splice(). No flag — derive cohort via behavioral retention bucket (users
|
|
91
|
+
* with zero activity past d30 vs the rest).
|
|
92
|
+
*
|
|
93
|
+
* MEASUREMENT NOTES (verified at 2K reduced scale):
|
|
94
|
+
* - Churn-hashed users born AFTER day 30 lose their ENTIRE stream: they
|
|
95
|
+
* appear as zero-event profiles (~2.4% of all profiles have no events).
|
|
96
|
+
* - Among event-visible users, the hashed cohort is ~19%; every one of
|
|
97
|
+
* them is born in days 1-30 (born_early share = 1.0 for the cohort).
|
|
98
|
+
* - Hook 5/10 deploy clones spawn at lastEvent + 1-48h AFTER the churn
|
|
99
|
+
* splice runs, so a churned sweet-spot user can carry clones up to
|
|
100
|
+
* ~day 33. Use day 34 (2026-02-04) as the behavioral silence cutoff
|
|
101
|
+
* to read the cohort cleanly (silent share 1.00 at that cutoff).
|
|
102
|
+
*
|
|
103
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
104
|
+
*
|
|
105
|
+
* Report 1: Retention Cliff
|
|
106
|
+
* - Cohort A: users with at least 1 event after 2026-02-04
|
|
107
|
+
* - Cohort B: users with events only before 2026-02-04
|
|
108
|
+
* - Compare cohort sizes — B should be ~19% of event-visible users
|
|
109
|
+
*
|
|
110
|
+
* Report 2: Activity Volume Pre/Post Day 30
|
|
111
|
+
* - Report type: Insights
|
|
112
|
+
* - Event: any event
|
|
113
|
+
* - Measure: Total per user
|
|
114
|
+
* - Line chart by day
|
|
115
|
+
* - Expected: a visible drop after d30 driven by the silent cohort
|
|
116
|
+
*
|
|
117
|
+
* REAL-WORLD ANALOGUE: Most SaaS churn happens silently — accounts simply
|
|
118
|
+
* stop logging in long before the formal cancellation lands.
|
|
119
|
+
*
|
|
120
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
121
|
+
* 3. ALERT ESCALATION REPLACEMENT (event)
|
|
122
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
123
|
+
*
|
|
124
|
+
* PATTERN: 30% of critical/emergency "alert triggered" events are REPLACED
|
|
125
|
+
* with an escalated "incident created" event. The escalated event keeps the
|
|
126
|
+
* source alert's properties (alert_id, severity, escalation fields).
|
|
127
|
+
*
|
|
128
|
+
* MEASUREMENT NOTES (verified at 2K reduced scale):
|
|
129
|
+
* - "incident created" is ALSO a weight-1 organic event in the events
|
|
130
|
+
* array. Organic incidents have NO alert_id (NULL); escalated ones
|
|
131
|
+
* carry alert_id from the source alert. Escalated is ~45% of total
|
|
132
|
+
* incident volume — don't count organic incidents against the 30%.
|
|
133
|
+
* - The exact invariant: escalated / (escalated + remaining crit/emerg
|
|
134
|
+
* alerts) = 0.30, because escalation removes the source alert.
|
|
135
|
+
*
|
|
136
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
137
|
+
*
|
|
138
|
+
* Report 1: Incident Created Discovery
|
|
139
|
+
* - Report type: Insights
|
|
140
|
+
* - Event: "incident created"
|
|
141
|
+
* - Measure: Total
|
|
142
|
+
* - Breakdown: "escalation_level"
|
|
143
|
+
* - Filter: alert_id is set (escalated only)
|
|
144
|
+
* - Expected: P1 and P2 incidents; 30% of pre-replacement crit/emerg volume
|
|
145
|
+
*
|
|
146
|
+
* Report 2: Alert vs Incident Ratio
|
|
147
|
+
* - Report type: Insights
|
|
148
|
+
* - Events: "alert triggered" (severity critical/emergency) AND
|
|
149
|
+
* "incident created" (alert_id set)
|
|
150
|
+
* - Measure: Total
|
|
151
|
+
* - Expected: incidents / (incidents + remaining crit/emerg alerts) = 0.30
|
|
152
|
+
*
|
|
153
|
+
* REAL-WORLD ANALOGUE: Severe alerts get auto-promoted into incident
|
|
154
|
+
* tickets that page on-call engineers and trigger customer comms.
|
|
155
|
+
*
|
|
156
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
157
|
+
* 4. INTEGRATION USERS SUCCEED (everything)
|
|
158
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
159
|
+
*
|
|
160
|
+
* PATTERN: Users with BOTH "slack" AND "pagerduty" "integration configured"
|
|
161
|
+
* events resolve alerts faster: response_time_mins reduced 60%, resolution_time_mins
|
|
162
|
+
* reduced 50%. No flag — derive cohort behaviorally.
|
|
163
|
+
*
|
|
164
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
165
|
+
*
|
|
166
|
+
* Report 1: Response Time by Integration Cohort
|
|
167
|
+
* - Cohort A: users who configured BOTH slack AND pagerduty integrations
|
|
168
|
+
* - Cohort B: rest
|
|
169
|
+
* - Event: "alert acknowledged"
|
|
170
|
+
* - Measure: Average of "response_time_mins"
|
|
171
|
+
* - Expected: A / B avg response ~ 0.39 (knob 0.4; H9 company-size
|
|
172
|
+
* scaling mixes evenly across both cohorts so the ratio stays clean)
|
|
173
|
+
*
|
|
174
|
+
* Report 2: Resolution Time by Integration Cohort
|
|
175
|
+
* - Cohort A vs B (as above)
|
|
176
|
+
* - Event: "alert resolved"
|
|
177
|
+
* - Measure: Average of "resolution_time_mins"
|
|
178
|
+
* - Expected: A / B avg resolution ~ 0.50 (knob 0.5)
|
|
179
|
+
*
|
|
180
|
+
* REAL-WORLD ANALOGUE: Teams that wire alerting into their existing comms
|
|
181
|
+
* stack respond minutes faster — the alert literally finds the human.
|
|
182
|
+
*
|
|
183
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
184
|
+
* 5. DOCS READERS DEPLOY MORE (everything)
|
|
185
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
186
|
+
*
|
|
187
|
+
* PATTERN: Users in the 4-7 documentation-viewed sweet spot get 2-3 extra
|
|
188
|
+
* production "service deployed" events cloned into their stream (clones at
|
|
189
|
+
* lastEvent + 1-48h; shared implementation with Hook 10 — same code block).
|
|
190
|
+
* Any doc view counts — there is no doc_section condition. No flag — derive
|
|
191
|
+
* cohort by counting doc views per user.
|
|
192
|
+
*
|
|
193
|
+
* MEASUREMENT NOTES (verified at 2K reduced scale, organic counterfactual):
|
|
194
|
+
* - ~1/3 of clones are shaved by the future-time guard (end-active users
|
|
195
|
+
* spawn clones past dataset end) → net ~ +1.7 deploys/user engineered.
|
|
196
|
+
* - Raw deploys/user by bucket is dominated by the ACTIVITY confound:
|
|
197
|
+
* users with more doc views have more of everything (organic
|
|
198
|
+
* deploys-per-other-event rises ~18% from low to over bucket).
|
|
199
|
+
* - The clean read is rate-over-rate: deploys per non-doc/non-deploy
|
|
200
|
+
* event, restricted to NON-CHURNED users (churn splice truncates doc
|
|
201
|
+
* counts and migrates users between buckets).
|
|
202
|
+
*
|
|
203
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
204
|
+
*
|
|
205
|
+
* Report 1: Per-User Deploy Rate by Docs Cohort
|
|
206
|
+
* - Cohort A: non-churned users with 4-7 "documentation viewed" events
|
|
207
|
+
* - Cohort B: non-churned users with 0-3
|
|
208
|
+
* - Event: "service deployed"
|
|
209
|
+
* - Measure: Total per user, normalized by overall activity
|
|
210
|
+
* - Expected: A / B deploys-per-other-event ~ 1.26
|
|
211
|
+
*
|
|
212
|
+
* REAL-WORLD ANALOGUE: Engineers who read the docs ship more confidently
|
|
213
|
+
* and more often than those who guess at the platform.
|
|
214
|
+
*
|
|
215
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
216
|
+
* 6. COST OVERRUN PATTERN (event — closure state)
|
|
217
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
218
|
+
*
|
|
219
|
+
* PATTERN: When cost_change_percent > 25 on a "cost report generated" event,
|
|
220
|
+
* the user is stored in a module-level Map. Their next "infrastructure scaled"
|
|
221
|
+
* event is forced to scale_direction = "down". No flag — discover by
|
|
222
|
+
* sequencing cost-report → infrastructure-scaled per user.
|
|
223
|
+
*
|
|
224
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
225
|
+
*
|
|
226
|
+
* Report 1: Scale Direction Distribution
|
|
227
|
+
* - Report type: Insights
|
|
228
|
+
* - Event: "infrastructure scaled"
|
|
229
|
+
* - Measure: Total
|
|
230
|
+
* - Breakdown: "scale_direction"
|
|
231
|
+
* - Expected: "down" share elevated above the 25% baseline (the
|
|
232
|
+
* scale_direction array is ["up","up","up","down"], NOT 50/50)
|
|
233
|
+
*
|
|
234
|
+
* Report 2: Sequencing Check
|
|
235
|
+
* - Inspect users whose most recent cost report had
|
|
236
|
+
* cost_change_percent > 25; their next "infrastructure scaled"
|
|
237
|
+
* should be scale_direction="down"
|
|
238
|
+
* - Expected: ~90% down-share for armed-state scale events vs ~27%
|
|
239
|
+
* unarmed. Not 100%: the hook arms on event-generation order while
|
|
240
|
+
* the read sequences by timestamp, and those orders differ slightly.
|
|
241
|
+
*
|
|
242
|
+
* REAL-WORLD ANALOGUE: A surprise cloud bill triggers an immediate
|
|
243
|
+
* downscale; no engineer ignores a 25% month-over-month cost jump.
|
|
244
|
+
*
|
|
245
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
246
|
+
* 7. FAILED DEPLOYMENT RECOVERY (everything)
|
|
247
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
248
|
+
*
|
|
249
|
+
* PATTERN: After a failed pipeline run, the user's next successful deploy has
|
|
250
|
+
* duration_sec * 1.5 (recovery deploys are slower). No flag — discover by
|
|
251
|
+
* sequencing failed → next-success pipeline events per user and comparing
|
|
252
|
+
* duration.
|
|
253
|
+
*
|
|
254
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
255
|
+
*
|
|
256
|
+
* Report 1: Pipeline Duration After Failure (sequencing query)
|
|
257
|
+
* - For each user, find runs where prior run was status="failed"
|
|
258
|
+
* - Compare avg duration_sec of those "next" runs vs all other successful runs
|
|
259
|
+
* - Expected: post-failure runs ~ 1.5x longer duration (measured 1.52
|
|
260
|
+
* at 2K when excluding each user's first run, which has no prior)
|
|
261
|
+
*
|
|
262
|
+
* REAL-WORLD ANALOGUE: After a bad deploy, teams add manual gates and
|
|
263
|
+
* extra verification steps that slow the very next release.
|
|
264
|
+
*
|
|
265
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
266
|
+
* 8. ENTERPRISE VS STARTUP (user)
|
|
267
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
268
|
+
*
|
|
269
|
+
* PATTERN: Company size determines seat_count, annual_contract_value, and
|
|
270
|
+
* customer_success_manager (enterprise only). All users get a
|
|
271
|
+
* customer_health_score on the profile.
|
|
272
|
+
*
|
|
273
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
274
|
+
*
|
|
275
|
+
* Report 1: ACV by Company Size
|
|
276
|
+
* - Report type: Insights
|
|
277
|
+
* - Event: any event
|
|
278
|
+
* - Measure: Unique users
|
|
279
|
+
* - Breakdown: "company_size" (user property)
|
|
280
|
+
* - Expected: startup ($0-3.6K), smb ($3.6K-12K), mid_market ($12K-50K),
|
|
281
|
+
* enterprise ($50K-500K)
|
|
282
|
+
*
|
|
283
|
+
* Report 2: Seat Count by Company Size
|
|
284
|
+
* - Report type: Insights
|
|
285
|
+
* - Event: any event
|
|
286
|
+
* - Measure: Average of "seat_count" (user property)
|
|
287
|
+
* - Breakdown: "company_size"
|
|
288
|
+
* - Expected: monotonic ramp from startup to enterprise
|
|
289
|
+
*
|
|
290
|
+
* REAL-WORLD ANALOGUE: B2B SaaS pricing scales orders of magnitude across
|
|
291
|
+
* customer segments — from a $99/mo startup to a $500K Fortune 500 contract.
|
|
292
|
+
*
|
|
293
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
294
|
+
* 9. INCIDENT RESPONSE TTC (everything)
|
|
295
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
296
|
+
*
|
|
297
|
+
* PATTERN: Enterprise companies resolve incidents faster; startups resolve
|
|
298
|
+
* slower. Two legs, same factors (enterprise 0.67x, startup 1.5x,
|
|
299
|
+
* smb/mid_market unchanged):
|
|
300
|
+
* 1. Property leg (everything hook): scales response_time_mins (on
|
|
301
|
+
* "alert acknowledged") and resolution_time_mins (on "alert
|
|
302
|
+
* resolved"). Carries the FULL engineered effect.
|
|
303
|
+
* 2. Funnel-timestamp leg (funnel-post hook, 1.6 rework): EVERY
|
|
304
|
+
* incident-funnel instance has its inter-step gaps scaled
|
|
305
|
+
* per-instance. Pre-1.6 this scaled one findFirstSequence chain in
|
|
306
|
+
* the everything hook — often stitched across separate funnel
|
|
307
|
+
* instances — so the report median was dominated by unscaled
|
|
308
|
+
* instances and the delta never survived.
|
|
309
|
+
* Compounds with Hook 4 (integration users) — an enterprise user with
|
|
310
|
+
* both Slack and PagerDuty stacks 0.4 x 0.67 on response time.
|
|
311
|
+
*
|
|
312
|
+
* MEASUREMENT NOTES (verified at 2K reduced scale):
|
|
313
|
+
* - The funnel TTC read ATTENUATES ASYMMETRICALLY under Mixpanel's
|
|
314
|
+
* greedy funnel evaluation. Organic standalone "alert triggered"
|
|
315
|
+
* soup events also start chains; the greedy evaluator takes the
|
|
316
|
+
* first completing chain. Compressed (enterprise) instances win
|
|
317
|
+
* that race → most of the 0.67 survives (observed ~0.79x baseline
|
|
318
|
+
* median). Stretched (startup) instances lose the race to organic
|
|
319
|
+
* alerts completing first → observed ~1.11x, not 1.5x.
|
|
320
|
+
* - Use a 24h conversion window: the TTC distribution is unimodal
|
|
321
|
+
* within 24h; longer windows only add a slow organic tail that
|
|
322
|
+
* dilutes the segment medians.
|
|
323
|
+
*
|
|
324
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
325
|
+
*
|
|
326
|
+
* Report 1: Avg Response Time by Company Size
|
|
327
|
+
* - Report type: Insights
|
|
328
|
+
* - Event: "alert acknowledged"
|
|
329
|
+
* - Measure: Average of "response_time_mins"
|
|
330
|
+
* - Breakdown: "company_size" (user property)
|
|
331
|
+
* - Expected: enterprise ~ 0.65x the smb/mid_market average;
|
|
332
|
+
* startup ~ 1.5x (full factors — property leg is unattenuated)
|
|
333
|
+
*
|
|
334
|
+
* Report 2: Incident Funnel TTC by Company Size
|
|
335
|
+
* - Report type: Funnels
|
|
336
|
+
* - Steps: "alert triggered" → "alert acknowledged" → "alert resolved"
|
|
337
|
+
* - Conversion window: 24 hours
|
|
338
|
+
* - Measure: Median time to convert
|
|
339
|
+
* - Breakdown: "company_size"
|
|
340
|
+
* - Expected: enterprise ~ 0.79x the smb/mid_market median;
|
|
341
|
+
* startup ~ 1.11x (greedy attenuation — see MEASUREMENT NOTES)
|
|
342
|
+
*
|
|
343
|
+
* Report 3: Avg Resolution Time by Company Size
|
|
344
|
+
* - Report type: Insights
|
|
345
|
+
* - Event: "alert resolved"
|
|
346
|
+
* - Measure: Average of "resolution_time_mins"
|
|
347
|
+
* - Breakdown: "company_size"
|
|
348
|
+
* - Expected: enterprise ~ 0.66x the smb/mid_market average;
|
|
349
|
+
* startup ~ 1.49x
|
|
350
|
+
*
|
|
351
|
+
* REAL-WORLD ANALOGUE: Enterprise teams have dedicated SRE rotations,
|
|
352
|
+
* automated runbooks, and premium support contracts that compress
|
|
353
|
+
* incident timelines. Startups rely on smaller teams with less tooling.
|
|
354
|
+
*
|
|
355
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
356
|
+
* 10. DOCS MAGIC NUMBER (everything)
|
|
357
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
358
|
+
*
|
|
359
|
+
* PATTERN: Users in the 4-7 documentation-viewed sweet spot get 2-3 extra
|
|
360
|
+
* production "service deployed" events cloned into their stream; users
|
|
361
|
+
* with 8+ documentation views are over-engaged browsers and 25% of their
|
|
362
|
+
* "service deployed" events are dropped (shared implementation with Hook
|
|
363
|
+
* 5 — same code block). No flag is stamped — discoverable only by binning
|
|
364
|
+
* users on doc-view count and comparing per-user deploy rate.
|
|
365
|
+
*
|
|
366
|
+
* MEASUREMENT NOTES (verified at 2K reduced scale, organic counterfactual):
|
|
367
|
+
* - The 25% drop is EXACT per user (paired organic-vs-hooked delta =
|
|
368
|
+
* 0.750), but the raw bucket comparison is confounded by the organic
|
|
369
|
+
* activity curve: over-bucket users are the most active, so their
|
|
370
|
+
* organic deploys-per-other-event runs ~1.18x the low bucket.
|
|
371
|
+
* - Net observed rate-over-rate (non-churned users): over/low ~ 0.89
|
|
372
|
+
* (= 0.75 engineered x 1.18 activity curve), over/sweet ~ 0.70
|
|
373
|
+
* (drop leg vs clone leg, confounds mostly cancel).
|
|
374
|
+
*
|
|
375
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
376
|
+
*
|
|
377
|
+
* Report 1: Deploy Rate by Docs-View Bucket
|
|
378
|
+
* - Report type: Insights (with cohort)
|
|
379
|
+
* - Cohort C: non-churned users with >= 8 "documentation viewed" events
|
|
380
|
+
* - Cohort B: non-churned users with 0-3
|
|
381
|
+
* - Event: "service deployed"
|
|
382
|
+
* - Measure: Total per user, normalized by overall activity
|
|
383
|
+
* - Expected: C / B deploys-per-other-event ~ 0.89
|
|
384
|
+
*
|
|
385
|
+
* Report 2: Heavy Readers vs Sweet Spot
|
|
386
|
+
* - Cohort C (8+) vs Cohort A (4-7), non-churned
|
|
387
|
+
* - Expected: C / A deploys-per-other-event ~ 0.70
|
|
388
|
+
*
|
|
389
|
+
* REAL-WORLD ANALOGUE: Engineers who read just enough docs deploy with
|
|
390
|
+
* confidence; those who read excessively may be stuck troubleshooting
|
|
391
|
+
* and never ship, or are evaluating the product without committing.
|
|
392
|
+
*
|
|
393
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
394
|
+
* 11. DEPLOY PIPELINE EXPERIMENT (funnel experiment — engine-managed)
|
|
395
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
396
|
+
*
|
|
397
|
+
* PATTERN: The deployment funnel (deployment pipeline run → service deployed
|
|
398
|
+
* → dashboard viewed) runs a "Canary Deploys" experiment starting 45 days
|
|
399
|
+
* before dataset end (2026-03-17). Users are deterministically assigned to
|
|
400
|
+
* Control or "Canary Deploys" variant (hash of user:experiment). The Canary
|
|
401
|
+
* variant gets 1.2x conversion multiplier and 0.85x time-to-convert
|
|
402
|
+
* multiplier. The engine emits `$experiment_started` events with
|
|
403
|
+
* `Experiment name` and `Variant name` properties (lowercase n — engine
|
|
404
|
+
* column casing). No hook code needed.
|
|
405
|
+
*
|
|
406
|
+
* MEASUREMENT NOTES (verified at 2K reduced scale):
|
|
407
|
+
* - Read PER-INSTANCE, not per-user: each $experiment_started anchors
|
|
408
|
+
* one funnel attempt (steps within 24h after it). User-level reads
|
|
409
|
+
* dilute the lift because multi-attempt users mix converted and
|
|
410
|
+
* unconverted instances (measured 1.15 user-level vs 1.21
|
|
411
|
+
* per-instance at 2K).
|
|
412
|
+
* - Both arms read HIGHER than the configured rates (0.87 vs knob
|
|
413
|
+
* 0.78, 0.72 vs 0.65) because organic deploy/dashboard soup events
|
|
414
|
+
* complete some engineered-failed instances. The RATIO survives:
|
|
415
|
+
* ~1.21 observed vs 1.2 knob.
|
|
416
|
+
*
|
|
417
|
+
* HOW TO FIND IT IN MIXPANEL:
|
|
418
|
+
*
|
|
419
|
+
* Report 1: Experiment Enrollment
|
|
420
|
+
* - Report type: Insights
|
|
421
|
+
* - Event: "$experiment_started"
|
|
422
|
+
* - Measure: Total
|
|
423
|
+
* - Breakdown: "Variant name"
|
|
424
|
+
* - Expected: roughly even user split between "Control" and
|
|
425
|
+
* "Canary Deploys" (deterministic hash, ~47-53%)
|
|
426
|
+
*
|
|
427
|
+
* Report 2: Deploy Funnel by Variant
|
|
428
|
+
* - Report type: Funnels
|
|
429
|
+
* - Steps: "deployment pipeline run" → "service deployed" → "dashboard viewed"
|
|
430
|
+
* - Breakdown: "Variant name"
|
|
431
|
+
* - Expected: Canary ~ 1.2x conversion vs Control (per-instance read)
|
|
432
|
+
*
|
|
433
|
+
* Report 3: Deploy TTC by Variant
|
|
434
|
+
* - Report type: Funnels
|
|
435
|
+
* - Steps: same as above
|
|
436
|
+
* - Measure: Median time to convert
|
|
437
|
+
* - Breakdown: "Variant name"
|
|
438
|
+
* - Expected: Canary median TTC ~ 0.81x Control (knob 0.85; small-n
|
|
439
|
+
* median noise at reduced scale)
|
|
440
|
+
*
|
|
441
|
+
* REAL-WORLD ANALOGUE: Teams A/B test canary deployment strategies.
|
|
442
|
+
* Canary deploys catch issues earlier, improving both success rate and
|
|
443
|
+
* deployment velocity.
|
|
444
|
+
*
|
|
445
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
446
|
+
* EXPECTED METRICS SUMMARY
|
|
447
|
+
* ─────────────────────────────────────────────────────────────────────────────
|
|
448
|
+
*
|
|
449
|
+
* All values measured at 2K reduced scale (seed harness-sass); ratios are
|
|
450
|
+
* scale-stable. "Organic" columns come from an exact counterfactual run:
|
|
451
|
+
* same seed with an identity hook — the dungeon's own chance instance is
|
|
452
|
+
* separate from the engine RNG, so users and organic events are identical.
|
|
453
|
+
*
|
|
454
|
+
* Hook | Metric | Expected
|
|
455
|
+
* -------------------------|---------------------------------------|----------
|
|
456
|
+
* H1a EOQ Spike | upgrades/day, days 100-110 vs rest | ~3.95x
|
|
457
|
+
* H1a EOQ Spike | plan_upgraded share in-window | ~0.45 (baseline 0.11)
|
|
458
|
+
* H1b EOQ Spike | invites/day, days 100-110 vs rest | ~1.55x
|
|
459
|
+
* H2 Churned Accounts | hash cohort share of event-users | ~19%
|
|
460
|
+
* H2 Churned Accounts | hash-true silent before 2026-02-04 | 100%
|
|
461
|
+
* H2 Churned Accounts | zero-event profiles (late-born churn) | ~2.4%
|
|
462
|
+
* H3 Alert Escalation | esc / (esc + remaining crit/emerg) | 0.30
|
|
463
|
+
* H3 Alert Escalation | esc share of total incidents | ~0.45
|
|
464
|
+
* H4 Integration Users | both-integ / rest avg response | ~0.39
|
|
465
|
+
* H4 Integration Users | both-integ / rest avg resolution | ~0.50
|
|
466
|
+
* H5 Docs Readers | sweet/low deploys-per-other (non-ch) | ~1.26
|
|
467
|
+
* H6 Cost Overrun | armed down-share vs unarmed | ~0.90 vs ~0.27
|
|
468
|
+
* H7 Deploy Recovery | recovery / other success duration | ~1.52
|
|
469
|
+
* H8 Enterprise v Startup | ACV ent/mid/smb/startup ($K) | ~273/31/7.8/1.8
|
|
470
|
+
* H9 Incident TTC (props) | ent & startup resp vs smb/mid avg | ~0.65x & ~1.50x
|
|
471
|
+
* H9 Incident TTC (funnel) | median TTC @24h window, ent & startup | ~0.79x & ~1.11x
|
|
472
|
+
* H10 Docs Magic Number | over/low deploys-per-other (non-ch) | ~0.89 (0.75 x 1.18 activity)
|
|
473
|
+
* H10 Docs Magic Number | over/sweet deploys-per-other | ~0.70
|
|
474
|
+
* H11 Deploy Experiment | per-instance conversion lift | ~1.21 (knob 1.2)
|
|
475
|
+
* H11 Deploy Experiment | Canary/Control median TTC | ~0.81 (knob 0.85)
|
|
476
|
+
*
|
|
477
|
+
* MEASUREMENT CAVEATS:
|
|
478
|
+
* - Activity confound: doc-view buckets correlate with overall activity;
|
|
479
|
+
* read deploys as a rate over other events, never raw counts.
|
|
480
|
+
* - Future-time guard shaves ~1/3 of H5 deploy clones (spawn past
|
|
481
|
+
* dataset end) — engineered +2.5 clones reads as ~+1.7.
|
|
482
|
+
* - Greedy funnel evaluation attenuates H9's stretched (startup) leg
|
|
483
|
+
* far more than the compressed (enterprise) leg — see Hook 9 notes.
|
|
484
|
+
* - H2 churn splice truncates doc counts: restrict H5/H10 cohorts to
|
|
485
|
+
* non-churned users or bucket membership migrates.
|
|
486
|
+
*/
|
|
487
|
+
|
|
488
|
+
// ── SCALE ──
|
|
489
|
+
const SEED = "harness-sass";
|
|
490
|
+
const NUM_USERS = 10_000;
|
|
491
|
+
const DATASET_START = "2026-01-01T00:00:00Z";
|
|
492
|
+
const DATASET_END = "2026-05-01T23:59:59Z";
|
|
493
|
+
const EVENTS_PER_DAY = 1.2;
|
|
494
|
+
const token = process.env.MP_TOKEN || "your-mixpanel-token";
|
|
495
|
+
|
|
496
|
+
const chance = u.initChance(SEED);
|
|
497
|
+
|
|
498
|
+
// ── KNOBS (tweak these to reshape stories) ──
|
|
499
|
+
const EOQ_START_DAY = 100;
|
|
500
|
+
const EOQ_END_DAY = 110;
|
|
501
|
+
const EOQ_UPGRADE_LIKELIHOOD = 40;
|
|
502
|
+
const EOQ_INVITE_CLONE_LIKELIHOOD = 50;
|
|
503
|
+
|
|
504
|
+
const CHURN_USER_HASH_MOD = 5;
|
|
505
|
+
const CHURN_CUTOFF_DAYS = 30;
|
|
506
|
+
|
|
507
|
+
const ALERT_ESCALATION_LIKELIHOOD = 30;
|
|
508
|
+
|
|
509
|
+
const INTEGRATION_RESPONSE_FACTOR = 0.4;
|
|
510
|
+
const INTEGRATION_RESOLUTION_FACTOR = 0.5;
|
|
511
|
+
|
|
512
|
+
const COST_OVERRUN_THRESHOLD = 25;
|
|
513
|
+
|
|
514
|
+
const FAILED_DEPLOY_RECOVERY_MULT = 1.5;
|
|
515
|
+
|
|
516
|
+
const DOCS_SWEET_MIN = 4;
|
|
517
|
+
const DOCS_SWEET_MAX = 7;
|
|
518
|
+
const DOCS_OVER_THRESHOLD = 8;
|
|
519
|
+
const DOCS_EXTRA_DEPLOYS_MIN = 2;
|
|
520
|
+
const DOCS_EXTRA_DEPLOYS_MAX = 3;
|
|
521
|
+
const DOCS_DEPLOY_DROP_LIKELIHOOD = 25;
|
|
522
|
+
|
|
523
|
+
const TTC_ENTERPRISE_FACTOR = 0.67;
|
|
524
|
+
const TTC_STARTUP_FACTOR = 1.5;
|
|
525
|
+
|
|
526
|
+
// ── DATA ARRAYS ──
|
|
527
|
+
const serviceIds = v.range(1, 201).map(() => `svc_${v.uid(8)}`);
|
|
528
|
+
const alertIds = v.range(1, 501).map(() => `alert_${v.uid(6)}`);
|
|
529
|
+
const pipelineIds = v.range(1, 101).map(() => `pipe_${v.uid(6)}`);
|
|
530
|
+
const runbookIds = v.range(1, 51).map(() => `rb_${v.uid(6)}`);
|
|
531
|
+
|
|
532
|
+
// ── HOOK STATE ──
|
|
533
|
+
// Module-level Map for closure-based state tracking across event-hook calls
|
|
534
|
+
const costOverrunUsers = new Map();
|
|
535
|
+
|
|
536
|
+
// ── HELPER FUNCTIONS ──
|
|
537
|
+
function handleEventHooks(record) {
|
|
538
|
+
// H3: ALERT ESCALATION REPLACEMENT — critical/emergency alerts sometimes
|
|
539
|
+
// become incident-created events.
|
|
540
|
+
if (record.event === "alert triggered") {
|
|
541
|
+
const severity = record.severity;
|
|
542
|
+
if ((severity === "critical" || severity === "emergency") && chance.bool({ likelihood: ALERT_ESCALATION_LIKELIHOOD })) {
|
|
543
|
+
return {
|
|
544
|
+
...record,
|
|
545
|
+
event: "incident created",
|
|
546
|
+
escalation_level: chance.pickone(["P1", "P2"]),
|
|
547
|
+
teams_paged: chance.integer({ min: 1, max: 5 }),
|
|
548
|
+
incident_id: `inc_${v.uid(8)}`,
|
|
549
|
+
original_severity: severity,
|
|
550
|
+
original_alert_type: record.alert_type,
|
|
551
|
+
auto_escalated: true,
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
// H6: COST OVERRUN PATTERN — cost reports with cost_change > 25% record
|
|
557
|
+
// the user, then the next infrastructure-scaled event from that user is
|
|
558
|
+
// forced to scale_direction = "down".
|
|
559
|
+
if (record.event === "cost report generated" && record.cost_change_percent > COST_OVERRUN_THRESHOLD) {
|
|
560
|
+
costOverrunUsers.set(record.user_id, true);
|
|
561
|
+
}
|
|
562
|
+
if (record.event === "infrastructure scaled" && costOverrunUsers.has(record.user_id)) {
|
|
563
|
+
record.scale_direction = "down";
|
|
564
|
+
costOverrunUsers.delete(record.user_id);
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
return record;
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
function handleUserHooks(record) {
|
|
571
|
+
// H8: ENTERPRISE VS STARTUP — company size determines seat count, ACV,
|
|
572
|
+
// and CSM. Real profile attrs.
|
|
573
|
+
const companySize = record.company_size;
|
|
574
|
+
if (companySize === "enterprise") {
|
|
575
|
+
record.seat_count = chance.integer({ min: 50, max: 500 });
|
|
576
|
+
record.annual_contract_value = chance.integer({ min: 50000, max: 500000 });
|
|
577
|
+
record.customer_success_manager = true;
|
|
578
|
+
} else if (companySize === "mid_market") {
|
|
579
|
+
record.seat_count = chance.integer({ min: 10, max: 50 });
|
|
580
|
+
record.annual_contract_value = chance.integer({ min: 12000, max: 50000 });
|
|
581
|
+
record.customer_success_manager = false;
|
|
582
|
+
} else if (companySize === "smb") {
|
|
583
|
+
record.seat_count = chance.integer({ min: 3, max: 10 });
|
|
584
|
+
record.annual_contract_value = chance.integer({ min: 3600, max: 12000 });
|
|
585
|
+
record.customer_success_manager = false;
|
|
586
|
+
} else if (companySize === "startup") {
|
|
587
|
+
record.seat_count = chance.integer({ min: 1, max: 5 });
|
|
588
|
+
record.annual_contract_value = chance.integer({ min: 0, max: 3600 });
|
|
589
|
+
record.customer_success_manager = false;
|
|
590
|
+
}
|
|
591
|
+
record.customer_health_score = chance.integer({ min: 1, max: 100 });
|
|
592
|
+
return record;
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
function handleFunnelPostHooks(record, meta) {
|
|
596
|
+
// H9: INCIDENT RESPONSE TTC — scale EVERY incident-funnel instance's
|
|
597
|
+
// inter-step gaps by company_size. Pre-1.6 this used findFirstSequence
|
|
598
|
+
// with a 30-day window in the everything hook: it scaled ONE sequence
|
|
599
|
+
// per user (often stitched across separate funnel instances), so the
|
|
600
|
+
// funnel median TTC was dominated by the user's unscaled instances and
|
|
601
|
+
// the engineered delta never survived to the report. Per-instance
|
|
602
|
+
// funnel-post scaling is what Mixpanel's funnel TTC actually reads.
|
|
603
|
+
// No other funnel shares the alert-step prefix, so scaling only this
|
|
604
|
+
// funnel cannot dilute reads elsewhere (marketplace H9 lesson).
|
|
605
|
+
if (meta?.funnel?.sequence?.[0] !== "alert triggered") return record;
|
|
606
|
+
const size = meta?.profile?.company_size;
|
|
607
|
+
const factor = (
|
|
608
|
+
size === "enterprise" ? TTC_ENTERPRISE_FACTOR :
|
|
609
|
+
size === "startup" ? TTC_STARTUP_FACTOR :
|
|
610
|
+
1.0
|
|
611
|
+
);
|
|
612
|
+
if (factor !== 1.0 && Array.isArray(record) && record.length > 1) {
|
|
613
|
+
for (let i = 1; i < record.length; i++) {
|
|
614
|
+
const prev = dayjs(record[i - 1].time);
|
|
615
|
+
const newGap = Math.round(dayjs(record[i].time).diff(prev) * factor);
|
|
616
|
+
record[i].time = prev.add(newGap, "milliseconds").toISOString();
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
return record;
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
function handleEverythingHooks(record, meta) {
|
|
623
|
+
const datasetStart = dayjs.unix(meta.datasetStart);
|
|
624
|
+
const userEvents = record;
|
|
625
|
+
const profile = meta.profile;
|
|
626
|
+
|
|
627
|
+
userEvents.forEach(e => {
|
|
628
|
+
e.plan_tier = profile.plan_tier;
|
|
629
|
+
e.cloud_provider = profile.cloud_provider;
|
|
630
|
+
});
|
|
631
|
+
|
|
632
|
+
// H1a: END-OF-QUARTER SPIKE — days 100-110, billing events flip
|
|
633
|
+
// event_type to plan_upgraded 40% of the time.
|
|
634
|
+
userEvents.forEach(e => {
|
|
635
|
+
if (e.event !== "billing event") return;
|
|
636
|
+
const dayInDataset = dayjs(e.time).diff(datasetStart, "days", true);
|
|
637
|
+
if (dayInDataset >= EOQ_START_DAY && dayInDataset <= EOQ_END_DAY && chance.bool({ likelihood: EOQ_UPGRADE_LIKELIHOOD })) {
|
|
638
|
+
e.event_type = "plan_upgraded";
|
|
639
|
+
}
|
|
640
|
+
});
|
|
641
|
+
|
|
642
|
+
// H1b: END-OF-QUARTER TEAM INVITE SPIKE — days 100-110, clone 50% of
|
|
643
|
+
// team-member-invited events (push, not return).
|
|
644
|
+
for (let i = userEvents.length - 1; i >= 0; i--) {
|
|
645
|
+
const e = userEvents[i];
|
|
646
|
+
if (e.event !== "team member invited") continue;
|
|
647
|
+
const dayInDataset = dayjs(e.time).diff(datasetStart, "days", true);
|
|
648
|
+
if (dayInDataset >= EOQ_START_DAY && dayInDataset <= EOQ_END_DAY && chance.bool({ likelihood: EOQ_INVITE_CLONE_LIKELIHOOD })) {
|
|
649
|
+
userEvents.push({
|
|
650
|
+
...e,
|
|
651
|
+
time: dayjs(e.time).add(chance.integer({ min: 1, max: 60 }), "minutes").toISOString(),
|
|
652
|
+
user_id: e.user_id,
|
|
653
|
+
role: chance.pickone(["editor", "viewer"]),
|
|
654
|
+
invitation_method: chance.pickone(["email", "sso", "slack"]),
|
|
655
|
+
});
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
// H2: CHURNED ACCOUNT SILENCING — ~20% of users (hash %5) have post-day-30
|
|
660
|
+
// events removed.
|
|
661
|
+
if (userEvents && userEvents.length > 0) {
|
|
662
|
+
const firstEvent = userEvents[0];
|
|
663
|
+
const idHash = String(firstEvent.user_id || firstEvent.device_id).split("").reduce((acc, char) => acc + char.charCodeAt(0), 0);
|
|
664
|
+
if ((idHash % CHURN_USER_HASH_MOD) === 0) {
|
|
665
|
+
for (let i = userEvents.length - 1; i >= 0; i--) {
|
|
666
|
+
const dayInDataset = dayjs(userEvents[i].time).diff(datasetStart, "days", true);
|
|
667
|
+
if (dayInDataset > CHURN_CUTOFF_DAYS) {
|
|
668
|
+
userEvents.splice(i, 1);
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
// H4: INTEGRATION USERS SUCCEED — Slack+PagerDuty users get alert
|
|
675
|
+
// response_time_mins 0.4x and resolution_time_mins 0.5x.
|
|
676
|
+
let hasSlack = false;
|
|
677
|
+
let hasPagerduty = false;
|
|
678
|
+
userEvents.forEach((event) => {
|
|
679
|
+
if (event.event === "integration configured") {
|
|
680
|
+
if (event.integration_type === "slack") hasSlack = true;
|
|
681
|
+
if (event.integration_type === "pagerduty") hasPagerduty = true;
|
|
682
|
+
}
|
|
683
|
+
});
|
|
684
|
+
if (hasSlack && hasPagerduty) {
|
|
685
|
+
userEvents.forEach((event) => {
|
|
686
|
+
if (event.event === "alert acknowledged" && event.response_time_mins) {
|
|
687
|
+
event.response_time_mins = Math.floor(event.response_time_mins * INTEGRATION_RESPONSE_FACTOR);
|
|
688
|
+
}
|
|
689
|
+
if (event.event === "alert resolved" && event.resolution_time_mins) {
|
|
690
|
+
event.resolution_time_mins = Math.floor(event.resolution_time_mins * INTEGRATION_RESOLUTION_FACTOR);
|
|
691
|
+
}
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
// H5 + H10: DOCS MAGIC NUMBER — sweet 4-7 docs → +40% extra cloned
|
|
696
|
+
// service-deployed events; over 8+ → drop 25% of service-deployed events.
|
|
697
|
+
const docsCount = userEvents.filter(e => e.event === "documentation viewed").length;
|
|
698
|
+
const deployTemplate = userEvents.find(e => e.event === "service deployed");
|
|
699
|
+
if (docsCount >= DOCS_SWEET_MIN && docsCount <= DOCS_SWEET_MAX && deployTemplate) {
|
|
700
|
+
const lastEvent = userEvents[userEvents.length - 1];
|
|
701
|
+
const extraDeploys = chance.integer({ min: DOCS_EXTRA_DEPLOYS_MIN, max: DOCS_EXTRA_DEPLOYS_MAX });
|
|
702
|
+
for (let i = 0; i < extraDeploys; i++) {
|
|
703
|
+
userEvents.push({
|
|
704
|
+
...deployTemplate,
|
|
705
|
+
time: dayjs(lastEvent.time).add(chance.integer({ min: 1, max: 48 }), "hours").toISOString(),
|
|
706
|
+
user_id: lastEvent.user_id,
|
|
707
|
+
service_id: chance.pickone(serviceIds),
|
|
708
|
+
service_type: chance.pickone(["web_app", "api", "database", "cache", "queue", "ml_model"]),
|
|
709
|
+
environment: "production",
|
|
710
|
+
cloud_provider: profile.cloud_provider,
|
|
711
|
+
});
|
|
712
|
+
}
|
|
713
|
+
} else if (docsCount >= DOCS_OVER_THRESHOLD) {
|
|
714
|
+
for (let i = userEvents.length - 1; i >= 0; i--) {
|
|
715
|
+
if (userEvents[i].event === "service deployed" && chance.bool({ likelihood: DOCS_DEPLOY_DROP_LIKELIHOOD })) {
|
|
716
|
+
userEvents.splice(i, 1);
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
// H7: FAILED DEPLOYMENT RECOVERY — find failed→success pairs in this
|
|
722
|
+
// user's pipeline events, multiply duration_sec by 1.5 on the recovery
|
|
723
|
+
// deploy.
|
|
724
|
+
const pipelineEvents = userEvents
|
|
725
|
+
.filter(e => e.event === "deployment pipeline run")
|
|
726
|
+
.sort((a, b) => a.time.localeCompare(b.time));
|
|
727
|
+
for (let i = 1; i < pipelineEvents.length; i++) {
|
|
728
|
+
if (pipelineEvents[i - 1].status === "failed" && pipelineEvents[i].status === "success") {
|
|
729
|
+
pipelineEvents[i].duration_sec = Math.floor((pipelineEvents[i].duration_sec || 300) * FAILED_DEPLOY_RECOVERY_MULT);
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
// H9 (property leg): INCIDENT RESPONSE TTC — enterprise resolves faster,
|
|
734
|
+
// startup slower. Scale response_time_mins on acknowledged events and
|
|
735
|
+
// resolution_time_mins on resolved events by company_size. The funnel
|
|
736
|
+
// timestamp leg lives in handleFunnelPostHooks (per-instance scaling).
|
|
737
|
+
// Compounds with H4 (integration users) — runs AFTER it, so an
|
|
738
|
+
// enterprise Slack+PagerDuty user stacks 0.4 x 0.67 on response time.
|
|
739
|
+
const companySegment = profile?.company_size;
|
|
740
|
+
const ttcFactor = (
|
|
741
|
+
companySegment === "enterprise" ? TTC_ENTERPRISE_FACTOR :
|
|
742
|
+
companySegment === "startup" ? TTC_STARTUP_FACTOR :
|
|
743
|
+
1.0
|
|
744
|
+
);
|
|
745
|
+
if (ttcFactor !== 1.0) {
|
|
746
|
+
// Property scale: affects Insights AVG reports
|
|
747
|
+
userEvents.forEach(e => {
|
|
748
|
+
if (e.event === "alert acknowledged" && e.response_time_mins) {
|
|
749
|
+
e.response_time_mins = Math.max(1, Math.round(e.response_time_mins * ttcFactor));
|
|
750
|
+
}
|
|
751
|
+
if (e.event === "alert resolved" && e.resolution_time_mins) {
|
|
752
|
+
e.resolution_time_mins = Math.max(1, Math.round(e.resolution_time_mins * ttcFactor));
|
|
753
|
+
}
|
|
754
|
+
});
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
return record;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
// ── CONFIG ──
|
|
761
|
+
/** @type {Config} */
|
|
762
|
+
const config = {
|
|
763
|
+
seed: SEED,
|
|
764
|
+
datasetStart: DATASET_START,
|
|
765
|
+
datasetEnd: DATASET_END,
|
|
766
|
+
avgEventsPerUserPerDay: EVENTS_PER_DAY,
|
|
767
|
+
numUsers: NUM_USERS,
|
|
768
|
+
format: "json",
|
|
769
|
+
gzip: true,
|
|
770
|
+
credentials: {
|
|
771
|
+
token,
|
|
772
|
+
},
|
|
773
|
+
switches: {
|
|
774
|
+
hasSessionIds: true,
|
|
775
|
+
alsoInferFunnels: false,
|
|
776
|
+
hasLocation: true,
|
|
777
|
+
hasAndroidDevices: false,
|
|
778
|
+
hasIOSDevices: false,
|
|
779
|
+
hasDesktopDevices: true,
|
|
780
|
+
hasBrowser: true,
|
|
781
|
+
hasCampaigns: false,
|
|
782
|
+
isAnonymous: false,
|
|
783
|
+
hasAdSpend: false,
|
|
784
|
+
hasAvatar: true,
|
|
785
|
+
},
|
|
786
|
+
// Phase 2 identity model — B2B SaaS reference. Engineers commonly use 1-2
|
|
787
|
+
// devices (desktop + work laptop). avgDevicePerUser:2 puts a meaningful
|
|
788
|
+
// per-session sticky-device pattern in Mixpanel device dashboards.
|
|
789
|
+
identity: {
|
|
790
|
+
avgDevicePerUser: 2,
|
|
791
|
+
},
|
|
792
|
+
concurrency: 1,
|
|
793
|
+
writeToDisk: false,
|
|
794
|
+
scdProps: {
|
|
795
|
+
primary_role: {
|
|
796
|
+
values: ["viewer", "editor", "admin", "owner"],
|
|
797
|
+
frequency: "month",
|
|
798
|
+
timing: "fuzzy",
|
|
799
|
+
max: 6
|
|
800
|
+
},
|
|
801
|
+
plan_tier: {
|
|
802
|
+
values: ["starter", "growth", "enterprise", "scale"],
|
|
803
|
+
frequency: "month",
|
|
804
|
+
timing: "fixed",
|
|
805
|
+
max: 6,
|
|
806
|
+
type: "company_id"
|
|
807
|
+
}
|
|
808
|
+
},
|
|
809
|
+
|
|
810
|
+
funnels: [
|
|
811
|
+
{
|
|
812
|
+
// First funnel — `workspace created` is the auth event for B2B users.
|
|
813
|
+
// Models real B2B onboarding: most teams take ≤1 retry before sticking.
|
|
814
|
+
sequence: ["workspace created", "service deployed", "dashboard viewed"],
|
|
815
|
+
isFirstFunnel: true,
|
|
816
|
+
conversionRate: 70,
|
|
817
|
+
timeToConvert: 2,
|
|
818
|
+
attempts: { min: 0, max: 1 },
|
|
819
|
+
},
|
|
820
|
+
{
|
|
821
|
+
// Daily monitoring: dashboards, queries, API calls (most common)
|
|
822
|
+
sequence: ["dashboard viewed", "query executed", "api call"],
|
|
823
|
+
conversionRate: 80,
|
|
824
|
+
timeToConvert: 0.5,
|
|
825
|
+
weight: 5,
|
|
826
|
+
},
|
|
827
|
+
{
|
|
828
|
+
// Incident response pipeline
|
|
829
|
+
sequence: ["alert triggered", "alert acknowledged", "alert resolved"],
|
|
830
|
+
conversionRate: 55,
|
|
831
|
+
timeToConvert: 6,
|
|
832
|
+
weight: 4,
|
|
833
|
+
},
|
|
834
|
+
{
|
|
835
|
+
// Deployment cycle
|
|
836
|
+
sequence: ["deployment pipeline run", "service deployed", "dashboard viewed"],
|
|
837
|
+
conversionRate: 65,
|
|
838
|
+
timeToConvert: 1,
|
|
839
|
+
weight: 3,
|
|
840
|
+
experiment: {
|
|
841
|
+
name: "Canary Deploys",
|
|
842
|
+
variants: [
|
|
843
|
+
{ name: "Control" },
|
|
844
|
+
{ name: "Canary Deploys", conversionMultiplier: 1.2, ttcMultiplier: 0.85 },
|
|
845
|
+
],
|
|
846
|
+
startDaysBeforeEnd: 45,
|
|
847
|
+
},
|
|
848
|
+
},
|
|
849
|
+
{
|
|
850
|
+
// Infrastructure management
|
|
851
|
+
sequence: ["cost report generated", "infrastructure scaled", "security scan"],
|
|
852
|
+
conversionRate: 50,
|
|
853
|
+
timeToConvert: 4,
|
|
854
|
+
weight: 2,
|
|
855
|
+
},
|
|
856
|
+
{
|
|
857
|
+
// Team and config management
|
|
858
|
+
sequence: ["team member invited", "integration configured", "feature flag toggled"],
|
|
859
|
+
conversionRate: 40,
|
|
860
|
+
timeToConvert: 8,
|
|
861
|
+
weight: 2,
|
|
862
|
+
},
|
|
863
|
+
{
|
|
864
|
+
// Documentation and runbook usage
|
|
865
|
+
sequence: ["documentation viewed", "runbook executed", "service deployed"],
|
|
866
|
+
conversionRate: 45,
|
|
867
|
+
timeToConvert: 2,
|
|
868
|
+
weight: 2,
|
|
869
|
+
},
|
|
870
|
+
{
|
|
871
|
+
// Billing and account management
|
|
872
|
+
sequence: ["billing event", "dashboard viewed"],
|
|
873
|
+
conversionRate: 60,
|
|
874
|
+
timeToConvert: 1,
|
|
875
|
+
weight: 1,
|
|
876
|
+
},
|
|
877
|
+
],
|
|
878
|
+
|
|
879
|
+
events: [
|
|
880
|
+
{
|
|
881
|
+
event: "workspace created",
|
|
882
|
+
weight: 1,
|
|
883
|
+
isFirstEvent: true,
|
|
884
|
+
// Phase 2 identity: workspace creation is the B2B equivalent of Sign Up
|
|
885
|
+
// — engine stamps user_id+device_id on this event when it fires inside
|
|
886
|
+
// the user's first funnel.
|
|
887
|
+
isAuthEvent: true,
|
|
888
|
+
properties: {
|
|
889
|
+
company_size: ["startup", "smb", "mid_market", "enterprise"],
|
|
890
|
+
industry: ["tech", "finance", "healthcare", "retail", "media"],
|
|
891
|
+
}
|
|
892
|
+
},
|
|
893
|
+
{
|
|
894
|
+
event: "service deployed",
|
|
895
|
+
weight: 10,
|
|
896
|
+
isStrictEvent: false,
|
|
897
|
+
properties: {
|
|
898
|
+
service_id: serviceIds,
|
|
899
|
+
service_type: ["web_app", "api", "database", "cache", "queue", "ml_model"],
|
|
900
|
+
environment: ["production", "staging", "dev"],
|
|
901
|
+
cloud_provider: ["aws", "gcp", "azure"],
|
|
902
|
+
}
|
|
903
|
+
},
|
|
904
|
+
{
|
|
905
|
+
event: "dashboard viewed",
|
|
906
|
+
weight: 20,
|
|
907
|
+
isStrictEvent: false,
|
|
908
|
+
properties: {
|
|
909
|
+
dashboard_type: ["overview", "cost", "performance", "security", "custom"],
|
|
910
|
+
time_range: ["1h", "6h", "24h", "7d", "30d"],
|
|
911
|
+
}
|
|
912
|
+
},
|
|
913
|
+
{
|
|
914
|
+
event: "alert triggered",
|
|
915
|
+
weight: 12,
|
|
916
|
+
isStrictEvent: false,
|
|
917
|
+
properties: {
|
|
918
|
+
alert_id: alertIds,
|
|
919
|
+
severity: ["info", "warning", "critical", "emergency"],
|
|
920
|
+
alert_type: ["cpu", "memory", "latency", "error_rate", "disk", "network"],
|
|
921
|
+
service_id: serviceIds,
|
|
922
|
+
}
|
|
923
|
+
},
|
|
924
|
+
{
|
|
925
|
+
event: "incident created",
|
|
926
|
+
weight: 1,
|
|
927
|
+
properties: {
|
|
928
|
+
escalation_level: ["P1", "P2"],
|
|
929
|
+
teams_paged: u.weighNumRange(1, 5),
|
|
930
|
+
incident_id: () => `inc_${v.uid(8)}`,
|
|
931
|
+
original_severity: ["critical", "emergency"],
|
|
932
|
+
original_alert_type: ["cpu", "memory", "latency", "error_rate", "disk", "network"],
|
|
933
|
+
service_id: serviceIds,
|
|
934
|
+
auto_escalated: [true],
|
|
935
|
+
}
|
|
936
|
+
},
|
|
937
|
+
{
|
|
938
|
+
event: "alert acknowledged",
|
|
939
|
+
weight: 8,
|
|
940
|
+
isStrictEvent: false,
|
|
941
|
+
properties: {
|
|
942
|
+
alert_id: alertIds,
|
|
943
|
+
response_time_mins: u.weighNumRange(1, 120),
|
|
944
|
+
acknowledged_by_role: ["engineer", "sre", "manager", "oncall"],
|
|
945
|
+
}
|
|
946
|
+
},
|
|
947
|
+
{
|
|
948
|
+
event: "alert resolved",
|
|
949
|
+
weight: 7,
|
|
950
|
+
isStrictEvent: false,
|
|
951
|
+
properties: {
|
|
952
|
+
alert_id: alertIds,
|
|
953
|
+
resolution_time_mins: u.weighNumRange(5, 1440),
|
|
954
|
+
root_cause: ["config_change", "capacity", "bug", "dependency", "network"],
|
|
955
|
+
}
|
|
956
|
+
},
|
|
957
|
+
{
|
|
958
|
+
event: "deployment pipeline run",
|
|
959
|
+
weight: 9,
|
|
960
|
+
isStrictEvent: false,
|
|
961
|
+
properties: {
|
|
962
|
+
pipeline_id: pipelineIds,
|
|
963
|
+
status: ["success", "failed", "cancelled"],
|
|
964
|
+
duration_sec: u.weighNumRange(30, 1800),
|
|
965
|
+
commit_count: u.weighNumRange(1, 20),
|
|
966
|
+
}
|
|
967
|
+
},
|
|
968
|
+
{
|
|
969
|
+
event: "infrastructure scaled",
|
|
970
|
+
weight: 5,
|
|
971
|
+
isStrictEvent: false,
|
|
972
|
+
properties: {
|
|
973
|
+
service_id: serviceIds,
|
|
974
|
+
scale_direction: ["up", "up", "up", "down"],
|
|
975
|
+
previous_capacity: u.weighNumRange(1, 100),
|
|
976
|
+
new_capacity: u.weighNumRange(1, 100),
|
|
977
|
+
auto_scaled: [false, false, false, false, false, false, true],
|
|
978
|
+
}
|
|
979
|
+
},
|
|
980
|
+
{
|
|
981
|
+
event: "cost report generated",
|
|
982
|
+
weight: 4,
|
|
983
|
+
isStrictEvent: false,
|
|
984
|
+
properties: {
|
|
985
|
+
report_period: ["daily", "weekly", "monthly"],
|
|
986
|
+
total_cost: u.weighNumRange(100, 50000),
|
|
987
|
+
cost_change_percent: u.weighNumRange(-30, 50),
|
|
988
|
+
}
|
|
989
|
+
},
|
|
990
|
+
{
|
|
991
|
+
event: "team member invited",
|
|
992
|
+
weight: 3,
|
|
993
|
+
isStrictEvent: false,
|
|
994
|
+
properties: {
|
|
995
|
+
role: ["admin", "editor", "viewer", "billing"],
|
|
996
|
+
invitation_method: ["email", "sso", "slack"],
|
|
997
|
+
}
|
|
998
|
+
},
|
|
999
|
+
{
|
|
1000
|
+
event: "integration configured",
|
|
1001
|
+
weight: 4,
|
|
1002
|
+
isStrictEvent: false,
|
|
1003
|
+
properties: {
|
|
1004
|
+
integration_type: ["slack", "pagerduty", "jira", "github", "datadog", "terraform"],
|
|
1005
|
+
status: ["active", "paused", "error"],
|
|
1006
|
+
}
|
|
1007
|
+
},
|
|
1008
|
+
{
|
|
1009
|
+
event: "query executed",
|
|
1010
|
+
weight: 15,
|
|
1011
|
+
properties: {
|
|
1012
|
+
query_type: ["metrics", "logs", "traces"],
|
|
1013
|
+
time_range_hours: u.weighNumRange(1, 720),
|
|
1014
|
+
result_count: u.weighNumRange(0, 10000),
|
|
1015
|
+
}
|
|
1016
|
+
},
|
|
1017
|
+
{
|
|
1018
|
+
event: "runbook executed",
|
|
1019
|
+
weight: 3,
|
|
1020
|
+
properties: {
|
|
1021
|
+
runbook_id: runbookIds,
|
|
1022
|
+
trigger: ["manual", "automated", "alert_triggered"],
|
|
1023
|
+
success: [false, false, false, false, false, false, true],
|
|
1024
|
+
}
|
|
1025
|
+
},
|
|
1026
|
+
{
|
|
1027
|
+
event: "billing event",
|
|
1028
|
+
weight: 3,
|
|
1029
|
+
isStrictEvent: false,
|
|
1030
|
+
properties: {
|
|
1031
|
+
event_type: ["invoice_generated", "invoice_generated", "payment_received", "payment_received", "payment_received", "payment_failed", "plan_upgraded", "plan_downgraded"],
|
|
1032
|
+
amount: u.weighNumRange(99, 25000),
|
|
1033
|
+
}
|
|
1034
|
+
},
|
|
1035
|
+
{
|
|
1036
|
+
event: "security scan",
|
|
1037
|
+
weight: 6,
|
|
1038
|
+
properties: {
|
|
1039
|
+
scan_type: ["vulnerability", "compliance", "access_audit"],
|
|
1040
|
+
findings_count: u.weighNumRange(0, 50),
|
|
1041
|
+
critical_findings: u.weighNumRange(0, 10),
|
|
1042
|
+
}
|
|
1043
|
+
},
|
|
1044
|
+
{
|
|
1045
|
+
event: "api call",
|
|
1046
|
+
weight: 16,
|
|
1047
|
+
properties: {
|
|
1048
|
+
endpoint: ["/deploy", "/status", "/metrics", "/alerts", "/config", "/billing"],
|
|
1049
|
+
method: ["GET", "POST", "PUT", "DELETE"],
|
|
1050
|
+
response_time_ms: u.weighNumRange(10, 5000),
|
|
1051
|
+
status_code: [200, 201, 400, 401, 403, 500, 503],
|
|
1052
|
+
}
|
|
1053
|
+
},
|
|
1054
|
+
{
|
|
1055
|
+
event: "documentation viewed",
|
|
1056
|
+
weight: 7,
|
|
1057
|
+
isStrictEvent: false,
|
|
1058
|
+
properties: {
|
|
1059
|
+
doc_section: ["getting_started", "api_reference", "best_practices", "troubleshooting", "changelog"],
|
|
1060
|
+
time_on_page_sec: u.weighNumRange(5, 600),
|
|
1061
|
+
}
|
|
1062
|
+
},
|
|
1063
|
+
{
|
|
1064
|
+
event: "feature flag toggled",
|
|
1065
|
+
weight: 4,
|
|
1066
|
+
properties: {
|
|
1067
|
+
flag_name: () => `flag_${chance.word()}`,
|
|
1068
|
+
new_state: ["disabled", "disabled", "disabled", "disabled", "disabled", "disabled", "enabled"],
|
|
1069
|
+
environment: ["production", "staging", "dev"],
|
|
1070
|
+
}
|
|
1071
|
+
},
|
|
1072
|
+
],
|
|
1073
|
+
|
|
1074
|
+
superProps: {
|
|
1075
|
+
plan_tier: ["free", "free", "team", "team", "business", "enterprise"],
|
|
1076
|
+
cloud_provider: ["aws", "gcp", "azure", "multi_cloud"],
|
|
1077
|
+
},
|
|
1078
|
+
|
|
1079
|
+
userProps: {
|
|
1080
|
+
company_size: ["startup", "startup", "smb", "mid_market", "enterprise"],
|
|
1081
|
+
primary_role: ["engineer", "sre", "devops", "manager", "executive"],
|
|
1082
|
+
team_name: ["Platform", "Backend", "Frontend", "Data", "Security", "Infrastructure"],
|
|
1083
|
+
seat_count: [1],
|
|
1084
|
+
annual_contract_value: [0],
|
|
1085
|
+
customer_success_manager: [false],
|
|
1086
|
+
customer_health_score: u.weighNumRange(1, 100),
|
|
1087
|
+
plan_tier: ["free", "free", "team", "team", "business", "enterprise"],
|
|
1088
|
+
cloud_provider: ["aws", "gcp", "azure", "multi_cloud"],
|
|
1089
|
+
},
|
|
1090
|
+
|
|
1091
|
+
groupKeys: [
|
|
1092
|
+
["company_id", 300, ["workspace created", "service deployed", "billing event", "team member invited"]],
|
|
1093
|
+
],
|
|
1094
|
+
|
|
1095
|
+
groupProps: {
|
|
1096
|
+
company_id: {
|
|
1097
|
+
name: () => `${chance.word({ capitalize: true })} ${chance.pickone(["Systems", "Technologies", "Labs", "Cloud", "Digital", "Networks", "Solutions"])}`,
|
|
1098
|
+
industry: ["tech", "finance", "healthcare", "retail", "media", "manufacturing", "logistics"],
|
|
1099
|
+
employee_count: ["1-10", "11-50", "51-200", "201-1000", "1001-5000", "5000+"],
|
|
1100
|
+
arr_bucket: ["<10k", "10k-50k", "50k-200k", "200k-1M", "1M+"],
|
|
1101
|
+
}
|
|
1102
|
+
},
|
|
1103
|
+
|
|
1104
|
+
lookupTables: [],
|
|
1105
|
+
|
|
1106
|
+
hook(record, type, meta) {
|
|
1107
|
+
if (type === "event") return handleEventHooks(record);
|
|
1108
|
+
if (type === "user") return handleUserHooks(record);
|
|
1109
|
+
if (type === "funnel-post") return handleFunnelPostHooks(record, meta);
|
|
1110
|
+
if (type === "everything") return handleEverythingHooks(record, meta);
|
|
1111
|
+
return record;
|
|
1112
|
+
}
|
|
1113
|
+
};
|
|
1114
|
+
|
|
1115
|
+
export default config;
|
|
1116
|
+
|
|
1117
|
+
// ── STORIES (verification contract — consumed by sass.verify.mjs) ──
|
|
1118
|
+
/*
|
|
1119
|
+
* DERIVATION NOTES (all numbers measured at 2K reduced scale, seed
|
|
1120
|
+
* harness-sass, plus an exact organic counterfactual run — same seed,
|
|
1121
|
+
* identity hook — used to separate engineered effects from the organic
|
|
1122
|
+
* activity curve):
|
|
1123
|
+
*
|
|
1124
|
+
* - H1a: upgrades/day 18.8 in-window vs 4.76 rest = 3.95x; in-window
|
|
1125
|
+
* plan_upgraded share 0.447 vs 0.113 baseline (1/8 of event_type array).
|
|
1126
|
+
* - H1b: invites/day 128.1 vs 82.4 = 1.55x.
|
|
1127
|
+
* - H2: hash cohort 368/1952 event-users = 18.9%, all born days 1-30;
|
|
1128
|
+
* silent-before-Feb-4 = 1.00 for hashed (H5 deploy clones reach at most
|
|
1129
|
+
* ~day 33 = last kept event + 48h), 0.00 for non-hashed; zero-event
|
|
1130
|
+
* profiles 48/2000 = 2.4% (churn-hashed users born after day 30).
|
|
1131
|
+
* - H3: esc 3067 / (3067 esc + 7124 remaining crit) = 0.301 (knob 0.30);
|
|
1132
|
+
* esc share of all incidents 3067/6881 = 0.446.
|
|
1133
|
+
* - H4: both-integ/rest avg response 24.54/62.85 = 0.390 (knob 0.4);
|
|
1134
|
+
* resolution 356.0/713.3 = 0.499 (knob 0.5).
|
|
1135
|
+
* - H5/H10 (non-churned, deploys-per-other-event): low 0.0777, sweet
|
|
1136
|
+
* 0.0979, over 0.0690 → sweet/low 1.26, over/low 0.89, over/sweet 0.70.
|
|
1137
|
+
* Organic counterfactual: over/low activity curve = 1.18; paired
|
|
1138
|
+
* per-user deltas: low 0.000 (exact), over 0.750 (exact knob).
|
|
1139
|
+
* - H6: armed down-share 0.896 (n=1394) vs unarmed 0.269 (n=6195).
|
|
1140
|
+
* - H7: recovery avg duration 1193 vs 787 = 1.52 (first-run rows with no
|
|
1141
|
+
* prior excluded).
|
|
1142
|
+
* - H8: ACV 272.6K/31.3K/7.8K/1.8K; seats 270/29.6/6.6/3.0; csm 1/0/0/0.
|
|
1143
|
+
* - H9 props: response ent 29.5 / startup 68.4 vs smb+mid avg 45.7 →
|
|
1144
|
+
* 0.646 / 1.495; resolution → 0.664 / 1.486.
|
|
1145
|
+
* - H9 funnel TTC (24h window, greedy attenuation — see Hook 9 doc):
|
|
1146
|
+
* medians ent 3.21h, mid 4.04h, smb 4.06h, startup 4.50h → ent 0.79x,
|
|
1147
|
+
* startup 1.11x vs smb/mid baseline.
|
|
1148
|
+
* - H11 per-instance (each $experiment_started anchors one attempt,
|
|
1149
|
+
* greedy min-chain within 24h): canary 0.871 (147 attempts) vs control
|
|
1150
|
+
* 0.720 (207) → lift 1.21; median TTC 34 vs 42 min = 0.81; enrolled
|
|
1151
|
+
* user split 35/74 = 0.47.
|
|
1152
|
+
* - Identity: uid share 1.0 (auth on first event), device share 0.998,
|
|
1153
|
+
* devices/user 2.08 (avgDevicePerUser: 2).
|
|
1154
|
+
*
|
|
1155
|
+
* Scale guards sit at ~50% of expected 10K populations, so 2K runs trip
|
|
1156
|
+
* WEAK by design; verdicts ship only from full-fidelity runs.
|
|
1157
|
+
*
|
|
1158
|
+
* Fix-round Q5 (2026-07-04, adversarial-review S1): NAILED bands re-derived
|
|
1159
|
+
* as knob ±10% wherever the knob converts directly to the metric; where the
|
|
1160
|
+
* realized magnitude is confounded (activity curves, greedy-evaluator
|
|
1161
|
+
* attenuation, clone-lift base rates, engine birth curves — H2 zero-event
|
|
1162
|
+
* share, H3 mix, H5 rate lift, H6 armed share, H9 funnel TTC, H10 both
|
|
1163
|
+
* legs), the assertion is a knob-bounded floor/ceiling/corridor that grades
|
|
1164
|
+
* STRONG by design. The measured values above remain as documentation of
|
|
1165
|
+
* the realized run, not as verdict targets.
|
|
1166
|
+
*/
|
|
1167
|
+
|
|
1168
|
+
const EV = `read_json_auto('{{PREFIX}}-EVENTS*.json', sample_size=-1, union_by_name=true)`;
|
|
1169
|
+
const US = `read_json_auto('{{PREFIX}}-USERS*.json', sample_size=-1, union_by_name=true)`;
|
|
1170
|
+
|
|
1171
|
+
const bandVerdict = (x, nailed, strong, detail, inverse = () => false) => {
|
|
1172
|
+
if (x == null || Number.isNaN(Number(x))) return { verdict: "NONE", detail: `${detail} — metric missing` };
|
|
1173
|
+
const v = Number(x);
|
|
1174
|
+
if (inverse(v)) return { verdict: "INVERSE", detail };
|
|
1175
|
+
if (v >= nailed[0] && v <= nailed[1]) return { verdict: "NAILED", detail };
|
|
1176
|
+
if (v >= strong[0] && v <= strong[1]) return { verdict: "STRONG", detail };
|
|
1177
|
+
return { verdict: "WEAK", detail };
|
|
1178
|
+
};
|
|
1179
|
+
const guarded = (ok, detail, inner) => ok ? inner() : { verdict: "WEAK", detail: `${detail} — cohort below scale guard (expected at reduced scale)` };
|
|
1180
|
+
const worstOf = (...verdicts) => { const order = ["INVERSE", "NONE", "WEAK", "STRONG", "NAILED"]; return order.find(o => verdicts.some(v => v.verdict === o)) || "NONE"; };
|
|
1181
|
+
const cellsOf = (rows, key) => Object.fromEntries((rows || []).map(r => [r[key], r]));
|
|
1182
|
+
|
|
1183
|
+
const EOQ_WIN = `time::TIMESTAMP >= TIMESTAMP '2026-04-11 00:00:00' AND time::TIMESTAMP <= TIMESTAMP '2026-04-21 00:00:00'`;
|
|
1184
|
+
|
|
1185
|
+
const DOC_BUCKETS_SQL = `WITH pu AS (
|
|
1186
|
+
SELECT user_id::VARCHAR AS uid,
|
|
1187
|
+
COUNT(*) FILTER (WHERE event = 'documentation viewed') AS docs,
|
|
1188
|
+
COUNT(*) FILTER (WHERE event = 'service deployed') AS deploys,
|
|
1189
|
+
COUNT(*) FILTER (WHERE event NOT IN ('service deployed', 'documentation viewed')) AS other,
|
|
1190
|
+
MAX(time::TIMESTAMP) AS last_t
|
|
1191
|
+
FROM ${EV} WHERE user_id IS NOT NULL GROUP BY 1
|
|
1192
|
+
)
|
|
1193
|
+
SELECT CASE WHEN docs >= 8 THEN 'over' WHEN docs >= 4 THEN 'sweet' ELSE 'low' END AS bucket,
|
|
1194
|
+
COUNT(*) AS users,
|
|
1195
|
+
SUM(deploys)::DOUBLE / SUM(other) AS d_per_o
|
|
1196
|
+
FROM pu WHERE last_t >= TIMESTAMP '2026-02-04 00:00:00'
|
|
1197
|
+
GROUP BY 1 ORDER BY 1`;
|
|
1198
|
+
|
|
1199
|
+
export const stories = [
|
|
1200
|
+
{
|
|
1201
|
+
id: "sass-h1-eoq-spike",
|
|
1202
|
+
hook: "H1",
|
|
1203
|
+
archetype: "temporal-inflection",
|
|
1204
|
+
narrative: "End-of-quarter (days 100-110): billing events shift toward plan_upgraded ~4x and team invites run ~1.55x.",
|
|
1205
|
+
assertions: [
|
|
1206
|
+
{
|
|
1207
|
+
breakdown: {
|
|
1208
|
+
type: "duckdb",
|
|
1209
|
+
sql: `SELECT
|
|
1210
|
+
COUNT(*) FILTER (WHERE in_win) AS bill_win,
|
|
1211
|
+
COUNT(*) FILTER (WHERE NOT in_win) AS bill_rest,
|
|
1212
|
+
COUNT(*) FILTER (WHERE in_win AND event_type = 'plan_upgraded') AS upg_win,
|
|
1213
|
+
COUNT(*) FILTER (WHERE NOT in_win AND event_type = 'plan_upgraded') AS upg_rest
|
|
1214
|
+
FROM (SELECT event_type, ${EOQ_WIN} AS in_win FROM ${EV} WHERE event = 'billing event')`,
|
|
1215
|
+
},
|
|
1216
|
+
assert: (rows) => {
|
|
1217
|
+
const r = rows?.[0] || {};
|
|
1218
|
+
return guarded(Number(r.bill_win) >= 1000 && Number(r.upg_rest) >= 1300, `billing volume: in-window=${r.bill_win ?? 0} rest-upgrades=${r.upg_rest ?? 0}`, () => {
|
|
1219
|
+
const rateRatio = (Number(r.upg_win) / 10) / (Number(r.upg_rest) / 111);
|
|
1220
|
+
const share = Number(r.upg_win) / Number(r.bill_win);
|
|
1221
|
+
const detail = `upgrades/day EOQ vs rest=${rateRatio.toFixed(2)} (knob-implied 3.8); in-window share=${share.toFixed(3)} (knob-implied 0.475, baseline 0.113)`;
|
|
1222
|
+
// Fix-round Q5 (S1): NAILED bands are knob ±10%. Implied in-window
|
|
1223
|
+
// share = 0.40 + 0.60×0.125 = 0.475 → rate ratio 0.475/0.125 = 3.8
|
|
1224
|
+
// → [3.42, 4.18]; share 0.475 → [0.43, 0.52].
|
|
1225
|
+
const legRate = bandVerdict(rateRatio, [3.42, 4.18], [2.9, 5.2], detail, v => v <= 1.3);
|
|
1226
|
+
const legShare = bandVerdict(share, [0.43, 0.52], [0.35, 0.55], detail, v => v <= 0.16);
|
|
1227
|
+
return { verdict: worstOf(legRate, legShare), detail };
|
|
1228
|
+
});
|
|
1229
|
+
},
|
|
1230
|
+
},
|
|
1231
|
+
{
|
|
1232
|
+
breakdown: {
|
|
1233
|
+
type: "duckdb",
|
|
1234
|
+
sql: `SELECT
|
|
1235
|
+
COUNT(*) FILTER (WHERE in_win) AS inv_win,
|
|
1236
|
+
COUNT(*) FILTER (WHERE NOT in_win) AS inv_rest
|
|
1237
|
+
FROM (SELECT ${EOQ_WIN} AS in_win FROM ${EV} WHERE event = 'team member invited')`,
|
|
1238
|
+
},
|
|
1239
|
+
assert: (rows) => {
|
|
1240
|
+
const r = rows?.[0] || {};
|
|
1241
|
+
return guarded(Number(r.inv_win) >= 3200, `in-window invites=${r.inv_win ?? 0}`, () => {
|
|
1242
|
+
const ratio = (Number(r.inv_win) / 10) / (Number(r.inv_rest) / 111);
|
|
1243
|
+
const detail = `invites/day EOQ vs rest=${ratio.toFixed(3)} (50% clone knob → 1.5x)`;
|
|
1244
|
+
// Fix-round Q5 (S1): knob 1 + 0.5 = 1.5x → NAILED = knob ±10%.
|
|
1245
|
+
return bandVerdict(ratio, [1.35, 1.65], [1.32, 1.80], detail, v => v <= 1.05);
|
|
1246
|
+
});
|
|
1247
|
+
},
|
|
1248
|
+
},
|
|
1249
|
+
],
|
|
1250
|
+
},
|
|
1251
|
+
{
|
|
1252
|
+
id: "sass-h2-churn-silence",
|
|
1253
|
+
hook: "H2",
|
|
1254
|
+
archetype: "retention-divergence",
|
|
1255
|
+
narrative: "~20% hash cohort goes fully silent after day 30; late-born churners appear as zero-event profiles.",
|
|
1256
|
+
assertions: [
|
|
1257
|
+
{
|
|
1258
|
+
breakdown: {
|
|
1259
|
+
type: "duckdb",
|
|
1260
|
+
sql: `WITH ue AS (
|
|
1261
|
+
SELECT user_id::VARCHAR AS uid, MAX(time::TIMESTAMP) AS last_t
|
|
1262
|
+
FROM ${EV} WHERE user_id IS NOT NULL GROUP BY 1
|
|
1263
|
+
)
|
|
1264
|
+
SELECT (list_sum([ascii(x) for x in string_split(uid, '')]) % 5 = 0) AS churn_hash,
|
|
1265
|
+
COUNT(*) AS users,
|
|
1266
|
+
AVG((last_t < TIMESTAMP '2026-02-04 00:00:00')::INT) AS silent_share
|
|
1267
|
+
FROM ue GROUP BY 1 ORDER BY 1`,
|
|
1268
|
+
},
|
|
1269
|
+
assert: (rows) => {
|
|
1270
|
+
const by = cellsOf(rows, "churn_hash");
|
|
1271
|
+
const hashed = by.true, rest = by.false;
|
|
1272
|
+
return guarded(Number(hashed?.users) >= 900, `hash cohort=${hashed?.users ?? 0}`, () => {
|
|
1273
|
+
const silentT = Number(hashed.silent_share), silentF = Number(rest?.silent_share);
|
|
1274
|
+
const share = Number(hashed.users) / (Number(hashed.users) + Number(rest?.users || 0));
|
|
1275
|
+
const detail = `hashed silent(<Feb 4)=${silentT.toFixed(3)} (expect 1.0), non-hashed=${silentF.toFixed(3)} (expect 0), cohort share=${share.toFixed(3)} (measured 0.189)`;
|
|
1276
|
+
const legT = bandVerdict(silentT, [0.97, 1.0], [0.90, 1.0], detail, v => v < 0.3);
|
|
1277
|
+
const legF = bandVerdict(silentF, [0, 0.02], [0, 0.05], detail, v => v >= 0.5);
|
|
1278
|
+
// Fix-round Q5 (S1): hash knob mod 5 → 0.20 → NAILED = knob ±10%
|
|
1279
|
+
// (event-visible share sits near the low edge — late-born
|
|
1280
|
+
// zero-event churners never appear in event data).
|
|
1281
|
+
const legShare = bandVerdict(share, [0.18, 0.22], [0.14, 0.25], detail);
|
|
1282
|
+
return { verdict: worstOf(legT, legF, legShare), detail };
|
|
1283
|
+
});
|
|
1284
|
+
},
|
|
1285
|
+
},
|
|
1286
|
+
{
|
|
1287
|
+
breakdown: {
|
|
1288
|
+
type: "duckdb",
|
|
1289
|
+
sql: `SELECT (SELECT COUNT(*) FROM ${US}) AS profiles,
|
|
1290
|
+
(SELECT COUNT(DISTINCT user_id::VARCHAR) FROM ${EV} WHERE user_id IS NOT NULL) AS event_users`,
|
|
1291
|
+
},
|
|
1292
|
+
assert: (rows) => {
|
|
1293
|
+
const r = rows?.[0] || {};
|
|
1294
|
+
return guarded(Number(r.profiles) >= 5000, `profiles=${r.profiles ?? 0}`, () => {
|
|
1295
|
+
const share = 1 - Number(r.event_users) / Number(r.profiles);
|
|
1296
|
+
const detail = `zero-event profile share=${share.toFixed(4)} (churn-hashed users born after day 30; hash-cohort ceiling 0.20)`;
|
|
1297
|
+
// Fix-round Q5 (S1): share = P(churn-hash) × P(born after day 30),
|
|
1298
|
+
// and the birth-time curve is engine behavior, not a knob — the
|
|
1299
|
+
// magnitude is not knob-derivable. Knob-derived corridor (0 <
|
|
1300
|
+
// share ≤ 0.20 hash-cohort ceiling) grades STRONG by design.
|
|
1301
|
+
if (share === 0) return { verdict: "NONE", detail: `${detail} — no zero-event profiles; late-born churn signal absent` };
|
|
1302
|
+
if (share > 0 && share <= 0.20) return { verdict: "STRONG", detail };
|
|
1303
|
+
return { verdict: "WEAK", detail };
|
|
1304
|
+
});
|
|
1305
|
+
},
|
|
1306
|
+
},
|
|
1307
|
+
],
|
|
1308
|
+
},
|
|
1309
|
+
{
|
|
1310
|
+
id: "sass-h3-alert-escalation",
|
|
1311
|
+
hook: "H3",
|
|
1312
|
+
archetype: "composition-drift",
|
|
1313
|
+
narrative: "30% of critical/emergency alerts are replaced by escalated incidents carrying alert_id; organic incidents have NULL alert_id.",
|
|
1314
|
+
assertions: [
|
|
1315
|
+
{
|
|
1316
|
+
breakdown: {
|
|
1317
|
+
type: "duckdb",
|
|
1318
|
+
sql: `SELECT
|
|
1319
|
+
COUNT(*) FILTER (WHERE event = 'incident created' AND alert_id IS NOT NULL) AS esc,
|
|
1320
|
+
COUNT(*) FILTER (WHERE event = 'incident created' AND alert_id IS NULL) AS organic,
|
|
1321
|
+
COUNT(*) FILTER (WHERE event = 'alert triggered' AND severity IN ('critical', 'emergency')) AS crit
|
|
1322
|
+
FROM ${EV}`,
|
|
1323
|
+
},
|
|
1324
|
+
assert: (rows) => {
|
|
1325
|
+
const r = rows?.[0] || {};
|
|
1326
|
+
return guarded(Number(r.esc) >= 7500, `escalated incidents=${r.esc ?? 0}`, () => {
|
|
1327
|
+
const escRate = Number(r.esc) / (Number(r.esc) + Number(r.crit));
|
|
1328
|
+
const escOfInc = Number(r.esc) / (Number(r.esc) + Number(r.organic));
|
|
1329
|
+
const detail = `esc/(esc+remaining crit)=${escRate.toFixed(3)} (knob 0.30); esc share of incidents=${escOfInc.toFixed(3)} (mix confounded — corridor check)`;
|
|
1330
|
+
const legRate = bandVerdict(escRate, [0.27, 0.33], [0.24, 0.37], detail, v => v < 0.05);
|
|
1331
|
+
// Fix-round Q5 (S1): escalated/organic mix depends on funnel-driven
|
|
1332
|
+
// alert volume vs weight-1 soup, not on a knob (naive weight math
|
|
1333
|
+
// gives ~0.64, far from the realized value). Corridor sanity check
|
|
1334
|
+
// grades STRONG by design; legRate carries the knob-derived 0.30
|
|
1335
|
+
// invariant.
|
|
1336
|
+
const legMix = escOfInc >= 0.2 && escOfInc <= 0.8 && Number(r.organic) > 0
|
|
1337
|
+
? { verdict: "STRONG", detail }
|
|
1338
|
+
: { verdict: "WEAK", detail };
|
|
1339
|
+
return { verdict: worstOf(legRate, legMix), detail };
|
|
1340
|
+
});
|
|
1341
|
+
},
|
|
1342
|
+
},
|
|
1343
|
+
],
|
|
1344
|
+
},
|
|
1345
|
+
{
|
|
1346
|
+
id: "sass-h4-integration-speed",
|
|
1347
|
+
hook: "H4",
|
|
1348
|
+
archetype: "cohort-prop-scale",
|
|
1349
|
+
narrative: "Users with both Slack and PagerDuty configured respond 0.4x and resolve 0.5x vs the rest.",
|
|
1350
|
+
assertions: [
|
|
1351
|
+
{
|
|
1352
|
+
breakdown: {
|
|
1353
|
+
type: "duckdb",
|
|
1354
|
+
sql: `WITH integ AS (
|
|
1355
|
+
SELECT user_id::VARCHAR AS uid,
|
|
1356
|
+
BOOL_OR(event = 'integration configured' AND integration_type = 'slack') AS s,
|
|
1357
|
+
BOOL_OR(event = 'integration configured' AND integration_type = 'pagerduty') AS p
|
|
1358
|
+
FROM ${EV} WHERE user_id IS NOT NULL GROUP BY 1
|
|
1359
|
+
)
|
|
1360
|
+
SELECT (s AND p) AS both_integ, COUNT(DISTINCT e.user_id::VARCHAR) AS users,
|
|
1361
|
+
AVG(response_time_mins) FILTER (WHERE event = 'alert acknowledged') AS avg_resp,
|
|
1362
|
+
AVG(resolution_time_mins) FILTER (WHERE event = 'alert resolved') AS avg_reso
|
|
1363
|
+
FROM ${EV} e JOIN integ i ON e.user_id::VARCHAR = i.uid GROUP BY 1 ORDER BY 1`,
|
|
1364
|
+
},
|
|
1365
|
+
assert: (rows) => {
|
|
1366
|
+
const by = cellsOf(rows, "both_integ");
|
|
1367
|
+
const both = by.true, rest = by.false;
|
|
1368
|
+
return guarded(Number(both?.users) >= 1100 && Number(rest?.users) >= 3700, `cohorts: both=${both?.users ?? 0} rest=${rest?.users ?? 0}`, () => {
|
|
1369
|
+
const respRatio = Number(both.avg_resp) / Number(rest.avg_resp);
|
|
1370
|
+
const resoRatio = Number(both.avg_reso) / Number(rest.avg_reso);
|
|
1371
|
+
const detail = `both/rest response=${respRatio.toFixed(3)} (knob 0.4, measured 0.390); resolution=${resoRatio.toFixed(3)} (knob 0.5, measured 0.499)`;
|
|
1372
|
+
// Fix-round Q5 (S1): NAILED = knob ±10% (0.4 → [0.36, 0.44],
|
|
1373
|
+
// 0.5 → [0.45, 0.55]).
|
|
1374
|
+
const legResp = bandVerdict(respRatio, [0.36, 0.44], [0.31, 0.50], detail, v => v >= 0.95);
|
|
1375
|
+
const legReso = bandVerdict(resoRatio, [0.45, 0.55], [0.40, 0.62], detail, v => v >= 0.95);
|
|
1376
|
+
return { verdict: worstOf(legResp, legReso), detail };
|
|
1377
|
+
});
|
|
1378
|
+
},
|
|
1379
|
+
},
|
|
1380
|
+
],
|
|
1381
|
+
},
|
|
1382
|
+
{
|
|
1383
|
+
id: "sass-h5-docs-deploy-lift",
|
|
1384
|
+
hook: "H5",
|
|
1385
|
+
archetype: "frequency-sweet-spot",
|
|
1386
|
+
narrative: "Sweet-spot doc readers (4-7 views) get cloned deploys: deploys-per-other-event lifts above the low bucket (measured ~1.26x; asserted as a knob-derived floor >1.05 — the rate magnitude is activity-confounded, fix-round Q5).",
|
|
1387
|
+
assertions: [
|
|
1388
|
+
{
|
|
1389
|
+
breakdown: { type: "duckdb", sql: DOC_BUCKETS_SQL },
|
|
1390
|
+
assert: (rows) => {
|
|
1391
|
+
const by = cellsOf(rows, "bucket");
|
|
1392
|
+
const sweet = by.sweet, low = by.low;
|
|
1393
|
+
return guarded(Number(sweet?.users) >= 1800 && Number(low?.users) >= 950, `buckets: sweet=${sweet?.users ?? 0} low=${low?.users ?? 0}`, () => {
|
|
1394
|
+
const ratio = Number(sweet.d_per_o) / Number(low.d_per_o);
|
|
1395
|
+
const detail = `sweet/low deploys-per-other=${ratio.toFixed(3)} (+2.5 clones engineered; rate lift not knob-derivable)`;
|
|
1396
|
+
// Fix-round Q5 (S1): the +2.5-clone knob converts to a RATE ratio
|
|
1397
|
+
// only through the organic deploy base rate, the future-guard
|
|
1398
|
+
// shave, and the activity curve — none knob-derivable. Knob-derived
|
|
1399
|
+
// floor (clones strictly add deploys → sweet/low > 1.05) grades
|
|
1400
|
+
// STRONG by design; INVERSE at ≤1.02 (no lift).
|
|
1401
|
+
if (ratio <= 1.02) return { verdict: "INVERSE", detail };
|
|
1402
|
+
if (ratio > 1.05) return { verdict: "STRONG", detail };
|
|
1403
|
+
return { verdict: "WEAK", detail };
|
|
1404
|
+
});
|
|
1405
|
+
},
|
|
1406
|
+
},
|
|
1407
|
+
],
|
|
1408
|
+
},
|
|
1409
|
+
{
|
|
1410
|
+
id: "sass-h6-cost-overrun",
|
|
1411
|
+
hook: "H6",
|
|
1412
|
+
archetype: "bespoke",
|
|
1413
|
+
narrative: "After a >25% cost spike, the user's next infrastructure scale is forced down: armed-state down-share ≥0.75 floor (mechanism-implied 1.0, read-side order gap attenuates; measured ~0.90) vs the 0.25-array baseline (knob ±10%).",
|
|
1414
|
+
assertions: [
|
|
1415
|
+
{
|
|
1416
|
+
breakdown: {
|
|
1417
|
+
type: "duckdb",
|
|
1418
|
+
sql: `WITH seq AS (
|
|
1419
|
+
SELECT user_id::VARCHAR AS uid, time::TIMESTAMP AS t, event, cost_change_percent, scale_direction
|
|
1420
|
+
FROM ${EV} WHERE event IN ('cost report generated', 'infrastructure scaled') AND user_id IS NOT NULL
|
|
1421
|
+
), marked AS (
|
|
1422
|
+
SELECT *,
|
|
1423
|
+
MAX(CASE WHEN event = 'cost report generated' AND cost_change_percent > 25 THEN t END)
|
|
1424
|
+
OVER (PARTITION BY uid ORDER BY t ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) AS last_spike,
|
|
1425
|
+
MAX(CASE WHEN event = 'infrastructure scaled' THEN t END)
|
|
1426
|
+
OVER (PARTITION BY uid ORDER BY t ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) AS last_infra
|
|
1427
|
+
FROM seq
|
|
1428
|
+
)
|
|
1429
|
+
SELECT (last_spike IS NOT NULL AND (last_infra IS NULL OR last_spike > last_infra)) AS armed,
|
|
1430
|
+
COUNT(*) AS n, AVG((scale_direction = 'down')::INT) AS down_share
|
|
1431
|
+
FROM marked WHERE event = 'infrastructure scaled' GROUP BY 1 ORDER BY 1`,
|
|
1432
|
+
},
|
|
1433
|
+
assert: (rows) => {
|
|
1434
|
+
const by = cellsOf(rows, "armed");
|
|
1435
|
+
const armed = by.true, unarmed = by.false;
|
|
1436
|
+
return guarded(Number(armed?.n) >= 3400 && Number(unarmed?.n) >= 15000, `scale events: armed=${armed?.n ?? 0} unarmed=${unarmed?.n ?? 0}`, () => {
|
|
1437
|
+
const a = Number(armed.down_share), u = Number(unarmed.down_share);
|
|
1438
|
+
const detail = `down-share armed=${a.toFixed(3)} (mechanism-implied 1.0; time-vs-generation order gap attenuates) vs unarmed=${u.toFixed(3)} (baseline 0.25 array)`;
|
|
1439
|
+
// Fix-round Q5 (S1): the mechanism forces scale_direction="down"
|
|
1440
|
+
// (implied 1.0), but the read-side generation-vs-timestamp order
|
|
1441
|
+
// gap attenuates it non-derivably. Knob-derived floor (≥0.75 = 3x
|
|
1442
|
+
// the 0.25 array baseline) grades STRONG by design.
|
|
1443
|
+
const legArmed = a <= 0.35 ? { verdict: "INVERSE", detail }
|
|
1444
|
+
: a >= 0.75 ? { verdict: "STRONG", detail }
|
|
1445
|
+
: { verdict: "WEAK", detail };
|
|
1446
|
+
// Fix-round Q5 (S1): NAILED = knob ±10% (0.25 → [0.225, 0.275]).
|
|
1447
|
+
const legBase = bandVerdict(u, [0.225, 0.275], [0.21, 0.34], detail);
|
|
1448
|
+
return { verdict: worstOf(legArmed, legBase), detail };
|
|
1449
|
+
});
|
|
1450
|
+
},
|
|
1451
|
+
},
|
|
1452
|
+
],
|
|
1453
|
+
},
|
|
1454
|
+
{
|
|
1455
|
+
id: "sass-h7-deploy-recovery",
|
|
1456
|
+
hook: "H7",
|
|
1457
|
+
archetype: "bespoke",
|
|
1458
|
+
narrative: "The first successful pipeline run after a failure runs ~1.5x longer.",
|
|
1459
|
+
assertions: [
|
|
1460
|
+
{
|
|
1461
|
+
breakdown: {
|
|
1462
|
+
type: "duckdb",
|
|
1463
|
+
sql: `WITH p AS (
|
|
1464
|
+
SELECT user_id::VARCHAR AS uid, status, duration_sec,
|
|
1465
|
+
LAG(status) OVER (PARTITION BY user_id::VARCHAR ORDER BY time) AS prev
|
|
1466
|
+
FROM ${EV} WHERE event = 'deployment pipeline run' AND user_id IS NOT NULL
|
|
1467
|
+
)
|
|
1468
|
+
SELECT (prev = 'failed') AS recovery, COUNT(*) AS n, AVG(duration_sec) AS avg_dur
|
|
1469
|
+
FROM p WHERE status = 'success' AND prev IS NOT NULL GROUP BY 1 ORDER BY 1`,
|
|
1470
|
+
},
|
|
1471
|
+
assert: (rows) => {
|
|
1472
|
+
const by = cellsOf(rows, "recovery");
|
|
1473
|
+
const rec = by.true, other = by.false;
|
|
1474
|
+
return guarded(Number(rec?.n) >= 3400 && Number(other?.n) >= 8000, `runs: recovery=${rec?.n ?? 0} other=${other?.n ?? 0}`, () => {
|
|
1475
|
+
const ratio = Number(rec.avg_dur) / Number(other.avg_dur);
|
|
1476
|
+
const detail = `recovery/other success duration=${ratio.toFixed(3)} (knob 1.5, measured 1.52)`;
|
|
1477
|
+
return bandVerdict(ratio, [1.43, 1.60], [1.33, 1.72], detail, v => v <= 1.03);
|
|
1478
|
+
});
|
|
1479
|
+
},
|
|
1480
|
+
},
|
|
1481
|
+
],
|
|
1482
|
+
},
|
|
1483
|
+
{
|
|
1484
|
+
id: "sass-h8-enterprise-startup",
|
|
1485
|
+
hook: "H8",
|
|
1486
|
+
archetype: "cohort-prop-scale",
|
|
1487
|
+
narrative: "Company size drives ACV (~$273K enterprise to ~$1.8K startup), seat count, and CSM assignment (enterprise only).",
|
|
1488
|
+
assertions: [
|
|
1489
|
+
{
|
|
1490
|
+
breakdown: {
|
|
1491
|
+
type: "duckdb",
|
|
1492
|
+
sql: `SELECT company_size, COUNT(*) AS users, AVG(annual_contract_value) AS acv,
|
|
1493
|
+
AVG(seat_count) AS seats, AVG(customer_success_manager::INT) AS csm
|
|
1494
|
+
FROM ${US} GROUP BY 1 ORDER BY 1`,
|
|
1495
|
+
},
|
|
1496
|
+
assert: (rows) => {
|
|
1497
|
+
const by = cellsOf(rows, "company_size");
|
|
1498
|
+
const ent = by.enterprise, mid = by.mid_market, smb = by.smb, st = by.startup;
|
|
1499
|
+
const sizes = `ent=${ent?.users ?? 0} mid=${mid?.users ?? 0} smb=${smb?.users ?? 0} startup=${st?.users ?? 0}`;
|
|
1500
|
+
return guarded(
|
|
1501
|
+
Number(ent?.users) >= 1000 && Number(mid?.users) >= 1000 && Number(smb?.users) >= 950 && Number(st?.users) >= 1900,
|
|
1502
|
+
`segments: ${sizes}`,
|
|
1503
|
+
() => {
|
|
1504
|
+
const detail = `ACV ent=${Math.round(ent.acv)} mid=${Math.round(mid.acv)} smb=${Math.round(smb.acv)} startup=${Math.round(st.acv)}; seats ${Number(ent.seats).toFixed(1)}/${Number(mid.seats).toFixed(1)}/${Number(smb.seats).toFixed(1)}/${Number(st.seats).toFixed(1)}; csm ${Number(ent.csm).toFixed(3)}/${Number(mid.csm).toFixed(3)}/${Number(smb.csm).toFixed(3)}/${Number(st.csm).toFixed(3)}`;
|
|
1505
|
+
// Fix-round Q5 (S1): NAILED = uniform-range mean ±10% (ent 275K,
|
|
1506
|
+
// mid 31K, smb 7.8K, startup 1.8K).
|
|
1507
|
+
const legEnt = bandVerdict(ent.acv, [255000, 295000], [235000, 315000], detail, v => v < 50000);
|
|
1508
|
+
const legMid = bandVerdict(mid.acv, [27900, 34100], [26000, 37500], detail);
|
|
1509
|
+
const legSmb = bandVerdict(smb.acv, [7200, 8400], [6500, 9200], detail);
|
|
1510
|
+
const legSt = bandVerdict(st.acv, [1620, 1980], [1300, 2300], detail);
|
|
1511
|
+
const seatsMonotonic = Number(ent.seats) > Number(mid.seats) && Number(mid.seats) > Number(smb.seats) && Number(smb.seats) > Number(st.seats);
|
|
1512
|
+
const legSeats = { verdict: seatsMonotonic ? "NAILED" : "INVERSE", detail };
|
|
1513
|
+
const csmOthers = Math.max(Number(mid.csm), Number(smb.csm), Number(st.csm));
|
|
1514
|
+
const legCsm = Number(ent.csm) === 1 && csmOthers === 0 ? { verdict: "NAILED", detail }
|
|
1515
|
+
: Number(ent.csm) >= 0.99 && csmOthers <= 0.01 ? { verdict: "STRONG", detail }
|
|
1516
|
+
: Number(ent.csm) < 0.5 ? { verdict: "INVERSE", detail }
|
|
1517
|
+
: { verdict: "WEAK", detail };
|
|
1518
|
+
return { verdict: worstOf(legEnt, legMid, legSmb, legSt, legSeats, legCsm), detail };
|
|
1519
|
+
}
|
|
1520
|
+
);
|
|
1521
|
+
},
|
|
1522
|
+
},
|
|
1523
|
+
],
|
|
1524
|
+
},
|
|
1525
|
+
{
|
|
1526
|
+
id: "sass-h9-incident-ttc",
|
|
1527
|
+
hook: "H9",
|
|
1528
|
+
archetype: "funnel-ttc-by-segment",
|
|
1529
|
+
narrative: "Enterprise responds/resolves 0.67x, startup 1.5x (property legs carry the knob ±10% NAILED read; funnel TTC attenuates asymmetrically under greedy evaluation and is asserted as knob-bounded corridors grading STRONG — fix-round Q5).",
|
|
1530
|
+
assertions: [
|
|
1531
|
+
{
|
|
1532
|
+
breakdown: {
|
|
1533
|
+
type: "duckdb",
|
|
1534
|
+
sql: `SELECT u.company_size AS seg,
|
|
1535
|
+
COUNT(*) FILTER (WHERE event = 'alert acknowledged') AS n_ack,
|
|
1536
|
+
AVG(response_time_mins) FILTER (WHERE event = 'alert acknowledged') AS resp,
|
|
1537
|
+
AVG(resolution_time_mins) FILTER (WHERE event = 'alert resolved') AS reso
|
|
1538
|
+
FROM ${EV} e JOIN ${US} u ON e.user_id::VARCHAR = u.distinct_id::VARCHAR
|
|
1539
|
+
GROUP BY 1 ORDER BY 1`,
|
|
1540
|
+
},
|
|
1541
|
+
assert: (rows) => {
|
|
1542
|
+
const by = cellsOf(rows, "seg");
|
|
1543
|
+
const ent = by.enterprise, mid = by.mid_market, smb = by.smb, st = by.startup;
|
|
1544
|
+
const counts = `ent=${ent?.n_ack ?? 0} mid=${mid?.n_ack ?? 0} smb=${smb?.n_ack ?? 0} startup=${st?.n_ack ?? 0}`;
|
|
1545
|
+
return guarded(
|
|
1546
|
+
[ent, mid, smb, st].every(s => Number(s?.n_ack) >= 7500),
|
|
1547
|
+
`acks by segment: ${counts}`,
|
|
1548
|
+
() => {
|
|
1549
|
+
const respBase = (Number(mid.resp) + Number(smb.resp)) / 2;
|
|
1550
|
+
const resoBase = (Number(mid.reso) + Number(smb.reso)) / 2;
|
|
1551
|
+
const respEnt = Number(ent.resp) / respBase, respSt = Number(st.resp) / respBase;
|
|
1552
|
+
const resoEnt = Number(ent.reso) / resoBase, resoSt = Number(st.reso) / resoBase;
|
|
1553
|
+
const detail = `property legs vs smb/mid avg: response ent=${respEnt.toFixed(3)} startup=${respSt.toFixed(3)} (knobs 0.67/1.5, measured 0.646/1.495); resolution ent=${resoEnt.toFixed(3)} startup=${resoSt.toFixed(3)} (measured 0.664/1.486)`;
|
|
1554
|
+
const legs = [
|
|
1555
|
+
bandVerdict(respEnt, [0.60, 0.70], [0.55, 0.76], detail, v => v >= 0.95),
|
|
1556
|
+
bandVerdict(respSt, [1.40, 1.59], [1.30, 1.71], detail, v => v <= 1.02),
|
|
1557
|
+
bandVerdict(resoEnt, [0.61, 0.71], [0.56, 0.77], detail, v => v >= 0.95),
|
|
1558
|
+
bandVerdict(resoSt, [1.39, 1.58], [1.29, 1.70], detail, v => v <= 1.02),
|
|
1559
|
+
];
|
|
1560
|
+
return { verdict: worstOf(...legs), detail };
|
|
1561
|
+
}
|
|
1562
|
+
);
|
|
1563
|
+
},
|
|
1564
|
+
},
|
|
1565
|
+
{
|
|
1566
|
+
breakdown: {
|
|
1567
|
+
type: "timeToConvert",
|
|
1568
|
+
steps: ["alert triggered", "alert acknowledged", "alert resolved"],
|
|
1569
|
+
breakdownByUserProperty: "company_size",
|
|
1570
|
+
conversionWindowMs: 24 * 3600 * 1000,
|
|
1571
|
+
},
|
|
1572
|
+
assert: (rows) => {
|
|
1573
|
+
const by = cellsOf(rows, "segment_value");
|
|
1574
|
+
const ent = by.enterprise, mid = by.mid_market, smb = by.smb, st = by.startup;
|
|
1575
|
+
const counts = `ent=${ent?.user_count ?? 0} mid=${mid?.user_count ?? 0} smb=${smb?.user_count ?? 0} startup=${st?.user_count ?? 0}`;
|
|
1576
|
+
return guarded(
|
|
1577
|
+
Number(ent?.user_count) >= 540 && Number(mid?.user_count) >= 580 && Number(smb?.user_count) >= 480 && Number(st?.user_count) >= 1100,
|
|
1578
|
+
`converters: ${counts}`,
|
|
1579
|
+
() => {
|
|
1580
|
+
const base = (Number(mid.median_ttc_ms) + Number(smb.median_ttc_ms)) / 2;
|
|
1581
|
+
const entRatio = Number(ent.median_ttc_ms) / base;
|
|
1582
|
+
const stRatio = Number(st.median_ttc_ms) / base;
|
|
1583
|
+
const detail = `funnel median TTC @24h vs smb/mid: ent=${entRatio.toFixed(3)} startup=${stRatio.toFixed(3)} (greedy attenuation of 0.67/1.5 knobs — corridor check; medians ${(Number(ent.median_ttc_ms) / 3600000).toFixed(2)}h/${(base / 3600000).toFixed(2)}h/${(Number(st.median_ttc_ms) / 3600000).toFixed(2)}h)`;
|
|
1584
|
+
// Fix-round Q5 (S1): greedy-evaluator attenuation of the 0.67/1.5
|
|
1585
|
+
// knobs is not knob-derivable (organic soup alerts win the greedy
|
|
1586
|
+
// race). Knob-bounded corridors — the true effect lies between
|
|
1587
|
+
// the full knob (±10%) and no-effect 1.0 — grade STRONG by
|
|
1588
|
+
// design; the property legs above carry the knob-derived NAILED
|
|
1589
|
+
// read.
|
|
1590
|
+
const legEnt = entRatio >= 1.0 ? { verdict: "INVERSE", detail }
|
|
1591
|
+
: entRatio >= 0.603 && entRatio <= 0.95 ? { verdict: "STRONG", detail }
|
|
1592
|
+
: { verdict: "WEAK", detail };
|
|
1593
|
+
const legSt = stRatio <= 0.98 ? { verdict: "INVERSE", detail }
|
|
1594
|
+
: stRatio > 1.03 && stRatio <= 1.65 ? { verdict: "STRONG", detail }
|
|
1595
|
+
: { verdict: "WEAK", detail };
|
|
1596
|
+
return { verdict: worstOf(legEnt, legSt), detail };
|
|
1597
|
+
}
|
|
1598
|
+
);
|
|
1599
|
+
},
|
|
1600
|
+
},
|
|
1601
|
+
],
|
|
1602
|
+
},
|
|
1603
|
+
{
|
|
1604
|
+
id: "sass-h10-docs-magic-number",
|
|
1605
|
+
hook: "H10",
|
|
1606
|
+
archetype: "frequency-sweet-spot",
|
|
1607
|
+
narrative: "Over-engaged doc readers (8+) lose 25% of deploys: asserted as knob-bounded corridors (over/low ∈ [0.675, 1.0), over/sweet ≤ 0.825) grading STRONG — the realized ratios compose the 0.75 keep-rate knob with a non-derivable activity curve and clone lift (fix-round Q5).",
|
|
1608
|
+
assertions: [
|
|
1609
|
+
{
|
|
1610
|
+
breakdown: { type: "duckdb", sql: DOC_BUCKETS_SQL },
|
|
1611
|
+
assert: (rows) => {
|
|
1612
|
+
const by = cellsOf(rows, "bucket");
|
|
1613
|
+
const over = by.over, sweet = by.sweet, low = by.low;
|
|
1614
|
+
return guarded(
|
|
1615
|
+
Number(over?.users) >= 1150 && Number(sweet?.users) >= 1800 && Number(low?.users) >= 950,
|
|
1616
|
+
`buckets: over=${over?.users ?? 0} sweet=${sweet?.users ?? 0} low=${low?.users ?? 0}`,
|
|
1617
|
+
() => {
|
|
1618
|
+
const overLow = Number(over.d_per_o) / Number(low.d_per_o);
|
|
1619
|
+
const overSweet = Number(over.d_per_o) / Number(sweet.d_per_o);
|
|
1620
|
+
const detail = `deploys-per-other over/low=${overLow.toFixed(3)} (0.75 keep-rate knob x non-derivable activity curve — corridor check); over/sweet=${overSweet.toFixed(3)} (0.75 knob ÷ non-derivable clone lift)`;
|
|
1621
|
+
// Fix-round Q5 (S1): the realized ratios compose the 0.75
|
|
1622
|
+
// keep-rate knob with the organic activity curve (over-readers
|
|
1623
|
+
// are more active) and the H5 clone lift — both measured, not
|
|
1624
|
+
// knob-derivable. Knob-bounded corridors grade STRONG by design:
|
|
1625
|
+
// over/low ∈ [0.675 (knob −10%), 1.0) — the drop can only reduce,
|
|
1626
|
+
// the activity curve alone would push ≥1; over/sweet ≤ 0.825
|
|
1627
|
+
// (= 0.75 × 1.1, assuming clone lift ≥ 1).
|
|
1628
|
+
const legLow = overLow >= 1.12 ? { verdict: "INVERSE", detail }
|
|
1629
|
+
: overLow >= 0.675 && overLow < 1.0 ? { verdict: "STRONG", detail }
|
|
1630
|
+
: { verdict: "WEAK", detail };
|
|
1631
|
+
const legSweet = overSweet >= 0.97 ? { verdict: "INVERSE", detail }
|
|
1632
|
+
: overSweet > 0 && overSweet <= 0.825 ? { verdict: "STRONG", detail }
|
|
1633
|
+
: { verdict: "WEAK", detail };
|
|
1634
|
+
return { verdict: worstOf(legLow, legSweet), detail };
|
|
1635
|
+
}
|
|
1636
|
+
);
|
|
1637
|
+
},
|
|
1638
|
+
},
|
|
1639
|
+
],
|
|
1640
|
+
},
|
|
1641
|
+
{
|
|
1642
|
+
id: "sass-h11-canary-experiment",
|
|
1643
|
+
hook: "H11",
|
|
1644
|
+
archetype: "experiment-lift",
|
|
1645
|
+
narrative: "Canary Deploys experiment: per-instance conversion lift ~1.21x (knob 1.2) and median TTC ~0.81x; even enrollment split. Identity invariants ride as the final assertion.",
|
|
1646
|
+
assertions: [
|
|
1647
|
+
{
|
|
1648
|
+
breakdown: {
|
|
1649
|
+
type: "duckdb",
|
|
1650
|
+
sql: `WITH exp AS (
|
|
1651
|
+
SELECT user_id::VARCHAR AS uid, time::TIMESTAMP AS t0, "Variant name" AS variant
|
|
1652
|
+
FROM ${EV} WHERE event = '$experiment_started'
|
|
1653
|
+
), ev2 AS (
|
|
1654
|
+
SELECT user_id::VARCHAR AS uid, event, time::TIMESTAMP AS t FROM ${EV}
|
|
1655
|
+
WHERE event IN ('deployment pipeline run', 'service deployed', 'dashboard viewed')
|
|
1656
|
+
), c1 AS (
|
|
1657
|
+
SELECT x.uid, x.variant, x.t0,
|
|
1658
|
+
(SELECT MIN(t) FROM ev2 e WHERE e.uid = x.uid AND e.event = 'deployment pipeline run' AND e.t >= x.t0) AS tp
|
|
1659
|
+
FROM exp x
|
|
1660
|
+
), c2 AS (
|
|
1661
|
+
SELECT c.*, (SELECT MIN(t) FROM ev2 e WHERE e.uid = c.uid AND e.event = 'service deployed' AND e.t >= c.tp) AS td
|
|
1662
|
+
FROM c1 c
|
|
1663
|
+
), c3 AS (
|
|
1664
|
+
SELECT c.*, (SELECT MIN(t) FROM ev2 e WHERE e.uid = c.uid AND e.event = 'dashboard viewed' AND e.t >= c.td) AS tb
|
|
1665
|
+
FROM c2 c
|
|
1666
|
+
)
|
|
1667
|
+
SELECT variant, COUNT(*) AS attempts, COUNT(DISTINCT uid) AS users,
|
|
1668
|
+
AVG((tb IS NOT NULL AND tb <= t0 + INTERVAL 24 HOUR)::INT) AS conv_rate,
|
|
1669
|
+
median(CASE WHEN tb IS NOT NULL AND tb <= t0 + INTERVAL 24 HOUR THEN date_diff('minute', tp, tb) END) AS med_ttc_min
|
|
1670
|
+
FROM c3 GROUP BY 1 ORDER BY 1`,
|
|
1671
|
+
},
|
|
1672
|
+
assert: (rows) => {
|
|
1673
|
+
const by = cellsOf(rows, "variant");
|
|
1674
|
+
const canary = by["Canary Deploys"], control = by.Control;
|
|
1675
|
+
return guarded(
|
|
1676
|
+
Number(canary?.attempts) >= 360 && Number(control?.attempts) >= 500,
|
|
1677
|
+
`attempts: canary=${canary?.attempts ?? 0} control=${control?.attempts ?? 0}`,
|
|
1678
|
+
() => {
|
|
1679
|
+
const lift = Number(canary.conv_rate) / Number(control.conv_rate);
|
|
1680
|
+
const ttcRatio = Number(canary.med_ttc_min) / Number(control.med_ttc_min);
|
|
1681
|
+
const detail = `per-instance conversion lift=${lift.toFixed(3)} (knob 1.2, measured 1.21; rates ${Number(canary.conv_rate).toFixed(3)}/${Number(control.conv_rate).toFixed(3)}); median TTC ratio=${ttcRatio.toFixed(3)} (knob 0.85, measured 0.81)`;
|
|
1682
|
+
// Fix-round Q5 (S1): NAILED = knob ±10% (1.2 → [1.08, 1.32],
|
|
1683
|
+
// 0.85 → [0.765, 0.935]).
|
|
1684
|
+
const legLift = bandVerdict(lift, [1.08, 1.32], [1.04, 1.42], detail, v => v <= 1.00);
|
|
1685
|
+
const legTtc = bandVerdict(ttcRatio, [0.765, 0.935], [0.65, 0.98], detail, v => v >= 1.03);
|
|
1686
|
+
return { verdict: worstOf(legLift, legTtc), detail };
|
|
1687
|
+
}
|
|
1688
|
+
);
|
|
1689
|
+
},
|
|
1690
|
+
},
|
|
1691
|
+
{
|
|
1692
|
+
breakdown: {
|
|
1693
|
+
type: "duckdb",
|
|
1694
|
+
sql: `SELECT "Variant name" AS variant, COUNT(DISTINCT user_id::VARCHAR) AS users
|
|
1695
|
+
FROM ${EV} WHERE event = '$experiment_started' GROUP BY 1 ORDER BY 1`,
|
|
1696
|
+
},
|
|
1697
|
+
assert: (rows) => {
|
|
1698
|
+
const by = cellsOf(rows, "variant");
|
|
1699
|
+
const canary = Number(by["Canary Deploys"]?.users || 0), control = Number(by.Control?.users || 0);
|
|
1700
|
+
return guarded(canary + control >= 180, `enrolled users=${canary + control}`, () => {
|
|
1701
|
+
const split = canary / (canary + control);
|
|
1702
|
+
const detail = `enrollment split canary=${split.toFixed(3)} of ${canary + control} users (deterministic 2-arm hash → 0.50)`;
|
|
1703
|
+
// Fix-round Q5 (S1): implied split 0.50 → NAILED = knob ±10%.
|
|
1704
|
+
return bandVerdict(split, [0.45, 0.55], [0.35, 0.62], detail);
|
|
1705
|
+
});
|
|
1706
|
+
},
|
|
1707
|
+
},
|
|
1708
|
+
{
|
|
1709
|
+
breakdown: {
|
|
1710
|
+
type: "duckdb",
|
|
1711
|
+
sql: `SELECT COUNT(*) AS n,
|
|
1712
|
+
AVG((user_id IS NOT NULL)::INT) AS uid_share,
|
|
1713
|
+
AVG((device_id IS NOT NULL)::INT) AS device_share,
|
|
1714
|
+
COUNT(DISTINCT device_id)::DOUBLE / COUNT(DISTINCT user_id) AS devices_per_user
|
|
1715
|
+
FROM ${EV}`,
|
|
1716
|
+
},
|
|
1717
|
+
assert: (rows) => {
|
|
1718
|
+
const r = rows?.[0] || {};
|
|
1719
|
+
return guarded(Number(r.n) >= 500000, `events=${r.n ?? 0}`, () => {
|
|
1720
|
+
const uid = Number(r.uid_share), dev = Number(r.device_share), dpu = Number(r.devices_per_user);
|
|
1721
|
+
const detail = `identity invariants: uid_share=${uid} device_share=${dev.toFixed(4)} devices/user=${dpu.toFixed(2)} over ${r.n} events (auth on first event; avgDevicePerUser: 2)`;
|
|
1722
|
+
// Fix-round Q5 (S1): dpu NAILED = knob ±10% (avgDevicePerUser: 2
|
|
1723
|
+
// → [1.8, 2.2]).
|
|
1724
|
+
if (uid === 1 && dev >= 0.99 && dpu >= 1.8 && dpu <= 2.2) return { verdict: "NAILED", detail };
|
|
1725
|
+
if (uid >= 0.999 && dev >= 0.98) return { verdict: "STRONG", detail };
|
|
1726
|
+
if (uid < 0.9) return { verdict: "INVERSE", detail };
|
|
1727
|
+
return { verdict: "WEAK", detail };
|
|
1728
|
+
});
|
|
1729
|
+
},
|
|
1730
|
+
},
|
|
1731
|
+
],
|
|
1732
|
+
},
|
|
1733
|
+
];
|