@pithy-sh/testers 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +17 -0
- package/docs/store-apis.md +107 -0
- package/package.json +62 -0
- package/pithy.manifest.json +52 -0
- package/src/activity/resolve.ts +273 -0
- package/src/audit/actions.ts +56 -0
- package/src/capability.ts +128 -0
- package/src/clock/days.ts +70 -0
- package/src/clock/replay.ts +190 -0
- package/src/cloudflare-test.d.ts +13 -0
- package/src/config/config.ts +518 -0
- package/src/crypto/token.ts +60 -0
- package/src/data/cohort.ts +83 -0
- package/src/data/enums.ts +134 -0
- package/src/data/event.ts +81 -0
- package/src/data/member.ts +79 -0
- package/src/data/snapshot.ts +280 -0
- package/src/data/tables.ts +49 -0
- package/src/error/errors.ts +229 -0
- package/src/health/score.ts +225 -0
- package/src/http/guards.ts +37 -0
- package/src/http/pages.ts +66 -0
- package/src/http/responses.ts +634 -0
- package/src/http/routes.ts +933 -0
- package/src/http/schemas.ts +210 -0
- package/src/http/scopes.ts +79 -0
- package/src/http/view.ts +304 -0
- package/src/index.ts +80 -0
- package/src/migrations/0001_cohorts.ts +202 -0
- package/src/nudge/cooldown.ts +104 -0
- package/src/nudge/copy.ts +179 -0
- package/src/nudge/enqueueSeam.ts +95 -0
- package/src/nudge/send.ts +89 -0
- package/src/projection/build.ts +285 -0
- package/src/projection/forecast.ts +348 -0
- package/src/projection/inputs.ts +63 -0
- package/src/projection/poissonBinomial.ts +91 -0
- package/src/projection/trend.ts +185 -0
- package/src/provision/provisionTesters.ts +109 -0
- package/src/provision/resolveTestersConfig.ts +155 -0
- package/src/roster/read.ts +227 -0
- package/src/roster/write.ts +511 -0
- package/src/seeds/example.ts +219 -0
- package/src/version.generated.ts +16 -0
- package/src/workflows/daily.ts +513 -0
- package/src/workflows/pass.ts +100 -0
- package/src/workflows/report.ts +52 -0
- package/src/workflows/retryPolicy.ts +48 -0
- package/src/workflows/specs.ts +73 -0
- package/src/workflows/worker.ts +132 -0
- package/src/workflows/wrangler.jsonc +66 -0
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { addDays, type DayKey } from "../clock/days";
|
|
5
|
+
import type { Observability, ProjectionBasis, ProjectionConfidence, RiskBand } from "../data/enums";
|
|
6
|
+
import { expectedSurvivors, probabilityAtLeast, survivalOverDays } from "./poissonBinomial";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The forecast: how likely this cohort is to finish its window, and what to do about it.
|
|
10
|
+
*
|
|
11
|
+
* **Success needs two independent things to go right, and they are reported separately.** Reaching the
|
|
12
|
+
* target is one problem — do enough of the people you invited actually confirm? Holding it is another —
|
|
13
|
+
* do enough of the people who confirmed stay confirmed for the remaining days? Multiplying them into a
|
|
14
|
+
* single percentage destroys the only actionable information in the pair. "62%" tells a developer
|
|
15
|
+
* nothing they can act on; "you will reach twelve (95%) but only hold it 65% of the time" tells them to
|
|
16
|
+
* invite four more people this afternoon.
|
|
17
|
+
*
|
|
18
|
+
* **The whole thing degrades honestly, and the mechanism is structural rather than a disclaimer.** When
|
|
19
|
+
* nothing about a cohort is observable — which is the normal case for an app whose test flow never asks
|
|
20
|
+
* anyone to sign in — `successProbability` is `null` with a stated basis, not a plausible-looking 0.5.
|
|
21
|
+
* When coverage is partial, the reported band widens in exact proportion to how blind we are, by
|
|
22
|
+
* re-running the same computation with the unobservable testers pinned pessimistic and optimistic. A
|
|
23
|
+
* dashboard cannot render a wide band as a confident number, which is the point: the band's width *is*
|
|
24
|
+
* the disclosure.
|
|
25
|
+
*
|
|
26
|
+
* And none of it is Google's number. Google computes the authoritative opt-in streak and exposes it
|
|
27
|
+
* through no API at all. Everything here forecasts Pithy's own estimate.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
/** Nothing is certain until Google says so, and Google is not talking. */
|
|
31
|
+
const MAX_REPORTED_PROBABILITY = 0.99;
|
|
32
|
+
|
|
33
|
+
/** The survival rate an unobservable tester is assumed to have when we are being pessimistic. */
|
|
34
|
+
const PESSIMISTIC_UNKNOWN_SURVIVAL = 0.95;
|
|
35
|
+
|
|
36
|
+
/** The survival rate an unobservable tester is assumed to have when we are being optimistic. */
|
|
37
|
+
const OPTIMISTIC_UNKNOWN_SURVIVAL = 0.999;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The per-day survival assumed for a tester who has not confirmed yet, on a cohort with nobody to
|
|
41
|
+
* average. Matches the healthy prior: someone who has not arrived cannot yet have gone quiet.
|
|
42
|
+
*/
|
|
43
|
+
const DEFAULT_PADDING_SURVIVAL = 0.998;
|
|
44
|
+
|
|
45
|
+
/** Coverage at or above this, on a cohort with some history, is worth calling high confidence. */
|
|
46
|
+
const HIGH_CONFIDENCE_COVERAGE = 0.75;
|
|
47
|
+
|
|
48
|
+
/** Coverage at or above this is worth calling moderate. */
|
|
49
|
+
const MODERATE_CONFIDENCE_COVERAGE = 0.4;
|
|
50
|
+
|
|
51
|
+
/** A cohort younger than this has not accumulated enough days for high confidence, however visible it is. */
|
|
52
|
+
const HIGH_CONFIDENCE_MIN_AGE_DAYS = 7;
|
|
53
|
+
|
|
54
|
+
/** The assumed invite-to-opt-in latency before a cohort has converted anyone, in days. */
|
|
55
|
+
const DEFAULT_CONVERSION_LATENCY_DAYS = 3;
|
|
56
|
+
|
|
57
|
+
/** How many conversions a cohort needs before its own observed latency beats the prior. */
|
|
58
|
+
const MIN_CONVERSIONS_FOR_OBSERVED_LATENCY = 5;
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* How much likelier an accepted tester is to convert than a merely-invited one.
|
|
62
|
+
*
|
|
63
|
+
* Someone who has answered the first email has demonstrably opened it and said yes — nothing more, since
|
|
64
|
+
* answering needs no account and installs nothing. That is still a far better predictor than an invitation
|
|
65
|
+
* nobody has replied to,
|
|
66
|
+
* and authenticated. They are further along than someone who has not opened the email at all, and
|
|
67
|
+
* treating the two identically would make the pipeline estimate uselessly pessimistic for exactly the
|
|
68
|
+
* cohorts that are going well.
|
|
69
|
+
*/
|
|
70
|
+
const ACCEPTED_CONVERSION_MULTIPLIER = 1.6;
|
|
71
|
+
|
|
72
|
+
/** However promising, nobody is a certainty to convert until they have. */
|
|
73
|
+
const MAX_PER_MEMBER_CONVERSION = 0.95;
|
|
74
|
+
|
|
75
|
+
/** One opted-in tester, reduced to what the forecast needs. */
|
|
76
|
+
export interface ForecastMember {
|
|
77
|
+
readonly riskBand: RiskBand;
|
|
78
|
+
readonly observability: Observability;
|
|
79
|
+
/** The published daily-survival prior their band selects. */
|
|
80
|
+
readonly dailySurvival: number;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** One tester who has not opted in yet, and how close they are to doing so. */
|
|
84
|
+
export interface PipelineMember {
|
|
85
|
+
/** `accepted` testers have answered the first email agreeing to test; `invited` ones have not replied. */
|
|
86
|
+
readonly stage: "invited" | "accepted";
|
|
87
|
+
/** Whether their address bounced. An unreachable invitee converts at zero — we cannot even chase them. */
|
|
88
|
+
readonly unreachable: boolean;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Everything the forecast needs. All of it is derived; none of it is stored. */
|
|
92
|
+
export interface ForecastInput {
|
|
93
|
+
/** The day being forecast from. */
|
|
94
|
+
readonly today: DayKey;
|
|
95
|
+
/** How many testers must be opted in simultaneously. */
|
|
96
|
+
readonly targetSize: number;
|
|
97
|
+
/** Pithy's estimate of how many are opted in right now. */
|
|
98
|
+
readonly optedInCount: number;
|
|
99
|
+
/** Window days still to hold, on Pithy's estimate. */
|
|
100
|
+
readonly daysRemaining: number;
|
|
101
|
+
/** The currently opted-in testers, with their bands. */
|
|
102
|
+
readonly members: readonly ForecastMember[];
|
|
103
|
+
/** Testers still in the pipeline — invited or accepted, not yet confirmed. */
|
|
104
|
+
readonly pipeline: readonly PipelineMember[];
|
|
105
|
+
/** How many members have ever reached `opted_in`, for the conversion rate. */
|
|
106
|
+
readonly optedInEver: number;
|
|
107
|
+
/** How many members have ever been invited, for the conversion rate. */
|
|
108
|
+
readonly invitedEver: number;
|
|
109
|
+
/** The observed median days from invitation to opt-in, or null before enough conversions exist. */
|
|
110
|
+
readonly medianConversionDays: number | null;
|
|
111
|
+
/** How many conversions that median was computed from. */
|
|
112
|
+
readonly conversionSampleSize: number;
|
|
113
|
+
/** How many days old the cohort is. Drives confidence. */
|
|
114
|
+
readonly cohortAgeDays: number;
|
|
115
|
+
/** The cohort's roster cap, so the over-provisioning advice cannot exceed it. */
|
|
116
|
+
readonly maxRosterSize: number;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Pithy's forecast for one cohort. Every field is an estimate; none of it reads Google. */
|
|
120
|
+
export interface Forecast {
|
|
121
|
+
readonly basis: ProjectionBasis;
|
|
122
|
+
readonly confidence: ProjectionConfidence | null;
|
|
123
|
+
/** Share of opted-in testers we can see at all, 0–1. The honest denominator behind everything else. */
|
|
124
|
+
readonly observedCoverage: number;
|
|
125
|
+
readonly probabilityReachTarget: number;
|
|
126
|
+
readonly probabilityHoldWindow: number | null;
|
|
127
|
+
readonly successProbability: number | null;
|
|
128
|
+
readonly successProbabilityLow: number | null;
|
|
129
|
+
readonly successProbabilityHigh: number | null;
|
|
130
|
+
readonly expectedSurvivors: number;
|
|
131
|
+
readonly projectedTargetMetOn: DayKey | null;
|
|
132
|
+
readonly projectedCompleteOn: DayKey | null;
|
|
133
|
+
/** How many more people to invite to close the gap at this cohort's own conversion rate. */
|
|
134
|
+
readonly invitesNeeded: number;
|
|
135
|
+
/** The roster size that survives the window at this cohort's own conversion and drop-off rates. */
|
|
136
|
+
readonly recommendedRosterSize: number;
|
|
137
|
+
/** Always `default` today: the survival priors are numbers Pithy chose, not values fitted to your data. */
|
|
138
|
+
readonly calibration: "default";
|
|
139
|
+
/** The named, versioned method. A chart must not mix methods across one series. */
|
|
140
|
+
readonly method: "poisson_binomial_v1";
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* The share of invitations that have turned into opt-ins, Laplace-smoothed.
|
|
145
|
+
*
|
|
146
|
+
* The smoothing is not decoration. A cohort whose first invitee confirmed has converted one of one, and
|
|
147
|
+
* reporting a 100% conversion rate off a single data point would tell a developer they need to invite
|
|
148
|
+
* exactly twelve people — which is the advice most likely to leave them at eleven on day fourteen.
|
|
149
|
+
* Adding one to the numerator and two to the denominator pulls a tiny sample toward a half and lets the
|
|
150
|
+
* real rate assert itself as evidence accumulates.
|
|
151
|
+
*/
|
|
152
|
+
export function conversionRate(optedInEver: number, invitedEver: number): number {
|
|
153
|
+
return (optedInEver + 1) / (invitedEver + 2);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** How likely one pipeline member is to convert, given the cohort's observed rate. */
|
|
157
|
+
function memberConversion(member: PipelineMember, rate: number): number {
|
|
158
|
+
if (member.unreachable) return 0;
|
|
159
|
+
if (member.stage === "invited") return Math.min(MAX_PER_MEMBER_CONVERSION, rate);
|
|
160
|
+
return Math.min(MAX_PER_MEMBER_CONVERSION, rate * ACCEPTED_CONVERSION_MULTIPLIER);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Confidence, from how much of the cohort is visible and how long it has been running. */
|
|
164
|
+
function confidenceFor(coverage: number, ageDays: number): ProjectionConfidence | null {
|
|
165
|
+
// Zero coverage is not weak confidence. It is no opinion at all, and it must be representable as one.
|
|
166
|
+
if (coverage <= 0) return null;
|
|
167
|
+
if (coverage >= HIGH_CONFIDENCE_COVERAGE && ageDays >= HIGH_CONFIDENCE_MIN_AGE_DAYS) return "high";
|
|
168
|
+
if (coverage >= MODERATE_CONFIDENCE_COVERAGE) return "moderate";
|
|
169
|
+
return "low";
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Run the hold computation with unobservable testers pinned to a given survival rate.
|
|
174
|
+
*
|
|
175
|
+
* **Hold answers a conditional question: given the target is reached, does it hold?** That distinction
|
|
176
|
+
* is the whole reason `P(success) = P(reach) × P(hold)` is a valid decomposition. Asking instead
|
|
177
|
+
* "do the testers who have already confirmed hold the target?" makes the two halves dependent, and
|
|
178
|
+
* worse, makes hold identically zero for every cohort below target — because `probabilityAtLeast`
|
|
179
|
+
* correctly returns zero when you ask for more survivors than there are people. Since a cohort is below
|
|
180
|
+
* target for most of its life, that would zero out the product's headline number almost always, and
|
|
181
|
+
* report it with a zero-width band at high confidence: exactly the false certainty the band exists to
|
|
182
|
+
* prevent.
|
|
183
|
+
*
|
|
184
|
+
* So a below-target cohort is padded with the testers it still needs, each carrying the mean survival
|
|
185
|
+
* of the people already in. It is an assumption, and a mild one: a tester who has not confirmed yet is
|
|
186
|
+
* assumed to behave like the ones who have.
|
|
187
|
+
*/
|
|
188
|
+
function holdProbability(
|
|
189
|
+
members: readonly ForecastMember[],
|
|
190
|
+
targetSize: number,
|
|
191
|
+
daysRemaining: number,
|
|
192
|
+
unknownSurvival: number | null,
|
|
193
|
+
): number {
|
|
194
|
+
const probabilities = members.map((member) => {
|
|
195
|
+
const daily =
|
|
196
|
+
unknownSurvival !== null && member.observability !== "observed" ? unknownSurvival : member.dailySurvival;
|
|
197
|
+
return survivalOverDays(daily, daysRemaining);
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
const deficit = targetSize - probabilities.length;
|
|
201
|
+
if (deficit > 0) {
|
|
202
|
+
// The mean of who is already in, or the healthy prior when nobody is — a brand-new cohort has no
|
|
203
|
+
// behavior to average, and assuming the worst of people who have not arrived yet would report a
|
|
204
|
+
// fresh cohort as doomed on its first day.
|
|
205
|
+
const mean =
|
|
206
|
+
probabilities.length > 0
|
|
207
|
+
? probabilities.reduce((sum, p) => sum + p, 0) / probabilities.length
|
|
208
|
+
: survivalOverDays(DEFAULT_PADDING_SURVIVAL, daysRemaining);
|
|
209
|
+
for (let i = 0; i < deficit; i++) probabilities.push(mean);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
return probabilityAtLeast(probabilities, targetSize);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** Cap a probability below one, so the response can never claim certainty. */
|
|
216
|
+
function cap(probability: number): number {
|
|
217
|
+
return Math.min(MAX_REPORTED_PROBABILITY, probability);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** Forecast one cohort. */
|
|
221
|
+
export function forecastCohort(input: ForecastInput): Forecast {
|
|
222
|
+
const observed = input.members.filter((member) => member.observability === "observed").length;
|
|
223
|
+
const coverage = input.members.length === 0 ? 0 : observed / input.members.length;
|
|
224
|
+
const rate = conversionRate(input.optedInEver, input.invitedEver);
|
|
225
|
+
|
|
226
|
+
const reachable = input.pipeline.filter((member) => !member.unreachable);
|
|
227
|
+
const conversions = reachable.map((member) => memberConversion(member, rate));
|
|
228
|
+
const deficit = Math.max(0, input.targetSize - input.optedInCount);
|
|
229
|
+
|
|
230
|
+
const probabilityReachTarget = deficit === 0 ? 1 : probabilityAtLeast(conversions, deficit);
|
|
231
|
+
const expectedConversions = conversions.reduce((sum, probability) => sum + probability, 0);
|
|
232
|
+
|
|
233
|
+
// How many more people to invite so the *expected* conversions cover the gap. Reported even when the
|
|
234
|
+
// probability looks survivable, because it is the one number a developer can act on this afternoon.
|
|
235
|
+
const invitesNeeded =
|
|
236
|
+
deficit === 0 ? 0 : Math.max(0, Math.ceil((deficit - expectedConversions) / Math.max(rate, 0.01)));
|
|
237
|
+
|
|
238
|
+
const latency =
|
|
239
|
+
input.medianConversionDays !== null && input.conversionSampleSize >= MIN_CONVERSIONS_FOR_OBSERVED_LATENCY
|
|
240
|
+
? input.medianConversionDays
|
|
241
|
+
: DEFAULT_CONVERSION_LATENCY_DAYS;
|
|
242
|
+
|
|
243
|
+
// Nothing observable at all: say so, rather than emit a number that looks like knowledge. This is the
|
|
244
|
+
// normal case for an app whose test flow never asks anyone to sign in, so it has to be a first-class
|
|
245
|
+
// answer rather than an edge case.
|
|
246
|
+
if (input.members.length > 0 && coverage === 0) {
|
|
247
|
+
return {
|
|
248
|
+
basis: "no_observable_signal",
|
|
249
|
+
confidence: null,
|
|
250
|
+
observedCoverage: 0,
|
|
251
|
+
probabilityReachTarget,
|
|
252
|
+
probabilityHoldWindow: null,
|
|
253
|
+
successProbability: null,
|
|
254
|
+
successProbabilityLow: null,
|
|
255
|
+
successProbabilityHigh: null,
|
|
256
|
+
expectedSurvivors: expectedSurvivors(
|
|
257
|
+
input.members.map((member) => survivalOverDays(member.dailySurvival, input.daysRemaining)),
|
|
258
|
+
),
|
|
259
|
+
projectedTargetMetOn: deficit === 0 ? input.today : null,
|
|
260
|
+
projectedCompleteOn: null,
|
|
261
|
+
invitesNeeded,
|
|
262
|
+
// The same drop-off factor the observed path uses. Passing 1 here would make the advice *less*
|
|
263
|
+
// conservative exactly when Pithy is blind, and would contradict `expectedSurvivors` two lines
|
|
264
|
+
// above, which already reads each member's own prior.
|
|
265
|
+
recommendedRosterSize: recommendRoster(input, rate, holdSurvivalFactor(input)),
|
|
266
|
+
calibration: "default",
|
|
267
|
+
method: "poisson_binomial_v1",
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
const hold = holdProbability(input.members, input.targetSize, input.daysRemaining, null);
|
|
272
|
+
const holdLow = holdProbability(input.members, input.targetSize, input.daysRemaining, PESSIMISTIC_UNKNOWN_SURVIVAL);
|
|
273
|
+
const holdHigh = holdProbability(input.members, input.targetSize, input.daysRemaining, OPTIMISTIC_UNKNOWN_SURVIVAL);
|
|
274
|
+
|
|
275
|
+
const survivors = expectedSurvivors(
|
|
276
|
+
input.members.map((member) => survivalOverDays(member.dailySurvival, input.daysRemaining)),
|
|
277
|
+
);
|
|
278
|
+
|
|
279
|
+
const targetMetOn =
|
|
280
|
+
deficit === 0 ? input.today : expectedConversions >= deficit ? addDays(input.today, Math.ceil(latency)) : null;
|
|
281
|
+
// `daysRemaining` means "further days after the day that produced this count" — `heldDays` counts the
|
|
282
|
+
// at-target run inclusive of today. So on the at-target branch `targetMetOn` is today, itself already
|
|
283
|
+
// held, and adding the whole remainder is right. On the below-target branch `targetMetOn` is the
|
|
284
|
+
// *first* held day, so the window closes one day earlier than the same arithmetic suggests. Without
|
|
285
|
+
// the adjustment the projected finish silently moved a day earlier the moment a cohort reached
|
|
286
|
+
// target, with nothing about the cohort having changed. The floor covers `resetPolicy: "pause"`,
|
|
287
|
+
// where a cohort can be below target with the remainder already at zero.
|
|
288
|
+
const completeOn =
|
|
289
|
+
targetMetOn === null
|
|
290
|
+
? null
|
|
291
|
+
: deficit === 0
|
|
292
|
+
? addDays(targetMetOn, input.daysRemaining)
|
|
293
|
+
: addDays(targetMetOn, Math.max(0, input.daysRemaining - 1));
|
|
294
|
+
|
|
295
|
+
const basis: ProjectionBasis =
|
|
296
|
+
input.cohortAgeDays === 0 && input.optedInCount === 0
|
|
297
|
+
? "no_history"
|
|
298
|
+
: deficit === 0 && input.daysRemaining === 0
|
|
299
|
+
? "target_met"
|
|
300
|
+
: targetMetOn === null
|
|
301
|
+
? "insufficient_pipeline"
|
|
302
|
+
: "estimated";
|
|
303
|
+
|
|
304
|
+
return {
|
|
305
|
+
basis,
|
|
306
|
+
confidence: confidenceFor(coverage, input.cohortAgeDays),
|
|
307
|
+
observedCoverage: coverage,
|
|
308
|
+
probabilityReachTarget,
|
|
309
|
+
probabilityHoldWindow: hold,
|
|
310
|
+
successProbability: cap(probabilityReachTarget * hold),
|
|
311
|
+
// Clamped around the point estimate rather than trusted to order themselves. The bracket constants
|
|
312
|
+
// are fixed while the priors they bracket are adopter-configurable, so a project that sets
|
|
313
|
+
// `survival.unknown` below 0.95 would otherwise report a point estimate outside its own band — and
|
|
314
|
+
// a chart drawing that gets a line escaping its own error bar.
|
|
315
|
+
successProbabilityLow: cap(probabilityReachTarget * Math.min(hold, holdLow)),
|
|
316
|
+
successProbabilityHigh: cap(probabilityReachTarget * Math.max(hold, holdHigh)),
|
|
317
|
+
expectedSurvivors: survivors,
|
|
318
|
+
projectedTargetMetOn: targetMetOn,
|
|
319
|
+
projectedCompleteOn: completeOn,
|
|
320
|
+
invitesNeeded,
|
|
321
|
+
recommendedRosterSize: recommendRoster(input, rate, holdSurvivalFactor(input)),
|
|
322
|
+
calibration: "default",
|
|
323
|
+
method: "poisson_binomial_v1",
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** The average per-tester chance of lasting the window — the drop-off half of the over-provisioning sum. */
|
|
328
|
+
function holdSurvivalFactor(input: ForecastInput): number {
|
|
329
|
+
if (input.members.length === 0) return 1;
|
|
330
|
+
const total = input.members.reduce(
|
|
331
|
+
(sum, member) => sum + survivalOverDays(member.dailySurvival, Math.max(input.daysRemaining, 1)),
|
|
332
|
+
0,
|
|
333
|
+
);
|
|
334
|
+
return Math.max(total / input.members.length, 0.01);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* The roster size that actually survives the window.
|
|
339
|
+
*
|
|
340
|
+
* "Carry twelve" is the advice that fails, because twelve is the number that must still be standing at
|
|
341
|
+
* the end rather than the number to start with. Dividing the target by both the conversion rate and the
|
|
342
|
+
* expected survival gives the number to start with — the answer to the question a developer is really
|
|
343
|
+
* asking, which is "how many people do I need to ask?"
|
|
344
|
+
*/
|
|
345
|
+
function recommendRoster(input: ForecastInput, rate: number, survival: number): number {
|
|
346
|
+
const raw = Math.ceil(input.targetSize / Math.max(survival, 0.01) / Math.max(rate, 0.01));
|
|
347
|
+
return Math.min(input.maxRosterSize, Math.max(input.targetSize, raw));
|
|
348
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { type DayKey, dayKey, daysBetween } from "../clock/days";
|
|
5
|
+
import type { TestersMember } from "../data/member";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The two forecast inputs derived from the roster rather than passed in, in one place.
|
|
9
|
+
*
|
|
10
|
+
* `forecastCohort` has two callers — `buildSnapshot`, which writes the daily row, and `toCohortView`,
|
|
11
|
+
* which answers `GET /testers/cohorts` — and they must hand it identical inputs or the card and the
|
|
12
|
+
* chart give a developer two different completion dates for the same cohort on the same morning. They
|
|
13
|
+
* did. The view passed `medianConversionDays: null, conversionSampleSize: 0`, which can never clear
|
|
14
|
+
* the five-conversion evidence gate, so every live read used the three-day default prior no matter how
|
|
15
|
+
* many conversions the cohort had actually measured; and it computed the cohort's age from elapsed
|
|
16
|
+
* milliseconds while the builder used UTC day keys, so the two disagreed by one for most of any day.
|
|
17
|
+
*
|
|
18
|
+
* Both are now derived here, and both callers call these functions. Divergence is a change to this
|
|
19
|
+
* file rather than something that can happen by omission.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/** The measured invite-to-opt-in latency, and how much evidence stands behind it. */
|
|
23
|
+
export interface ConversionLatency {
|
|
24
|
+
/** The median days from invitation to opt-in, or null when nobody has converted yet. */
|
|
25
|
+
readonly medianConversionDays: number | null;
|
|
26
|
+
/** How many conversions that median is drawn from — the forecast gates on this before trusting it. */
|
|
27
|
+
readonly conversionSampleSize: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** The middle value, rounded on an even count. Shared, because the health series takes a median too. */
|
|
31
|
+
export function median(values: readonly number[]): number | null {
|
|
32
|
+
if (values.length === 0) return null;
|
|
33
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
34
|
+
const middle = Math.floor(sorted.length / 2);
|
|
35
|
+
if (sorted.length % 2 === 1) return sorted[middle] ?? null;
|
|
36
|
+
return Math.round(((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* How long this cohort's own testers take to go from invited to opted in.
|
|
41
|
+
*
|
|
42
|
+
* Day keys rather than elapsed milliseconds, so an invitation at 23:00 and an opt-in at 01:00 counts
|
|
43
|
+
* as one day rather than zero. Negative spans are dropped rather than clamped: they mean the two
|
|
44
|
+
* timestamps disagree, and a corrupt row should not pull the median toward zero.
|
|
45
|
+
*/
|
|
46
|
+
export function conversionLatency(members: readonly TestersMember[]): ConversionLatency {
|
|
47
|
+
const conversions = members
|
|
48
|
+
.filter((member) => member.optedInAt !== null)
|
|
49
|
+
.map((member) => daysBetween(dayKey(member.invitedAt), dayKey(member.optedInAt as Date)))
|
|
50
|
+
.filter((days) => days >= 0);
|
|
51
|
+
return { medianConversionDays: median(conversions), conversionSampleSize: conversions.length };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* How old the cohort is, in whole UTC days.
|
|
56
|
+
*
|
|
57
|
+
* Day keys, because the forecast reads this at two hard thresholds — seven days for `high` confidence,
|
|
58
|
+
* zero days for a `no_history` basis — and an hour of drift either side of a threshold is a different
|
|
59
|
+
* answer on the wire. The whole package indexes on UTC day keys for exactly this reason.
|
|
60
|
+
*/
|
|
61
|
+
export function cohortAgeDays(createdAt: Date, today: DayKey): number {
|
|
62
|
+
return Math.max(0, daysBetween(dayKey(createdAt), today));
|
|
63
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The exact distribution of "how many of these testers are still opted in when the window closes".
|
|
6
|
+
*
|
|
7
|
+
* Each tester has their own survival probability — a tester who opens the app daily and one who has
|
|
8
|
+
* been dark for nine days are not the same coin — so the count of survivors is Poisson-binomial rather
|
|
9
|
+
* than binomial, and there is no closed form for it.
|
|
10
|
+
*
|
|
11
|
+
* **Exact dynamic programming, not a normal approximation and not Monte Carlo.** The DP is one
|
|
12
|
+
* convolution per tester, so a hundred-person roster is about ten thousand multiply-adds: fast enough
|
|
13
|
+
* to run inside a request, and it buys three things an approximation would not. It is deterministic,
|
|
14
|
+
* so the same roster always yields the same number and a test can assert an exact value rather than a
|
|
15
|
+
* tolerance. It is exact in the tail, which is the only region anyone cares about — "at least twelve of
|
|
16
|
+
* fourteen" is a tail question, and the normal approximation is at its worst there, on small n with
|
|
17
|
+
* heterogeneous and near-one probabilities, which describes every cohort this package will ever see.
|
|
18
|
+
* And it is explainable in one sentence: we roll each tester's chance of lasting the remaining days into
|
|
19
|
+
* the exact odds that at least twelve of them do. A developer can audit that. Nobody can audit a seed.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The full distribution: `result[k]` is the probability that exactly `k` of the given testers survive.
|
|
24
|
+
*
|
|
25
|
+
* The array is `probabilities.length + 1` long, because zero survivors is an outcome too — and for a
|
|
26
|
+
* cohort of four people trying to hold twelve, it is not even an unlikely one.
|
|
27
|
+
*/
|
|
28
|
+
export function survivorDistribution(probabilities: readonly number[]): number[] {
|
|
29
|
+
// The empty roster has exactly one outcome: nobody survives, with certainty.
|
|
30
|
+
let distribution = [1];
|
|
31
|
+
for (const probability of probabilities) {
|
|
32
|
+
const next = new Array<number>(distribution.length + 1).fill(0);
|
|
33
|
+
for (let survivors = 0; survivors < distribution.length; survivors++) {
|
|
34
|
+
const mass = distribution[survivors] ?? 0;
|
|
35
|
+
if (mass === 0) continue;
|
|
36
|
+
// This tester either lasts, moving the mass one place up, or does not, leaving it where it is.
|
|
37
|
+
next[survivors] = (next[survivors] ?? 0) + mass * (1 - probability);
|
|
38
|
+
next[survivors + 1] = (next[survivors + 1] ?? 0) + mass * probability;
|
|
39
|
+
}
|
|
40
|
+
distribution = next;
|
|
41
|
+
}
|
|
42
|
+
return distribution;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The probability that at least `atLeast` of these testers survive.
|
|
47
|
+
*
|
|
48
|
+
* Summed from the top down rather than as `1 - P(fewer)`, because the interesting answers live in the
|
|
49
|
+
* upper tail and subtracting two near-equal numbers there loses precision exactly where the result
|
|
50
|
+
* matters most.
|
|
51
|
+
*/
|
|
52
|
+
export function probabilityAtLeast(probabilities: readonly number[], atLeast: number): number {
|
|
53
|
+
// Needing none of them is certain; needing more of them than exist is impossible. Both are worth
|
|
54
|
+
// answering directly rather than letting the sum arrive at them, so the caller never has to reason
|
|
55
|
+
// about an empty range.
|
|
56
|
+
if (atLeast <= 0) return 1;
|
|
57
|
+
if (atLeast > probabilities.length) return 0;
|
|
58
|
+
|
|
59
|
+
const distribution = survivorDistribution(probabilities);
|
|
60
|
+
let total = 0;
|
|
61
|
+
for (let survivors = distribution.length - 1; survivors >= atLeast; survivors--) {
|
|
62
|
+
total += distribution[survivors] ?? 0;
|
|
63
|
+
}
|
|
64
|
+
// Floating-point accumulation over a hundred convolutions can land a hair outside [0,1]; a
|
|
65
|
+
// probability that renders as 1.0000000000000002 is a bug report waiting to happen.
|
|
66
|
+
return Math.min(1, Math.max(0, total));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* How many testers we expect to survive — the sum of the individual probabilities.
|
|
71
|
+
*
|
|
72
|
+
* The most interpretable number the model produces, and the one that goes on the card: "we expect 11.3
|
|
73
|
+
* of your 14 to still be in on day fourteen" is a sentence a developer can act on, where a percentage
|
|
74
|
+
* is a sentence they can only feel something about.
|
|
75
|
+
*/
|
|
76
|
+
export function expectedSurvivors(probabilities: readonly number[]): number {
|
|
77
|
+
return probabilities.reduce((total, probability) => total + probability, 0);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* A tester's chance of lasting `days` more days, at a given per-day survival rate.
|
|
82
|
+
*
|
|
83
|
+
* Independence across days is assumed and stated. It is not quite true — a tester who uninstalls is
|
|
84
|
+
* gone for every subsequent day, not independently gone each day — but the per-day rate is already
|
|
85
|
+
* calibrated against that behavior rather than against a memoryless process, and a hazard model would
|
|
86
|
+
* add parameters nobody can audit to a number that is a declared prior in the first place.
|
|
87
|
+
*/
|
|
88
|
+
export function survivalOverDays(dailySurvival: number, days: number): number {
|
|
89
|
+
if (days <= 0) return 1;
|
|
90
|
+
return dailySurvival ** days;
|
|
91
|
+
}
|