@shipeasy/openapi 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-643YSMLU.js → chunk-4QSU744W.js} +183 -1
- package/dist/chunk-4QSU744W.js.map +1 -0
- package/dist/{chunk-MRW3DJ57.js → chunk-ZPKHTUVV.js} +1238 -335
- package/dist/chunk-ZPKHTUVV.js.map +1 -0
- package/dist/{client-K0ShmsC7.d.ts → client-WGNXT4xl.d.ts} +5818 -2647
- package/dist/client.d.ts +1 -1
- package/dist/client.js +51 -1
- package/dist/custom.js +4 -4
- package/dist/custom.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +52 -2
- package/dist/index.js.map +1 -1
- package/dist/schemas.d.ts +3538 -534
- package/dist/schemas.js +253 -5
- package/openapi.json +12914 -7158
- package/openapi.yaml +7672 -3742
- package/package.json +1 -1
- package/spec/components/errors.yaml +33 -1
- package/spec/components/schemas/alerts.yaml +138 -0
- package/spec/components/schemas/attributes.yaml +53 -3
- package/spec/components/schemas/common.yaml +16 -0
- package/spec/components/schemas/configs.yaml +58 -0
- package/spec/components/schemas/connectors.yaml +5 -1
- package/spec/components/schemas/errors-api.yaml +116 -6
- package/spec/components/schemas/experiment-api-row.yaml +293 -0
- package/spec/components/schemas/experiments.yaml +395 -380
- package/spec/components/schemas/gates.yaml +355 -272
- package/spec/components/schemas/i18n.yaml +169 -40
- package/spec/components/schemas/keys.yaml +2 -1
- package/spec/components/schemas/killswitches.yaml +36 -0
- package/spec/components/schemas/metrics.yaml +200 -23
- package/spec/components/schemas/ops.yaml +918 -29
- package/spec/components/schemas/projects.yaml +246 -2
- package/spec/components/schemas/universes.yaml +83 -1
- package/spec/openapi.yaml +61 -0
- package/spec/paths/alert-rules.yaml +1 -0
- package/spec/paths/alerts.yaml +184 -0
- package/spec/paths/attributes.yaml +23 -5
- package/spec/paths/configs.yaml +71 -0
- package/spec/paths/errors.yaml +112 -2
- package/spec/paths/events.yaml +1 -0
- package/spec/paths/experiments.yaml +244 -16
- package/spec/paths/gates.yaml +138 -0
- package/spec/paths/i18n.yaml +290 -6
- package/spec/paths/keys.yaml +1 -0
- package/spec/paths/killswitches.yaml +7 -0
- package/spec/paths/metrics.yaml +187 -0
- package/spec/paths/ops.yaml +410 -0
- package/spec/paths/projects.yaml +62 -2
- package/spec/paths/universes.yaml +1 -0
- package/dist/chunk-643YSMLU.js.map +0 -1
- package/dist/chunk-MRW3DJ57.js.map +0 -1
|
@@ -4,149 +4,7 @@ ListExperimentsResponse:
|
|
|
4
4
|
data:
|
|
5
5
|
type: array
|
|
6
6
|
items:
|
|
7
|
-
|
|
8
|
-
properties:
|
|
9
|
-
id:
|
|
10
|
-
type: string
|
|
11
|
-
description: Stable opaque experiment id (`exp_…`).
|
|
12
|
-
name:
|
|
13
|
-
type: string
|
|
14
|
-
maxLength: 128
|
|
15
|
-
pattern: ^[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?)?$
|
|
16
|
-
description: Stable experiment key. Single segment or `folder.name` (a-z, 0-9, `_`/`-`; max 128 chars). Used by SDKs as `Shipeasy.getExperiment(user, '<name>')`. Immutable after create.
|
|
17
|
-
description:
|
|
18
|
-
anyOf:
|
|
19
|
-
- type: string
|
|
20
|
-
- type: "null"
|
|
21
|
-
hypothesis:
|
|
22
|
-
anyOf:
|
|
23
|
-
- type: string
|
|
24
|
-
- type: "null"
|
|
25
|
-
tag:
|
|
26
|
-
anyOf:
|
|
27
|
-
- type: string
|
|
28
|
-
- type: "null"
|
|
29
|
-
ownerEmail:
|
|
30
|
-
anyOf:
|
|
31
|
-
- type: string
|
|
32
|
-
- type: "null"
|
|
33
|
-
audience:
|
|
34
|
-
anyOf:
|
|
35
|
-
- type: string
|
|
36
|
-
- type: "null"
|
|
37
|
-
bucketBy:
|
|
38
|
-
anyOf:
|
|
39
|
-
- type: string
|
|
40
|
-
minLength: 1
|
|
41
|
-
maxLength: 128
|
|
42
|
-
pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
|
|
43
|
-
description: User-attribute name used as the bucketing key — e.g. `user_id` (default), `session_id`, `device_id`, or a custom attribute like `company_id` to keep a whole org on one variant. Defaults to `user_id` when omitted.
|
|
44
|
-
- type: "null"
|
|
45
|
-
folder:
|
|
46
|
-
anyOf:
|
|
47
|
-
- type: string
|
|
48
|
-
- type: "null"
|
|
49
|
-
status:
|
|
50
|
-
type: string
|
|
51
|
-
enum:
|
|
52
|
-
- draft
|
|
53
|
-
- running
|
|
54
|
-
- stopped
|
|
55
|
-
- archived
|
|
56
|
-
universe:
|
|
57
|
-
type: string
|
|
58
|
-
description: Universe name this experiment draws from.
|
|
59
|
-
targetingGate:
|
|
60
|
-
anyOf:
|
|
61
|
-
- type: string
|
|
62
|
-
- type: "null"
|
|
63
|
-
allocationPct:
|
|
64
|
-
type: integer
|
|
65
|
-
minimum: -9007199254740991
|
|
66
|
-
maximum: 9007199254740991
|
|
67
|
-
description: Allocation in basis points (0–10000).
|
|
68
|
-
salt:
|
|
69
|
-
type: string
|
|
70
|
-
params:
|
|
71
|
-
type: object
|
|
72
|
-
additionalProperties:
|
|
73
|
-
type: string
|
|
74
|
-
enum:
|
|
75
|
-
- string
|
|
76
|
-
- bool
|
|
77
|
-
- number
|
|
78
|
-
groups:
|
|
79
|
-
type: array
|
|
80
|
-
items:
|
|
81
|
-
type: object
|
|
82
|
-
properties:
|
|
83
|
-
name:
|
|
84
|
-
type: string
|
|
85
|
-
minLength: 1
|
|
86
|
-
maxLength: 64
|
|
87
|
-
description: Group label (e.g. `control`, `treatment_a`). Max 64 chars.
|
|
88
|
-
weight:
|
|
89
|
-
type: integer
|
|
90
|
-
minimum: 0
|
|
91
|
-
maximum: 10000
|
|
92
|
-
description: Allocation weight in basis points (0–10000). The sum across all groups must equal exactly **10000** (100%).
|
|
93
|
-
params:
|
|
94
|
-
default: {}
|
|
95
|
-
description: Per-group parameter values delivered to the SDK when a caller is hashed into this group. Keys must match the experiment's `params` schema.
|
|
96
|
-
type: object
|
|
97
|
-
additionalProperties: {}
|
|
98
|
-
required:
|
|
99
|
-
- name
|
|
100
|
-
- weight
|
|
101
|
-
additionalProperties: false
|
|
102
|
-
description: One variant in the experiment. Weights across all groups must sum to 10000.
|
|
103
|
-
significanceThreshold:
|
|
104
|
-
type: number
|
|
105
|
-
minRuntimeDays:
|
|
106
|
-
type: integer
|
|
107
|
-
minimum: -9007199254740991
|
|
108
|
-
maximum: 9007199254740991
|
|
109
|
-
minSampleSize:
|
|
110
|
-
type: integer
|
|
111
|
-
minimum: -9007199254740991
|
|
112
|
-
maximum: 9007199254740991
|
|
113
|
-
sequentialTesting:
|
|
114
|
-
type: boolean
|
|
115
|
-
startedAt:
|
|
116
|
-
anyOf:
|
|
117
|
-
- type: string
|
|
118
|
-
- type: "null"
|
|
119
|
-
description: ISO-8601 timestamp the experiment last transitioned to `running`, or `null`.
|
|
120
|
-
stoppedAt:
|
|
121
|
-
anyOf:
|
|
122
|
-
- type: string
|
|
123
|
-
- type: "null"
|
|
124
|
-
updatedAt:
|
|
125
|
-
type: string
|
|
126
|
-
required:
|
|
127
|
-
- id
|
|
128
|
-
- name
|
|
129
|
-
- description
|
|
130
|
-
- hypothesis
|
|
131
|
-
- tag
|
|
132
|
-
- ownerEmail
|
|
133
|
-
- audience
|
|
134
|
-
- bucketBy
|
|
135
|
-
- folder
|
|
136
|
-
- status
|
|
137
|
-
- universe
|
|
138
|
-
- targetingGate
|
|
139
|
-
- allocationPct
|
|
140
|
-
- salt
|
|
141
|
-
- params
|
|
142
|
-
- groups
|
|
143
|
-
- significanceThreshold
|
|
144
|
-
- minRuntimeDays
|
|
145
|
-
- minSampleSize
|
|
146
|
-
- sequentialTesting
|
|
147
|
-
- startedAt
|
|
148
|
-
- updatedAt
|
|
149
|
-
additionalProperties: false
|
|
7
|
+
$ref: ./experiment-api-row.yaml#/ExperimentApiRow
|
|
150
8
|
next_cursor:
|
|
151
9
|
anyOf:
|
|
152
10
|
- type: string
|
|
@@ -156,149 +14,7 @@ ListExperimentsResponse:
|
|
|
156
14
|
- next_cursor
|
|
157
15
|
additionalProperties: false
|
|
158
16
|
GetExperimentResponse:
|
|
159
|
-
|
|
160
|
-
properties:
|
|
161
|
-
id:
|
|
162
|
-
type: string
|
|
163
|
-
description: Stable opaque experiment id (`exp_…`).
|
|
164
|
-
name:
|
|
165
|
-
type: string
|
|
166
|
-
maxLength: 128
|
|
167
|
-
pattern: ^[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?)?$
|
|
168
|
-
description: Stable experiment key. Single segment or `folder.name` (a-z, 0-9, `_`/`-`; max 128 chars). Used by SDKs as `Shipeasy.getExperiment(user, '<name>')`. Immutable after create.
|
|
169
|
-
description:
|
|
170
|
-
anyOf:
|
|
171
|
-
- type: string
|
|
172
|
-
- type: "null"
|
|
173
|
-
hypothesis:
|
|
174
|
-
anyOf:
|
|
175
|
-
- type: string
|
|
176
|
-
- type: "null"
|
|
177
|
-
tag:
|
|
178
|
-
anyOf:
|
|
179
|
-
- type: string
|
|
180
|
-
- type: "null"
|
|
181
|
-
ownerEmail:
|
|
182
|
-
anyOf:
|
|
183
|
-
- type: string
|
|
184
|
-
- type: "null"
|
|
185
|
-
audience:
|
|
186
|
-
anyOf:
|
|
187
|
-
- type: string
|
|
188
|
-
- type: "null"
|
|
189
|
-
bucketBy:
|
|
190
|
-
anyOf:
|
|
191
|
-
- type: string
|
|
192
|
-
minLength: 1
|
|
193
|
-
maxLength: 128
|
|
194
|
-
pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
|
|
195
|
-
description: User-attribute name used as the bucketing key — e.g. `user_id` (default), `session_id`, `device_id`, or a custom attribute like `company_id` to keep a whole org on one variant. Defaults to `user_id` when omitted.
|
|
196
|
-
- type: "null"
|
|
197
|
-
folder:
|
|
198
|
-
anyOf:
|
|
199
|
-
- type: string
|
|
200
|
-
- type: "null"
|
|
201
|
-
status:
|
|
202
|
-
type: string
|
|
203
|
-
enum:
|
|
204
|
-
- draft
|
|
205
|
-
- running
|
|
206
|
-
- stopped
|
|
207
|
-
- archived
|
|
208
|
-
universe:
|
|
209
|
-
type: string
|
|
210
|
-
description: Universe name this experiment draws from.
|
|
211
|
-
targetingGate:
|
|
212
|
-
anyOf:
|
|
213
|
-
- type: string
|
|
214
|
-
- type: "null"
|
|
215
|
-
allocationPct:
|
|
216
|
-
type: integer
|
|
217
|
-
minimum: -9007199254740991
|
|
218
|
-
maximum: 9007199254740991
|
|
219
|
-
description: Allocation in basis points (0–10000).
|
|
220
|
-
salt:
|
|
221
|
-
type: string
|
|
222
|
-
params:
|
|
223
|
-
type: object
|
|
224
|
-
additionalProperties:
|
|
225
|
-
type: string
|
|
226
|
-
enum:
|
|
227
|
-
- string
|
|
228
|
-
- bool
|
|
229
|
-
- number
|
|
230
|
-
groups:
|
|
231
|
-
type: array
|
|
232
|
-
items:
|
|
233
|
-
type: object
|
|
234
|
-
properties:
|
|
235
|
-
name:
|
|
236
|
-
type: string
|
|
237
|
-
minLength: 1
|
|
238
|
-
maxLength: 64
|
|
239
|
-
description: Group label (e.g. `control`, `treatment_a`). Max 64 chars.
|
|
240
|
-
weight:
|
|
241
|
-
type: integer
|
|
242
|
-
minimum: 0
|
|
243
|
-
maximum: 10000
|
|
244
|
-
description: Allocation weight in basis points (0–10000). The sum across all groups must equal exactly **10000** (100%).
|
|
245
|
-
params:
|
|
246
|
-
default: {}
|
|
247
|
-
description: Per-group parameter values delivered to the SDK when a caller is hashed into this group. Keys must match the experiment's `params` schema.
|
|
248
|
-
type: object
|
|
249
|
-
additionalProperties: {}
|
|
250
|
-
required:
|
|
251
|
-
- name
|
|
252
|
-
- weight
|
|
253
|
-
additionalProperties: false
|
|
254
|
-
description: One variant in the experiment. Weights across all groups must sum to 10000.
|
|
255
|
-
significanceThreshold:
|
|
256
|
-
type: number
|
|
257
|
-
minRuntimeDays:
|
|
258
|
-
type: integer
|
|
259
|
-
minimum: -9007199254740991
|
|
260
|
-
maximum: 9007199254740991
|
|
261
|
-
minSampleSize:
|
|
262
|
-
type: integer
|
|
263
|
-
minimum: -9007199254740991
|
|
264
|
-
maximum: 9007199254740991
|
|
265
|
-
sequentialTesting:
|
|
266
|
-
type: boolean
|
|
267
|
-
startedAt:
|
|
268
|
-
anyOf:
|
|
269
|
-
- type: string
|
|
270
|
-
- type: "null"
|
|
271
|
-
description: ISO-8601 timestamp the experiment last transitioned to `running`, or `null`.
|
|
272
|
-
stoppedAt:
|
|
273
|
-
anyOf:
|
|
274
|
-
- type: string
|
|
275
|
-
- type: "null"
|
|
276
|
-
updatedAt:
|
|
277
|
-
type: string
|
|
278
|
-
required:
|
|
279
|
-
- id
|
|
280
|
-
- name
|
|
281
|
-
- description
|
|
282
|
-
- hypothesis
|
|
283
|
-
- tag
|
|
284
|
-
- ownerEmail
|
|
285
|
-
- audience
|
|
286
|
-
- bucketBy
|
|
287
|
-
- folder
|
|
288
|
-
- status
|
|
289
|
-
- universe
|
|
290
|
-
- targetingGate
|
|
291
|
-
- allocationPct
|
|
292
|
-
- salt
|
|
293
|
-
- params
|
|
294
|
-
- groups
|
|
295
|
-
- significanceThreshold
|
|
296
|
-
- minRuntimeDays
|
|
297
|
-
- minSampleSize
|
|
298
|
-
- sequentialTesting
|
|
299
|
-
- startedAt
|
|
300
|
-
- updatedAt
|
|
301
|
-
additionalProperties: false
|
|
17
|
+
$ref: ./experiment-api-row.yaml#/ExperimentApiRow
|
|
302
18
|
CreateExperimentRequest:
|
|
303
19
|
type: object
|
|
304
20
|
properties:
|
|
@@ -359,13 +75,19 @@ CreateExperimentRequest:
|
|
|
359
75
|
description: Name of an existing universe in the project. Returns `422` if the universe doesn't exist.
|
|
360
76
|
targeting_gate:
|
|
361
77
|
default: null
|
|
362
|
-
description: Optional gate name. Only callers that pass the gate are enrolled in the experiment.
|
|
78
|
+
description: Optional gate name (a `targeting`-type flag). Only callers that pass the gate are enrolled in the experiment.
|
|
79
|
+
anyOf:
|
|
80
|
+
- type: string
|
|
81
|
+
- type: "null"
|
|
82
|
+
holdout_gate:
|
|
83
|
+
default: null
|
|
84
|
+
description: Optional per-experiment holdout gate — the name of a `holdout`-type flag (public % + whitelist). A caller the flag passes is *held out* (never assigned, sees the universe defaults). Distinct from the universe-level holdout.
|
|
363
85
|
anyOf:
|
|
364
86
|
- type: string
|
|
365
87
|
- type: "null"
|
|
366
88
|
allocation_pct:
|
|
367
89
|
default: 0
|
|
368
|
-
description: Share of the (gated) audience allocated to the experiment, in basis points (0–10000 = 0%–100%). `0` = unallocated. Use `allocation_percent` (0–100) below to think in percent. Immutable while the experiment is running.
|
|
90
|
+
description: Share of the (gated) audience allocated to the experiment, in basis points (0–10000 = 0%–100%). `0` = unallocated. Under pooled assignment this is the size of the universe-pool slice claimed. Use `allocation_percent` (0–100) below to think in percent. Immutable while the experiment is running.
|
|
369
91
|
type: integer
|
|
370
92
|
minimum: 0
|
|
371
93
|
maximum: 10000
|
|
@@ -374,6 +96,11 @@ CreateExperimentRequest:
|
|
|
374
96
|
type: number
|
|
375
97
|
minimum: 0
|
|
376
98
|
maximum: 100
|
|
99
|
+
reserved_headroom:
|
|
100
|
+
description: Basis points of this experiment's split kept empty (0–10000) so a new variant can be appended into it while running without reshuffling. Group weights must sum to `10000 − reserved_headroom`. Defaults to the universe's `recommended_headroom` when omitted.
|
|
101
|
+
type: integer
|
|
102
|
+
minimum: 0
|
|
103
|
+
maximum: 10000
|
|
377
104
|
salt:
|
|
378
105
|
description: Hash salt for bucketing. Auto-generated if omitted. Immutable while running.
|
|
379
106
|
type: string
|
|
@@ -388,7 +115,7 @@ CreateExperimentRequest:
|
|
|
388
115
|
- string
|
|
389
116
|
- bool
|
|
390
117
|
- number
|
|
391
|
-
description: "
|
|
118
|
+
description: "**Deprecated** — the universe now owns the config schema (`param_schema`). Retained for back-compat; new experiments should leave this empty and declare params on the universe. Map of param-name → scalar type."
|
|
392
119
|
groups:
|
|
393
120
|
minItems: 2
|
|
394
121
|
type: array
|
|
@@ -404,18 +131,18 @@ CreateExperimentRequest:
|
|
|
404
131
|
type: integer
|
|
405
132
|
minimum: 0
|
|
406
133
|
maximum: 10000
|
|
407
|
-
description: Allocation weight in basis points (0–10000).
|
|
134
|
+
description: Allocation weight in basis points (0–10000). Weights across all groups must sum to `10000 − reserved_headroom` (100% minus any reserved tail).
|
|
408
135
|
params:
|
|
409
136
|
default: {}
|
|
410
|
-
description: Per-
|
|
137
|
+
description: Per-variant **override map** — a subset of the universe's `param_schema` keys. A key the variant doesn't set inherits the universe default. Unknown keys (not in the universe schema) are rejected.
|
|
411
138
|
type: object
|
|
412
139
|
additionalProperties: {}
|
|
413
140
|
required:
|
|
414
141
|
- name
|
|
415
142
|
- weight
|
|
416
143
|
additionalProperties: false
|
|
417
|
-
description: One variant
|
|
418
|
-
description: Two or more variants. Weights must sum to
|
|
144
|
+
description: One variant. Its `params` may only override keys declared on the universe schema.
|
|
145
|
+
description: Two or more variants. Weights must sum to `10000 − reserved_headroom`. Existing weights are immutable while running, but a new variant may be appended into the reserved tail.
|
|
419
146
|
significance_threshold:
|
|
420
147
|
default: 0.05
|
|
421
148
|
description: p-value cutoff used by the analysis pass. Defaults to `0.05`. Values other than 0.05 require Pro plan or higher.
|
|
@@ -516,11 +243,21 @@ UpdateExperimentRequest:
|
|
|
516
243
|
anyOf:
|
|
517
244
|
- type: string
|
|
518
245
|
- type: "null"
|
|
246
|
+
holdout_gate:
|
|
247
|
+
description: Per-experiment holdout gate — the name of a `holdout`-type flag, or `null` to clear. A caller the flag passes is held out.
|
|
248
|
+
anyOf:
|
|
249
|
+
- type: string
|
|
250
|
+
- type: "null"
|
|
519
251
|
allocation_pct:
|
|
520
252
|
description: Basis-points allocation (0–10000). Use `allocation_percent` (0–100) for percent. Immutable while the experiment is running.
|
|
521
253
|
type: integer
|
|
522
254
|
minimum: 0
|
|
523
255
|
maximum: 10000
|
|
256
|
+
reserved_headroom:
|
|
257
|
+
description: Basis points of the split kept empty for appended variants. Group weights must sum to `10000 − reserved_headroom`. May be shrunk (never grown into existing weights) while running when appending a variant.
|
|
258
|
+
type: integer
|
|
259
|
+
minimum: 0
|
|
260
|
+
maximum: 10000
|
|
524
261
|
allocation_percent:
|
|
525
262
|
description: Allocation as a **percentage** (0–100). Friendlier alias for `allocation_pct`; converted to basis points server-side. Wins over `allocation_pct` if both are supplied. Immutable while running.
|
|
526
263
|
type: number
|
|
@@ -542,9 +279,9 @@ UpdateExperimentRequest:
|
|
|
542
279
|
- string
|
|
543
280
|
- bool
|
|
544
281
|
- number
|
|
545
|
-
description: "
|
|
282
|
+
description: "**Deprecated** — the universe owns the config schema (`param_schema`). Retained for back-compat. Map of param-name → scalar type."
|
|
546
283
|
groups:
|
|
547
|
-
description: Replacement groups. Weights must sum to 10000
|
|
284
|
+
description: Replacement groups. Weights must sum to `10000 − reserved_headroom`. Existing weights/values are immutable while running; a new variant may be appended into the reserved tail.
|
|
548
285
|
minItems: 2
|
|
549
286
|
type: array
|
|
550
287
|
items:
|
|
@@ -559,17 +296,17 @@ UpdateExperimentRequest:
|
|
|
559
296
|
type: integer
|
|
560
297
|
minimum: 0
|
|
561
298
|
maximum: 10000
|
|
562
|
-
description: Allocation weight in basis points (0–10000).
|
|
299
|
+
description: Allocation weight in basis points (0–10000). Weights across all groups must sum to `10000 − reserved_headroom`.
|
|
563
300
|
params:
|
|
564
301
|
default: {}
|
|
565
|
-
description: Per-
|
|
302
|
+
description: Per-variant **override map** — a subset of the universe's `param_schema` keys. Unknown keys are rejected; unset keys inherit the universe default.
|
|
566
303
|
type: object
|
|
567
304
|
additionalProperties: {}
|
|
568
305
|
required:
|
|
569
306
|
- name
|
|
570
307
|
- weight
|
|
571
308
|
additionalProperties: false
|
|
572
|
-
description: One variant
|
|
309
|
+
description: One variant. Its `params` may only override keys declared on the universe schema.
|
|
573
310
|
significance_threshold:
|
|
574
311
|
type: number
|
|
575
312
|
minimum: 0.0001
|
|
@@ -594,7 +331,7 @@ UpdateExperimentRequest:
|
|
|
594
331
|
items:
|
|
595
332
|
$ref: ./common.yaml#/ExperimentInlineMetric
|
|
596
333
|
additionalProperties: false
|
|
597
|
-
description: Body for `PATCH /api/admin/experiments/{id}`. Partial — only supplied fields change. `allocation_pct`, `
|
|
334
|
+
description: Body for `PATCH /api/admin/experiments/{id}`. Partial — only supplied fields change. `allocation_pct`, `salt`, `universe` and existing group weights/values are immutable while running (stop first); a new variant may still be appended into the reserved tail.
|
|
598
335
|
UpdateExperimentResponse:
|
|
599
336
|
type: object
|
|
600
337
|
properties:
|
|
@@ -662,6 +399,12 @@ SetExperimentMetricsRequest:
|
|
|
662
399
|
- guardrail
|
|
663
400
|
- secondary
|
|
664
401
|
description: Metric role. `goal` drives the decision, `guardrail` blocks ship if degraded, `secondary` is informational.
|
|
402
|
+
min_effect_of_interest:
|
|
403
|
+
description: Per-experiment override of the metric's `default_min_effect_of_interest` (relative, 0–1) — the smallest change worth acting on for THIS experiment's decision, which depends on the intervention's cost/risk. `null`/omitted falls back to the metric default. For a `guardrail` this is the non-inferiority margin (how large a regression is tolerated); for a `goal` it is the superiority threshold.
|
|
404
|
+
default: null
|
|
405
|
+
anyOf:
|
|
406
|
+
- type: number
|
|
407
|
+
- type: "null"
|
|
665
408
|
required:
|
|
666
409
|
- metric_id
|
|
667
410
|
- role
|
|
@@ -689,9 +432,15 @@ SetExperimentMetricsResponse:
|
|
|
689
432
|
- goal
|
|
690
433
|
- guardrail
|
|
691
434
|
- secondary
|
|
435
|
+
min_effect_of_interest:
|
|
436
|
+
description: The resolved per-experiment override, or `null` when this attachment inherits the metric's `default_min_effect_of_interest`.
|
|
437
|
+
anyOf:
|
|
438
|
+
- type: number
|
|
439
|
+
- type: "null"
|
|
692
440
|
required:
|
|
693
441
|
- metric_id
|
|
694
442
|
- role
|
|
443
|
+
- min_effect_of_interest
|
|
695
444
|
additionalProperties: false
|
|
696
445
|
required:
|
|
697
446
|
- id
|
|
@@ -725,46 +474,8 @@ GetExperimentResultsResponse:
|
|
|
725
474
|
results:
|
|
726
475
|
type: array
|
|
727
476
|
items:
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
metric:
|
|
731
|
-
type: string
|
|
732
|
-
group_name:
|
|
733
|
-
type: string
|
|
734
|
-
ds:
|
|
735
|
-
type: string
|
|
736
|
-
description: Date slice (`YYYY-MM-DD`).
|
|
737
|
-
n:
|
|
738
|
-
anyOf:
|
|
739
|
-
- type: number
|
|
740
|
-
- type: "null"
|
|
741
|
-
mean:
|
|
742
|
-
anyOf:
|
|
743
|
-
- type: number
|
|
744
|
-
- type: "null"
|
|
745
|
-
delta_pct:
|
|
746
|
-
anyOf:
|
|
747
|
-
- type: number
|
|
748
|
-
- type: "null"
|
|
749
|
-
p_value:
|
|
750
|
-
anyOf:
|
|
751
|
-
- type: number
|
|
752
|
-
- type: "null"
|
|
753
|
-
srm_detected:
|
|
754
|
-
anyOf:
|
|
755
|
-
- type: number
|
|
756
|
-
- type: "null"
|
|
757
|
-
description: "`1` if sample-ratio mismatch detected, else `0`."
|
|
758
|
-
required:
|
|
759
|
-
- metric
|
|
760
|
-
- group_name
|
|
761
|
-
- ds
|
|
762
|
-
- n
|
|
763
|
-
- mean
|
|
764
|
-
- delta_pct
|
|
765
|
-
- p_value
|
|
766
|
-
- srm_detected
|
|
767
|
-
additionalProperties: false
|
|
477
|
+
$ref: "#/ExperimentResultRow"
|
|
478
|
+
description: One analysis row per metric/group/day — the persisted `experiment_results` rows for this experiment.
|
|
768
479
|
verdict:
|
|
769
480
|
type: string
|
|
770
481
|
enum:
|
|
@@ -807,50 +518,144 @@ GetExperimentTimeseriesResponse:
|
|
|
807
518
|
series:
|
|
808
519
|
type: array
|
|
809
520
|
items:
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
metric:
|
|
813
|
-
type: string
|
|
814
|
-
group_name:
|
|
815
|
-
type: string
|
|
816
|
-
ds:
|
|
817
|
-
type: string
|
|
818
|
-
description: Date slice (`YYYY-MM-DD`).
|
|
819
|
-
n:
|
|
820
|
-
anyOf:
|
|
821
|
-
- type: number
|
|
822
|
-
- type: "null"
|
|
823
|
-
mean:
|
|
824
|
-
anyOf:
|
|
825
|
-
- type: number
|
|
826
|
-
- type: "null"
|
|
827
|
-
delta_pct:
|
|
828
|
-
anyOf:
|
|
829
|
-
- type: number
|
|
830
|
-
- type: "null"
|
|
831
|
-
p_value:
|
|
832
|
-
anyOf:
|
|
833
|
-
- type: number
|
|
834
|
-
- type: "null"
|
|
835
|
-
srm_detected:
|
|
836
|
-
anyOf:
|
|
837
|
-
- type: number
|
|
838
|
-
- type: "null"
|
|
839
|
-
description: "`1` if sample-ratio mismatch detected, else `0`."
|
|
840
|
-
required:
|
|
841
|
-
- metric
|
|
842
|
-
- group_name
|
|
843
|
-
- ds
|
|
844
|
-
- n
|
|
845
|
-
- mean
|
|
846
|
-
- delta_pct
|
|
847
|
-
- p_value
|
|
848
|
-
- srm_detected
|
|
849
|
-
additionalProperties: false
|
|
521
|
+
$ref: "#/ExperimentResultRow"
|
|
522
|
+
description: Every daily analysis slice (same row shape as `/results`), oldest to newest.
|
|
850
523
|
required:
|
|
851
524
|
- experiment
|
|
852
525
|
- series
|
|
853
526
|
additionalProperties: false
|
|
527
|
+
ExperimentResultRow:
|
|
528
|
+
type: object
|
|
529
|
+
description: One persisted analysis row (`experiment_results`) for a single metric/group/day slice. Camel-case wire names; nullable statistics are `null` on control rows, empty arms, or when the analysis pass hasn't produced them.
|
|
530
|
+
properties:
|
|
531
|
+
metric:
|
|
532
|
+
type: string
|
|
533
|
+
description: Metric name this row was computed for.
|
|
534
|
+
groupName:
|
|
535
|
+
type: string
|
|
536
|
+
description: Variant (group) label the row belongs to, e.g. `control` or `treatment`.
|
|
537
|
+
ds:
|
|
538
|
+
type: string
|
|
539
|
+
description: Date slice (`YYYY-MM-DD`).
|
|
540
|
+
n:
|
|
541
|
+
description: Sample size (exposed units) in this group for the slice, or `null` when unknown.
|
|
542
|
+
anyOf:
|
|
543
|
+
- type: integer
|
|
544
|
+
- type: "null"
|
|
545
|
+
mean:
|
|
546
|
+
description: Per-unit metric mean in this group, or `null` on empty arms.
|
|
547
|
+
anyOf:
|
|
548
|
+
- type: number
|
|
549
|
+
- type: "null"
|
|
550
|
+
variance:
|
|
551
|
+
description: Per-unit metric variance in this group, or `null` on empty arms.
|
|
552
|
+
anyOf:
|
|
553
|
+
- type: number
|
|
554
|
+
- type: "null"
|
|
555
|
+
delta:
|
|
556
|
+
description: Absolute difference vs. control (metric units). `null` on the control row.
|
|
557
|
+
anyOf:
|
|
558
|
+
- type: number
|
|
559
|
+
- type: "null"
|
|
560
|
+
deltaPct:
|
|
561
|
+
description: Relative difference vs. control, in percent. `null` on the control row.
|
|
562
|
+
anyOf:
|
|
563
|
+
- type: number
|
|
564
|
+
- type: "null"
|
|
565
|
+
ci95Low:
|
|
566
|
+
description: Lower bound of the 95% confidence interval on `delta`, or `null`. Kept for back-compat — prefer `ciLow`/`ciHigh`.
|
|
567
|
+
anyOf:
|
|
568
|
+
- type: number
|
|
569
|
+
- type: "null"
|
|
570
|
+
ci95High:
|
|
571
|
+
description: Upper bound of the 95% confidence interval on `delta`, or `null`. Kept for back-compat — prefer `ciLow`/`ciHigh`.
|
|
572
|
+
anyOf:
|
|
573
|
+
- type: number
|
|
574
|
+
- type: "null"
|
|
575
|
+
ci99Low:
|
|
576
|
+
description: Lower bound of the 99% confidence interval on `delta`, or `null`. Kept for back-compat — prefer `ciLow`/`ciHigh`.
|
|
577
|
+
anyOf:
|
|
578
|
+
- type: number
|
|
579
|
+
- type: "null"
|
|
580
|
+
ci99High:
|
|
581
|
+
description: Upper bound of the 99% confidence interval on `delta`, or `null`. Kept for back-compat — prefer `ciLow`/`ciHigh`.
|
|
582
|
+
anyOf:
|
|
583
|
+
- type: number
|
|
584
|
+
- type: "null"
|
|
585
|
+
ciLow:
|
|
586
|
+
description: Lower bound of the confidence interval at the project's configured `ci_confidence` (not always 95%), or `null`. When both `ciLow`/`ciHigh` are `null`, fall back to `ci95Low`/`ci95High` (pre-migration rows).
|
|
587
|
+
anyOf:
|
|
588
|
+
- type: number
|
|
589
|
+
- type: "null"
|
|
590
|
+
ciHigh:
|
|
591
|
+
description: Upper bound of the confidence interval at the project's configured `ci_confidence` (not always 95%), or `null`. When both `ciLow`/`ciHigh` are `null`, fall back to `ci95Low`/`ci95High` (pre-migration rows).
|
|
592
|
+
anyOf:
|
|
593
|
+
- type: number
|
|
594
|
+
- type: "null"
|
|
595
|
+
pValue:
|
|
596
|
+
description: Two-sided p-value vs. control, or `null` on the control row / empty arms.
|
|
597
|
+
anyOf:
|
|
598
|
+
- type: number
|
|
599
|
+
- type: "null"
|
|
600
|
+
expectedN:
|
|
601
|
+
description: Sample size per arm the power calculation expects for a conclusive read, or `null` when not computed.
|
|
602
|
+
anyOf:
|
|
603
|
+
- type: integer
|
|
604
|
+
- type: "null"
|
|
605
|
+
realizedMde:
|
|
606
|
+
description: "Realized minimum detectable effect (absolute, in metric units) — the smallest true effect this run could detect at the experiment's significance level and target power GIVEN the observed variance and per-arm sample sizes. A derived OUTPUT, not an input knob; compare it against the metric's minimum effect of interest to judge whether the run is adequately powered. `null` on the control row and on empty arms."
|
|
607
|
+
anyOf:
|
|
608
|
+
- type: number
|
|
609
|
+
- type: "null"
|
|
610
|
+
srmPValue:
|
|
611
|
+
description: p-value of the sample-ratio-mismatch chi-squared test, or `null` when not computed.
|
|
612
|
+
anyOf:
|
|
613
|
+
- type: number
|
|
614
|
+
- type: "null"
|
|
615
|
+
srmDetected:
|
|
616
|
+
type: integer
|
|
617
|
+
description: "`1` if sample-ratio mismatch detected, else `0`."
|
|
618
|
+
msprtSignificant:
|
|
619
|
+
description: "Sequential (mSPRT) significance flag: `1` significant, `0` not, `null` when sequential testing is off or not computed."
|
|
620
|
+
anyOf:
|
|
621
|
+
- type: integer
|
|
622
|
+
- type: "null"
|
|
623
|
+
msprtLambda:
|
|
624
|
+
description: mSPRT mixture likelihood-ratio statistic, or `null` when sequential testing is off or not computed.
|
|
625
|
+
anyOf:
|
|
626
|
+
- type: number
|
|
627
|
+
- type: "null"
|
|
628
|
+
isFinal:
|
|
629
|
+
type: integer
|
|
630
|
+
description: "`1` when this row belongs to the final analysis pass after the experiment stopped, else `0`."
|
|
631
|
+
peekWarning:
|
|
632
|
+
type: integer
|
|
633
|
+
description: "`1` when the row was read before `min_runtime_days`/`min_sample_size` were met (peeking inflates false positives), else `0`."
|
|
634
|
+
required:
|
|
635
|
+
- metric
|
|
636
|
+
- groupName
|
|
637
|
+
- ds
|
|
638
|
+
- n
|
|
639
|
+
- mean
|
|
640
|
+
- variance
|
|
641
|
+
- delta
|
|
642
|
+
- deltaPct
|
|
643
|
+
- ci95Low
|
|
644
|
+
- ci95High
|
|
645
|
+
- ci99Low
|
|
646
|
+
- ci99High
|
|
647
|
+
- ciLow
|
|
648
|
+
- ciHigh
|
|
649
|
+
- pValue
|
|
650
|
+
- expectedN
|
|
651
|
+
- realizedMde
|
|
652
|
+
- srmPValue
|
|
653
|
+
- srmDetected
|
|
654
|
+
- msprtSignificant
|
|
655
|
+
- msprtLambda
|
|
656
|
+
- isFinal
|
|
657
|
+
- peekWarning
|
|
658
|
+
additionalProperties: false
|
|
854
659
|
ReanalyzeExperimentResponse:
|
|
855
660
|
type: object
|
|
856
661
|
properties:
|
|
@@ -858,8 +663,218 @@ ReanalyzeExperimentResponse:
|
|
|
858
663
|
type: string
|
|
859
664
|
queued:
|
|
860
665
|
type: boolean
|
|
861
|
-
|
|
666
|
+
description: "`true` when the analysis job was enqueued; `false` when the worker enqueue failed or the analysis queue is not configured (retry later)."
|
|
862
667
|
required:
|
|
863
668
|
- id
|
|
864
669
|
- queued
|
|
865
670
|
additionalProperties: false
|
|
671
|
+
CreateExperimentReadoutRequest:
|
|
672
|
+
type: object
|
|
673
|
+
description: Body for `POST /api/admin/experiments/{id}/readouts`. Mints an immutable, dated readout snapshot of the current results view — "this is what we saw when we decided".
|
|
674
|
+
properties:
|
|
675
|
+
kind:
|
|
676
|
+
type: string
|
|
677
|
+
enum:
|
|
678
|
+
- manual
|
|
679
|
+
- ship
|
|
680
|
+
- stop
|
|
681
|
+
description: Why the snapshot is being minted — `manual` ("Share readout"), or automatically on `ship` / `stop`.
|
|
682
|
+
acknowledgedCaveatIds:
|
|
683
|
+
type: array
|
|
684
|
+
items:
|
|
685
|
+
type: string
|
|
686
|
+
description: Caveat id, as surfaced in the results view's caveat list.
|
|
687
|
+
description: Ids of the open caveats the caller ticked (decision-gating acknowledgment). Unlisted caveats are stored as unacknowledged.
|
|
688
|
+
requireAllAcknowledged:
|
|
689
|
+
type: boolean
|
|
690
|
+
description: When `true`, refuse (`422`) to mint while any open caveat is not listed in `acknowledgedCaveatIds` — server-side ship gating.
|
|
691
|
+
required:
|
|
692
|
+
- kind
|
|
693
|
+
additionalProperties: false
|
|
694
|
+
CreateExperimentReadoutResponse:
|
|
695
|
+
type: object
|
|
696
|
+
properties:
|
|
697
|
+
id:
|
|
698
|
+
type: string
|
|
699
|
+
description: Id of the newly minted readout snapshot.
|
|
700
|
+
required:
|
|
701
|
+
- id
|
|
702
|
+
additionalProperties: false
|
|
703
|
+
ExperimentReadoutMetric:
|
|
704
|
+
type: object
|
|
705
|
+
description: Frozen per-metric numbers as rendered at mint time. Display-shaped, not row-shaped — the snapshot keeps rendering exactly what was decided on even if derivations change later.
|
|
706
|
+
properties:
|
|
707
|
+
name:
|
|
708
|
+
type: string
|
|
709
|
+
description: Metric name.
|
|
710
|
+
role:
|
|
711
|
+
type: string
|
|
712
|
+
enum:
|
|
713
|
+
- goal
|
|
714
|
+
- guardrail
|
|
715
|
+
- secondary
|
|
716
|
+
description: Metric role at mint time. `goal` drives the decision, `guardrail` blocks ship if degraded, `secondary` is informational.
|
|
717
|
+
delta:
|
|
718
|
+
description: Absolute difference vs. control (metric units), or `null`.
|
|
719
|
+
anyOf:
|
|
720
|
+
- type: number
|
|
721
|
+
- type: "null"
|
|
722
|
+
deltaPct:
|
|
723
|
+
description: Relative difference vs. control in percent, or `null`.
|
|
724
|
+
anyOf:
|
|
725
|
+
- type: number
|
|
726
|
+
- type: "null"
|
|
727
|
+
ci:
|
|
728
|
+
description: "Two-item `[low, high]` confidence interval on the delta at the project's configured confidence, or `null`."
|
|
729
|
+
anyOf:
|
|
730
|
+
- type: array
|
|
731
|
+
items:
|
|
732
|
+
type: number
|
|
733
|
+
minItems: 2
|
|
734
|
+
maxItems: 2
|
|
735
|
+
- type: "null"
|
|
736
|
+
p:
|
|
737
|
+
description: p-value vs. control, or `null`.
|
|
738
|
+
anyOf:
|
|
739
|
+
- type: number
|
|
740
|
+
- type: "null"
|
|
741
|
+
sig:
|
|
742
|
+
type: boolean
|
|
743
|
+
description: Whether the metric was statistically significant at mint time.
|
|
744
|
+
pass:
|
|
745
|
+
type: boolean
|
|
746
|
+
description: Guardrail pass/fail at mint time. Omitted for metrics where pass/fail doesn't apply.
|
|
747
|
+
required:
|
|
748
|
+
- name
|
|
749
|
+
- role
|
|
750
|
+
- delta
|
|
751
|
+
- deltaPct
|
|
752
|
+
- ci
|
|
753
|
+
- p
|
|
754
|
+
- sig
|
|
755
|
+
additionalProperties: false
|
|
756
|
+
ExperimentReadoutCaveat:
|
|
757
|
+
type: object
|
|
758
|
+
description: One caveat from the results view, frozen with its acknowledgment state at mint time.
|
|
759
|
+
properties:
|
|
760
|
+
id:
|
|
761
|
+
type: string
|
|
762
|
+
description: Stable caveat id.
|
|
763
|
+
severity:
|
|
764
|
+
type: string
|
|
765
|
+
enum:
|
|
766
|
+
- info
|
|
767
|
+
- warn
|
|
768
|
+
- danger
|
|
769
|
+
description: Caveat severity as rendered in the results view.
|
|
770
|
+
text:
|
|
771
|
+
type: string
|
|
772
|
+
description: Human-readable caveat text.
|
|
773
|
+
acknowledged:
|
|
774
|
+
type: boolean
|
|
775
|
+
description: Whether the caller ticked this caveat when minting (decision-gating acknowledgment).
|
|
776
|
+
required:
|
|
777
|
+
- id
|
|
778
|
+
- severity
|
|
779
|
+
- text
|
|
780
|
+
- acknowledged
|
|
781
|
+
additionalProperties: false
|
|
782
|
+
ExperimentReadoutApiRow:
|
|
783
|
+
type: object
|
|
784
|
+
description: An immutable, dated readout snapshot — the frozen results view captured when a decision was made. Never updated after insert.
|
|
785
|
+
properties:
|
|
786
|
+
id:
|
|
787
|
+
type: string
|
|
788
|
+
description: Stable readout id.
|
|
789
|
+
experimentId:
|
|
790
|
+
type: string
|
|
791
|
+
description: Id of the experiment the snapshot belongs to.
|
|
792
|
+
experimentName:
|
|
793
|
+
type: string
|
|
794
|
+
description: Experiment name at mint time (denormalised — survives renames).
|
|
795
|
+
kind:
|
|
796
|
+
type: string
|
|
797
|
+
enum:
|
|
798
|
+
- manual
|
|
799
|
+
- ship
|
|
800
|
+
- stop
|
|
801
|
+
description: Why the snapshot was minted — `manual` ("Share readout"), or automatically on `ship` / `stop`.
|
|
802
|
+
createdBy:
|
|
803
|
+
type: string
|
|
804
|
+
description: Email of the actor who minted the snapshot.
|
|
805
|
+
createdAt:
|
|
806
|
+
type: string
|
|
807
|
+
description: ISO-8601 mint timestamp.
|
|
808
|
+
verdict:
|
|
809
|
+
type: string
|
|
810
|
+
description: Server-computed verdict at mint time (e.g. `ship`, `hold`, `wait`, `invalid`).
|
|
811
|
+
title:
|
|
812
|
+
type: string
|
|
813
|
+
description: Headline of the results view at mint time.
|
|
814
|
+
why:
|
|
815
|
+
type: string
|
|
816
|
+
description: One-line explanation backing the verdict at mint time.
|
|
817
|
+
configHash:
|
|
818
|
+
type: string
|
|
819
|
+
description: sha-256 hex over the assignment-relevant config (salt, groups, allocation, universe, gate) — the snapshot is only comparable to the live view while this hash still matches.
|
|
820
|
+
metricsJson:
|
|
821
|
+
description: Frozen per-metric numbers as rendered at mint time, or `null` for legacy rows without a metrics slice.
|
|
822
|
+
anyOf:
|
|
823
|
+
- type: array
|
|
824
|
+
items:
|
|
825
|
+
$ref: "#/ExperimentReadoutMetric"
|
|
826
|
+
- type: "null"
|
|
827
|
+
caveatsJson:
|
|
828
|
+
description: The caveat list with its acknowledgment state at mint time, or `null` for legacy rows without one.
|
|
829
|
+
anyOf:
|
|
830
|
+
- type: array
|
|
831
|
+
items:
|
|
832
|
+
$ref: "#/ExperimentReadoutCaveat"
|
|
833
|
+
- type: "null"
|
|
834
|
+
usersJson:
|
|
835
|
+
description: Enrollment at mint time — per-group sample sizes plus the number of days the experiment had been running — or `null` when not captured.
|
|
836
|
+
anyOf:
|
|
837
|
+
- type: object
|
|
838
|
+
properties:
|
|
839
|
+
groups:
|
|
840
|
+
type: array
|
|
841
|
+
items:
|
|
842
|
+
type: object
|
|
843
|
+
properties:
|
|
844
|
+
name:
|
|
845
|
+
type: string
|
|
846
|
+
description: Group label.
|
|
847
|
+
n:
|
|
848
|
+
type: number
|
|
849
|
+
description: Enrolled units in the group at mint time.
|
|
850
|
+
required:
|
|
851
|
+
- name
|
|
852
|
+
- n
|
|
853
|
+
additionalProperties: false
|
|
854
|
+
description: One group's enrollment at mint time.
|
|
855
|
+
description: Per-group enrollment counts.
|
|
856
|
+
days:
|
|
857
|
+
type: number
|
|
858
|
+
description: Days the experiment had been running at mint time.
|
|
859
|
+
required:
|
|
860
|
+
- groups
|
|
861
|
+
- days
|
|
862
|
+
additionalProperties: false
|
|
863
|
+
- type: "null"
|
|
864
|
+
required:
|
|
865
|
+
- id
|
|
866
|
+
- experimentId
|
|
867
|
+
- experimentName
|
|
868
|
+
- kind
|
|
869
|
+
- createdBy
|
|
870
|
+
- createdAt
|
|
871
|
+
- verdict
|
|
872
|
+
- title
|
|
873
|
+
- why
|
|
874
|
+
- configHash
|
|
875
|
+
- metricsJson
|
|
876
|
+
- caveatsJson
|
|
877
|
+
- usersJson
|
|
878
|
+
additionalProperties: false
|
|
879
|
+
GetExperimentReadoutResponse:
|
|
880
|
+
$ref: "#/ExperimentReadoutApiRow"
|