@shipeasy/openapi 2.2.0 → 2.3.0

package/spec/components/schemas/experiments.yaml

@@ -4,149 +4,7 @@ ListExperimentsResponse:
     data:
       type: array
       items:
-        type: object
-        properties:
-          id:
-            type: string
-            description: Stable opaque experiment id (`exp_…`).
-          name:
-            type: string
-            maxLength: 128
-            pattern: ^[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?)?$
-            description: Stable experiment key. Single segment or `folder.name` (a-z, 0-9, `_`/`-`; max 128 chars). Used by SDKs as `Shipeasy.getExperiment(user, '<name>')`. Immutable after create.
-          description:
-            anyOf:
-              - type: string
-              - type: "null"
-          hypothesis:
-            anyOf:
-              - type: string
-              - type: "null"
-          tag:
-            anyOf:
-              - type: string
-              - type: "null"
-          ownerEmail:
-            anyOf:
-              - type: string
-              - type: "null"
-          audience:
-            anyOf:
-              - type: string
-              - type: "null"
-          bucketBy:
-            anyOf:
-              - type: string
-                minLength: 1
-                maxLength: 128
-                pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
-                description: User-attribute name used as the bucketing key — e.g. `user_id` (default), `session_id`, `device_id`, or a custom attribute like `company_id` to keep a whole org on one variant. Defaults to `user_id` when omitted.
-              - type: "null"
-          folder:
-            anyOf:
-              - type: string
-              - type: "null"
-          status:
-            type: string
-            enum:
-              - draft
-              - running
-              - stopped
-              - archived
-          universe:
-            type: string
-            description: Universe name this experiment draws from.
-          targetingGate:
-            anyOf:
-              - type: string
-              - type: "null"
-          allocationPct:
-            type: integer
-            minimum: -9007199254740991
-            maximum: 9007199254740991
-            description: Allocation in basis points (0–10000).
-          salt:
-            type: string
-          params:
-            type: object
-            additionalProperties:
-              type: string
-              enum:
-                - string
-                - bool
-                - number
-          groups:
-            type: array
-            items:
-              type: object
-              properties:
-                name:
-                  type: string
-                  minLength: 1
-                  maxLength: 64
-                  description: Group label (e.g. `control`, `treatment_a`). Max 64 chars.
-                weight:
-                  type: integer
-                  minimum: 0
-                  maximum: 10000
-                  description: Allocation weight in basis points (0–10000). The sum across all groups must equal exactly **10000** (100%).
-                params:
-                  default: {}
-                  description: Per-group parameter values delivered to the SDK when a caller is hashed into this group. Keys must match the experiment's `params` schema.
-                  type: object
-                  additionalProperties: {}
-              required:
-                - name
-                - weight
-              additionalProperties: false
-              description: One variant in the experiment. Weights across all groups must sum to 10000.
-          significanceThreshold:
-            type: number
-          minRuntimeDays:
-            type: integer
-            minimum: -9007199254740991
-            maximum: 9007199254740991
-          minSampleSize:
-            type: integer
-            minimum: -9007199254740991
-            maximum: 9007199254740991
-          sequentialTesting:
-            type: boolean
-          startedAt:
-            anyOf:
-              - type: string
-              - type: "null"
-            description: ISO-8601 timestamp the experiment last transitioned to `running`, or `null`.
-          stoppedAt:
-            anyOf:
-              - type: string
-              - type: "null"
-          updatedAt:
-            type: string
-        required:
-          - id
-          - name
-          - description
-          - hypothesis
-          - tag
-          - ownerEmail
-          - audience
-          - bucketBy
-          - folder
-          - status
-          - universe
-          - targetingGate
-          - allocationPct
-          - salt
-          - params
-          - groups
-          - significanceThreshold
-          - minRuntimeDays
-          - minSampleSize
-          - sequentialTesting
-          - startedAt
-          - updatedAt
-        additionalProperties: false
+        $ref: ./experiment-api-row.yaml#/ExperimentApiRow
     next_cursor:
       anyOf:
         - type: string
@@ -156,149 +14,7 @@ ListExperimentsResponse:
     - next_cursor
   additionalProperties: false
 GetExperimentResponse:
-  type: object
-  properties:
-    id:
-      type: string
-      description: Stable opaque experiment id (`exp_…`).
-    name:
-      type: string
-      maxLength: 128
-      pattern: ^[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?)?$
-      description: Stable experiment key. Single segment or `folder.name` (a-z, 0-9, `_`/`-`; max 128 chars). Used by SDKs as `Shipeasy.getExperiment(user, '<name>')`. Immutable after create.
-    description:
-      anyOf:
-        - type: string
-        - type: "null"
-    hypothesis:
-      anyOf:
-        - type: string
-        - type: "null"
-    tag:
-      anyOf:
-        - type: string
-        - type: "null"
-    ownerEmail:
-      anyOf:
-        - type: string
-        - type: "null"
-    audience:
-      anyOf:
-        - type: string
-        - type: "null"
-    bucketBy:
-      anyOf:
-        - type: string
-          minLength: 1
-          maxLength: 128
-          pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
-          description: User-attribute name used as the bucketing key — e.g. `user_id` (default), `session_id`, `device_id`, or a custom attribute like `company_id` to keep a whole org on one variant. Defaults to `user_id` when omitted.
-        - type: "null"
-    folder:
-      anyOf:
-        - type: string
-        - type: "null"
-    status:
-      type: string
-      enum:
-        - draft
-        - running
-        - stopped
-        - archived
-    universe:
-      type: string
-      description: Universe name this experiment draws from.
-    targetingGate:
-      anyOf:
-        - type: string
-        - type: "null"
-    allocationPct:
-      type: integer
-      minimum: -9007199254740991
-      maximum: 9007199254740991
-      description: Allocation in basis points (0–10000).
-    salt:
-      type: string
-    params:
-      type: object
-      additionalProperties:
-        type: string
-        enum:
-          - string
-          - bool
-          - number
-    groups:
-      type: array
-      items:
-        type: object
-        properties:
-          name:
-            type: string
-            minLength: 1
-            maxLength: 64
-            description: Group label (e.g. `control`, `treatment_a`). Max 64 chars.
-          weight:
-            type: integer
-            minimum: 0
-            maximum: 10000
-            description: Allocation weight in basis points (0–10000). The sum across all groups must equal exactly **10000** (100%).
-          params:
-            default: {}
-            description: Per-group parameter values delivered to the SDK when a caller is hashed into this group. Keys must match the experiment's `params` schema.
-            type: object
-            additionalProperties: {}
-        required:
-          - name
-          - weight
-        additionalProperties: false
-        description: One variant in the experiment. Weights across all groups must sum to 10000.
-    significanceThreshold:
-      type: number
-    minRuntimeDays:
-      type: integer
-      minimum: -9007199254740991
-      maximum: 9007199254740991
-    minSampleSize:
-      type: integer
-      minimum: -9007199254740991
-      maximum: 9007199254740991
-    sequentialTesting:
-      type: boolean
-    startedAt:
-      anyOf:
-        - type: string
-        - type: "null"
-      description: ISO-8601 timestamp the experiment last transitioned to `running`, or `null`.
-    stoppedAt:
-      anyOf:
-        - type: string
-        - type: "null"
-    updatedAt:
-      type: string
-  required:
-    - id
-    - name
-    - description
-    - hypothesis
-    - tag
-    - ownerEmail
-    - audience
-    - bucketBy
-    - folder
-    - status
-    - universe
-    - targetingGate
-    - allocationPct
-    - salt
-    - params
-    - groups
-    - significanceThreshold
-    - minRuntimeDays
-    - minSampleSize
-    - sequentialTesting
-    - startedAt
-    - updatedAt
-  additionalProperties: false
+  $ref: ./experiment-api-row.yaml#/ExperimentApiRow
 CreateExperimentRequest:
   type: object
   properties:
@@ -359,13 +75,19 @@ CreateExperimentRequest:
       description: Name of an existing universe in the project. Returns `422` if the universe doesn't exist.
     targeting_gate:
       default: null
-      description: Optional gate name. Only callers that pass the gate are enrolled in the experiment.
+      description: Optional gate name (a `targeting`-type flag). Only callers that pass the gate are enrolled in the experiment.
+      anyOf:
+        - type: string
+        - type: "null"
+    holdout_gate:
+      default: null
+      description: Optional per-experiment holdout gate — the name of a `holdout`-type flag (public % + whitelist). A caller the flag passes is *held out* (never assigned, sees the universe defaults). Distinct from the universe-level holdout.
       anyOf:
         - type: string
         - type: "null"
     allocation_pct:
       default: 0
-      description: Share of the (gated) audience allocated to the experiment, in basis points (0–10000 = 0%–100%). `0` = unallocated. Use `allocation_percent` (0–100) below to think in percent. Immutable while the experiment is running.
+      description: Share of the (gated) audience allocated to the experiment, in basis points (0–10000 = 0%–100%). `0` = unallocated. Under pooled assignment this is the size of the universe-pool slice claimed. Use `allocation_percent` (0–100) below to think in percent. Immutable while the experiment is running.
       type: integer
       minimum: 0
       maximum: 10000
@@ -374,6 +96,11 @@ CreateExperimentRequest:
       type: number
       minimum: 0
       maximum: 100
+    reserved_headroom:
+      description: Basis points of this experiment's split kept empty (0–10000) so a new variant can be appended into it while running without reshuffling. Group weights must sum to `10000 − reserved_headroom`. Defaults to the universe's `recommended_headroom` when omitted.
+      type: integer
+      minimum: 0
+      maximum: 10000
     salt:
       description: Hash salt for bucketing. Auto-generated if omitted. Immutable while running.
       type: string
@@ -388,7 +115,7 @@ CreateExperimentRequest:
           - string
           - bool
           - number
-      description: "Map of param-name → scalar type. Defines the shape of `groups[].params`. Example: `{ headline: 'string', show_cta: 'bool' }`."
+      description: "**Deprecated** — the universe now owns the config schema (`param_schema`). Retained for back-compat; new experiments should leave this empty and declare params on the universe. Map of param-name → scalar type."
     groups:
       minItems: 2
       type: array
@@ -404,18 +131,18 @@ CreateExperimentRequest:
             type: integer
             minimum: 0
             maximum: 10000
-            description: Allocation weight in basis points (0–10000). The sum across all groups must equal exactly **10000** (100%).
+            description: Allocation weight in basis points (0–10000). Weights across all groups must sum to `10000 − reserved_headroom` (100% minus any reserved tail).
           params:
             default: {}
-            description: Per-group parameter values delivered to the SDK when a caller is hashed into this group. Keys must match the experiment's `params` schema.
+            description: Per-variant **override map** — a subset of the universe's `param_schema` keys. A key the variant doesn't set inherits the universe default. Unknown keys (not in the universe schema) are rejected.
             type: object
             additionalProperties: {}
         required:
           - name
           - weight
         additionalProperties: false
-        description: One variant in the experiment. Weights across all groups must sum to 10000.
-      description: Two or more variants. Weights must sum to exactly 10000 (100%). Immutable while running.
+        description: One variant. Its `params` may only override keys declared on the universe schema.
+      description: Two or more variants. Weights must sum to `10000 − reserved_headroom`. Existing weights are immutable while running, but a new variant may be appended into the reserved tail.
     significance_threshold:
       default: 0.05
       description: p-value cutoff used by the analysis pass. Defaults to `0.05`. Values other than 0.05 require Pro plan or higher.
@@ -516,11 +243,21 @@ UpdateExperimentRequest:
       anyOf:
         - type: string
         - type: "null"
+    holdout_gate:
+      description: Per-experiment holdout gate — the name of a `holdout`-type flag, or `null` to clear. A caller the flag passes is held out.
+      anyOf:
+        - type: string
+        - type: "null"
     allocation_pct:
       description: Basis-points allocation (0–10000). Use `allocation_percent` (0–100) for percent. Immutable while the experiment is running.
       type: integer
       minimum: 0
       maximum: 10000
+    reserved_headroom:
+      description: Basis points of the split kept empty for appended variants. Group weights must sum to `10000 − reserved_headroom`. May be shrunk (never grown into existing weights) while running when appending a variant.
+      type: integer
+      minimum: 0
+      maximum: 10000
     allocation_percent:
       description: Allocation as a **percentage** (0–100). Friendlier alias for `allocation_pct`; converted to basis points server-side. Wins over `allocation_pct` if both are supplied. Immutable while running.
       type: number
@@ -542,9 +279,9 @@ UpdateExperimentRequest:
           - string
           - bool
           - number
-      description: "Map of param-name → scalar type. Defines the shape of `groups[].params`. Example: `{ headline: 'string', show_cta: 'bool' }`."
+      description: "**Deprecated** — the universe owns the config schema (`param_schema`). Retained for back-compat. Map of param-name → scalar type."
     groups:
-      description: Replacement groups. Weights must sum to 10000. Immutable while running.
+      description: Replacement groups. Weights must sum to `10000 − reserved_headroom`. Existing weights/values are immutable while running; a new variant may be appended into the reserved tail.
       minItems: 2
       type: array
       items:
@@ -559,17 +296,17 @@ UpdateExperimentRequest:
             type: integer
             minimum: 0
             maximum: 10000
-            description: Allocation weight in basis points (0–10000). The sum across all groups must equal exactly **10000** (100%).
+            description: Allocation weight in basis points (0–10000). Weights across all groups must sum to `10000 − reserved_headroom`.
           params:
             default: {}
-            description: Per-group parameter values delivered to the SDK when a caller is hashed into this group. Keys must match the experiment's `params` schema.
+            description: Per-variant **override map** — a subset of the universe's `param_schema` keys. Unknown keys are rejected; unset keys inherit the universe default.
             type: object
             additionalProperties: {}
         required:
           - name
           - weight
         additionalProperties: false
-        description: One variant in the experiment. Weights across all groups must sum to 10000.
+        description: One variant. Its `params` may only override keys declared on the universe schema.
     significance_threshold:
       type: number
       minimum: 0.0001
@@ -594,7 +331,7 @@ UpdateExperimentRequest:
       items:
         $ref: ./common.yaml#/ExperimentInlineMetric
   additionalProperties: false
-  description: Body for `PATCH /api/admin/experiments/{id}`. Partial — only supplied fields change. `allocation_pct`, `groups`, `salt`, `universe`, `params` are immutable while the experiment is running (stop first).
+  description: Body for `PATCH /api/admin/experiments/{id}`. Partial — only supplied fields change. `allocation_pct`, `salt`, `universe` and existing group weights/values are immutable while running (stop first); a new variant may still be appended into the reserved tail.
 UpdateExperimentResponse:
   type: object
   properties:
@@ -662,6 +399,12 @@ SetExperimentMetricsRequest:
               - guardrail
               - secondary
             description: Metric role. `goal` drives the decision, `guardrail` blocks ship if degraded, `secondary` is informational.
+          min_effect_of_interest:
+            description: Per-experiment override of the metric's `default_min_effect_of_interest` (relative, 0–1) — the smallest change worth acting on for THIS experiment's decision, which depends on the intervention's cost/risk. `null`/omitted falls back to the metric default. For a `guardrail` this is the non-inferiority margin (how large a regression is tolerated); for a `goal` it is the superiority threshold.
+            default: null
+            anyOf:
+              - type: number
+              - type: "null"
         required:
           - metric_id
           - role
@@ -689,9 +432,15 @@ SetExperimentMetricsResponse:
               - goal
               - guardrail
               - secondary
+          min_effect_of_interest:
+            description: The resolved per-experiment override, or `null` when this attachment inherits the metric's `default_min_effect_of_interest`.
+            anyOf:
+              - type: number
+              - type: "null"
         required:
           - metric_id
           - role
+          - min_effect_of_interest
         additionalProperties: false
   required:
     - id
@@ -755,6 +504,11 @@ GetExperimentResultsResponse:
               - type: number
               - type: "null"
             description: "`1` if sample-ratio mismatch detected, else `0`."
+          realized_mde:
+            anyOf:
+              - type: number
+              - type: "null"
+            description: "Realized minimum detectable effect (absolute, in metric units) — the smallest true effect this run could detect at the experiment's significance level and target power GIVEN the observed variance and per-arm sample sizes. A derived OUTPUT, not an input knob; compare it against the metric's minimum effect of interest to judge whether the run is adequately powered. `null` on the control row and on empty arms."
         required:
           - metric
           - group_name
@@ -764,6 +518,7 @@ GetExperimentResultsResponse:
           - delta_pct
           - p_value
           - srm_detected
+          - realized_mde
         additionalProperties: false
     verdict:
       type: string

package/spec/components/schemas/gates.yaml

@@ -21,6 +21,12 @@ ListGatesResponse:
                 minimum: 0
                 maximum: 1
             description: Master switch. Returned as `1`/`0` from D1; treat truthy as enabled.
+          type:
+            type: string
+            enum:
+              - targeting
+              - holdout
+            description: Gate kind — `targeting` (normal flag) or `holdout` (restricted public % + whitelist).
           rolloutPct:
             type: integer
             minimum: 0
@@ -301,6 +307,13 @@ CreateGateRequest:
       maxLength: 128
       pattern: ^[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9_-]*[a-z0-9])?)?$
       description: Stable gate key used by SDKs (`Shipeasy.checkGate(user, '<name>')`). Single segment or `folder.name`. Lowercase letters, digits, `_` or `-`; max 128 chars. Immutable after create — rename = delete + recreate.
+    type:
+      default: targeting
+      type: string
+      enum:
+        - targeting
+        - holdout
+      description: Gate kind. `targeting` (default) is a normal flag with the full builder. `holdout` is a **restricted** flag — only a public rollout % and a whitelist are allowed; attribute rules and a gatekeeper stack are rejected. Used as an experiment's `holdout_gate`.
     enabled:
       default: true
       description: Master switch. Defaults to `true`. Set `false` to create the gate disabled (evaluates to `false` regardless of rules/rollout); flip on via `POST /{id}/enable` or PATCH.
@@ -588,6 +601,12 @@ CreateGateResponse:
 UpdateGateRequest:
   type: object
   properties:
+    type:
+      type: string
+      enum:
+        - targeting
+        - holdout
+      description: Gate kind. Switching to `holdout` requires the gate carry only a public rollout % + whitelist (attribute rules / stack are rejected).
     rollout_pct:
       description: New rollout in **basis points** (0–10000 = 0%–100%) — `100` here means **1%**. Use `rollout_percent` (0–100) below for percent. Omit both to leave unchanged.
       type: integer

package/spec/components/schemas/keys.yaml

@@ -114,7 +114,8 @@ CreateKeyRequest:
           - events:write
           - configs:write
           - experiments:write
-      description: Optional permission strings recorded on the key for audit/display.
+          - tickets:public_create
+      description: "Optional permission strings recorded on the key. `tickets:public_create` is enforced: a client key needs it (plus the project's `allowPublicTickets` setting) to file a `pending_approval` bug via the public `POST /cli/report` endpoint. The rest are audit/display only."
     expiresInDays:
       anyOf:
         - type: integer

package/spec/components/schemas/metrics.yaml

@@ -34,8 +34,8 @@ ListMetricsResponse:
       winsorizePct:
         description: Winsorise percentile applied to the metric.
         type: number
-      minDetectableEffect:
-        description: Configured MDE, or `null`.
+      defaultMinEffectOfInterest:
+        description: Metric-level default minimum effect of interest (relative, 0–1), or `null`.
         anyOf:
           - type: number
           - type: "null"
@@ -89,8 +89,8 @@ GetMetricResponse:
     winsorizePct:
       description: Winsorise percentile applied to the metric.
       type: number
-    minDetectableEffect:
-      description: Configured MDE, or `null`.
+    defaultMinEffectOfInterest:
+      description: Metric-level default minimum effect of interest (relative, 0–1), or `null`.
       anyOf:
         - type: number
         - type: "null"
@@ -397,9 +397,9 @@ MetricWinsorizePct:
   type: integer
   minimum: 1
   maximum: 99
-MetricMinDetectableEffect:
+MetricDefaultMinEffectOfInterest:
   default: null
-  description: Minimum detectable effect (relative, 0–1) for power planning. `null` to omit.
+  description: Default minimum effect of interest (relative, 0–1) — the smallest change in this metric worth acting on, used as the power-planning baseline. Intrinsic to the metric; an experiment overrides it per-attachment with `min_effect_of_interest` when a specific decision has a different cost/risk bar. `null` to omit.
   anyOf:
     - type: number
     - type: "null"
@@ -429,8 +429,8 @@ CreateMetricWithQuery:
       $ref: "#/MetricQueryDsl"
     winsorize_pct:
       $ref: "#/MetricWinsorizePct"
-    min_detectable_effect:
-      $ref: "#/MetricMinDetectableEffect"
+    default_min_effect_of_interest:
+      $ref: "#/MetricDefaultMinEffectOfInterest"
     direction:
       $ref: "#/MetricDirection"
   required:
@@ -452,8 +452,8 @@ CreateMetricWithQueryIr:
       $ref: "#/QueryIr"
     winsorize_pct:
       $ref: "#/MetricWinsorizePct"
-    min_detectable_effect:
-      $ref: "#/MetricMinDetectableEffect"
+    default_min_effect_of_interest:
+      $ref: "#/MetricDefaultMinEffectOfInterest"
     direction:
       $ref: "#/MetricDirection"
   required:
@@ -479,8 +479,8 @@ UpdateMetricWithQuery:
       $ref: "#/MetricQueryDsl"
     winsorize_pct:
       $ref: "#/MetricWinsorizePct"
-    min_detectable_effect:
-      $ref: "#/MetricMinDetectableEffect"
+    default_min_effect_of_interest:
+      $ref: "#/MetricDefaultMinEffectOfInterest"
     direction:
       $ref: "#/MetricDirection"
   required:
@@ -498,8 +498,8 @@ UpdateMetricWithQueryIr:
       $ref: "#/QueryIr"
     winsorize_pct:
       $ref: "#/MetricWinsorizePct"
-    min_detectable_effect:
-      $ref: "#/MetricMinDetectableEffect"
+    default_min_effect_of_interest:
+      $ref: "#/MetricDefaultMinEffectOfInterest"
     direction:
       $ref: "#/MetricDirection"
   required:
@@ -515,12 +515,12 @@ UpdateMetricFields:
       $ref: "#/MetricEventName"
     winsorize_pct:
       $ref: "#/MetricWinsorizePct"
-    min_detectable_effect:
-      $ref: "#/MetricMinDetectableEffect"
+    default_min_effect_of_interest:
+      $ref: "#/MetricDefaultMinEffectOfInterest"
     direction:
       $ref: "#/MetricDirection"
   additionalProperties: false
-  description: Update-metric variant that leaves the query untouched (metadata-only edit — folder, event, winsorisation, MDE, direction).
+  description: Update-metric variant that leaves the query untouched (metadata-only edit — folder, event, winsorisation, default minimum effect of interest, direction).
 CreateMetricResponse:
   type: object
   properties: