@roarkanalytics/sdk 4.7.0 → 4.8.0

package/src/resources/simulation-run-plan-job.ts

@@ -259,6 +259,19 @@ export namespace SimulationRunPlanJobGetByIDResponse {
      */
     startedAt?: string | null;
 
+    /**
+     * For a run that swept a property (accent, background noise, speech pace and so
+     * on): which check failures the property caused.
+     *
+     * Each value is compared with every other value combined using a one-sided Fisher
+     * exact test, and every comparison in the run is corrected together with
+     * Benjamini-Hochberg. A value is only called worse when the difference is
+     * statistically significant, so a value that happened to fail a few more
+     * simulations by chance is not reported as a problem. Invalidated simulations and
+     * simulations that failed on the Roark platform are left out.
+     */
+    sweepAttribution?: Data.SweepAttribution | null;
+
     /**
      * Pass/fail verdict for the run, judged against the success criteria pinned on the
      * run plan when the run started.
@@ -673,6 +686,195 @@ export namespace SimulationRunPlanJobGetByIDResponse {
       }
     }
 
+    /**
+     * For a run that swept a property (accent, background noise, speech pace and so
+     * on): which check failures the property caused.
+     *
+     * Each value is compared with every other value combined using a one-sided Fisher
+     * exact test, and every comparison in the run is corrected together with
+     * Benjamini-Hochberg. A value is only called worse when the difference is
+     * statistically significant, so a value that happened to fail a few more
+     * simulations by chance is not reported as a problem. Invalidated simulations and
+     * simulations that failed on the Roark platform are left out.
+     */
+    export interface SweepAttribution {
+      /**
+       * Plain sentences on what this run could not see, ready to show to a reader.
+       */
+      caveats: Array<string>;
+
+      /**
+       * Every check the run was graded on: `PROPERTY_ATTRIBUTABLE` first, then
+       * `FOUND_UNATTRIBUTED`, then `WITHIN_NOISE`.
+       */
+      checks: Array<SweepAttribution.Check>;
+
+      /**
+       * The Benjamini-Hochberg false discovery rate the comparisons are held to.
+       */
+      falseDiscoveryRate: number;
+
+      /**
+       * The overall failure rate, 0-100, at which a check with no standout value is
+       * reported as `FOUND_UNATTRIBUTED`.
+       */
+      foundUnattributedFailureRate: number;
+
+      /**
+       * Counted simulations a value needs before it is compared.
+       */
+      minArmCalls: number;
+
+      /**
+       * Roughly how many percentage points more often a value would need to fail than
+       * the rest of the run to be flagged at this sample size. Optimistic: it uses
+       * simulation counts rather than the verdicts on each check and ignores the
+       * correction, so checks graded on fewer simulations need larger gaps. Large when
+       * few simulations ran per value: finding no significant difference then means the
+       * run could not see one, not that none exists.
+       */
+      minimumDetectableGap: number | null;
+
+      /**
+       * How many values had enough counted simulations to be compared.
+       */
+      testableValueCount: number;
+
+      /**
+       * How many value and check pairs were actually tested. A value can have enough
+       * simulations and still go untested (no other value graded that check, or too few
+       * verdicts on it). When this is 0 no comparison ran, so an empty `worseValues`
+       * everywhere means nothing was tested, not that no value did worse.
+       */
+      testedComparisonCount: number;
+
+      /**
+       * Every value of the swept property, baseline first.
+       */
+      values: Array<SweepAttribution.Value>;
+    }
+
+    export namespace SweepAttribution {
+      /**
+       * How one check's failures relate to the swept property.
+       */
+      export interface Check {
+        /**
+         * How this check relates to the swept property:
+         *
+         * - `PROPERTY_ATTRIBUTABLE`: at least one value failed it significantly more often
+         *   than every other value combined. `worseValues` names them. This is the only
+         *   case in which a value is called worse.
+         * - `FOUND_UNATTRIBUTED`: no value stands out, but the check failed on at least
+         *   `foundUnattributedFailureRate`% of counted simulations overall. A real issue
+         *   with the agent on which no value stood out, so the run cannot tie it to the
+         *   property. It does not show the property had no effect: a small run may be
+         *   unable to see one.
+         * - `WITHIN_NOISE`: neither. Any differences between values are within what chance
+         *   produces.
+         */
+        attribution: 'PROPERTY_ATTRIBUTABLE' | 'FOUND_UNATTRIBUTED' | 'WITHIN_NOISE';
+
+        evaluated: number;
+
+        failed: number;
+
+        /**
+         * Share of counted simulations across every value that failed the check, 0-100.
+         */
+        failureRate: number | null;
+
+        metricDefinitionId: string;
+
+        metricName: string;
+
+        /**
+         * The values significantly worse than the rest, most significant first. Empty
+         * unless `attribution` is `PROPERTY_ATTRIBUTABLE`.
+         */
+        worseValues: Array<Check.WorseValue>;
+      }
+
+      export namespace Check {
+        /**
+         * A value that failed a check significantly more often than the rest of the run.
+         */
+        export interface WorseValue {
+          /**
+           * How likely a difference at least this large would be by chance alone, after
+           * correcting for every comparison in the run (Benjamini-Hochberg). The value is
+           * called worse only when this is at most `falseDiscoveryRate`.
+           */
+          adjustedPValue: number;
+
+          evaluated: number;
+
+          failed: number;
+
+          /**
+           * Share of the counted simulations at this value that failed the check, 0-100.
+           */
+          failureRate: number | null;
+
+          label: string;
+
+          /**
+           * The same share across every other value combined, 0-100. This is what the value
+           * is compared with.
+           */
+          restFailureRate: number | null;
+
+          value: string;
+        }
+      }
+
+      /**
+       * One value of the swept property.
+       */
+      export interface Value {
+        /**
+         * Simulations run at this value. Simulations that failed on the Roark platform are
+         * left out, since they say nothing about your agent.
+         */
+        attempted: number;
+
+        /**
+         * Simulations that actually tested the property: not invalidated, and graded by at
+         * least one check. Only these are used in the comparison.
+         */
+        counted: number;
+
+        /**
+         * Whether this is the baseline the plan named.
+         */
+        isBaseline: boolean;
+
+        /**
+         * The value in words, e.g. `American`.
+         */
+        label: string;
+
+        /**
+         * The mean of each check's pass rate at this value, 0-100, the same rule as the
+         * run's `score`. Descriptive only: a lower score alone never makes a value worse.
+         * Null when nothing counted.
+         */
+        score: number | null;
+
+        /**
+         * Whether this value had at least `minArmCalls` counted simulations. A value below
+         * that is "insufficient data": it is reported with its numbers but never compared
+         * or ranked, because a rate from one or two calls cannot be told apart from luck.
+         */
+        testable: boolean;
+
+        /**
+         * The stored value of the swept property, e.g. `US`.
+         */
+        value: string;
+      }
+    }
+
     /**
      * Pass/fail verdict for the run, judged against the success criteria pinned on the
      * run plan when the run started.

package/src/resources/simulation-run-plan.ts

@@ -139,9 +139,11 @@ export namespace SimulationRunPlanCreateResponse {
       agentEndpoints: Array<RunPlan.AgentEndpoint>;
 
       /**
-       * The value of `comparisonProperty` every other value is measured against, such as
-       * `NONE` for `BACKGROUND_NOISE`. `null` when no comparison is declared, or when
-       * the property has no obvious norm and none was chosen.
+       * The reference value of `comparisonProperty`, such as `NONE` for
+       * `BACKGROUND_NOISE`: shown first in the results. Whether a value did
+       * significantly worse does not depend on it: that is decided against every other
+       * value combined (see `sweepAttribution`). `null` when no comparison is declared,
+       * or when the property has no obvious norm and none was chosen.
        */
       comparisonBaseline: string | null;
 
@@ -491,9 +493,11 @@ export namespace SimulationRunPlanUpdateResponse {
     agentEndpoints: Array<Data.AgentEndpoint>;
 
     /**
-     * The value of `comparisonProperty` every other value is measured against, such as
-     * `NONE` for `BACKGROUND_NOISE`. `null` when no comparison is declared, or when
-     * the property has no obvious norm and none was chosen.
+     * The reference value of `comparisonProperty`, such as `NONE` for
+     * `BACKGROUND_NOISE`: shown first in the results. Whether a value did
+     * significantly worse does not depend on it: that is decided against every other
+     * value combined (see `sweepAttribution`). `null` when no comparison is declared,
+     * or when the property has no obvious norm and none was chosen.
      */
     comparisonBaseline: string | null;
 
@@ -807,9 +811,11 @@ export namespace SimulationRunPlanListResponse {
     agentEndpoints: Array<Data.AgentEndpoint>;
 
     /**
-     * The value of `comparisonProperty` every other value is measured against, such as
-     * `NONE` for `BACKGROUND_NOISE`. `null` when no comparison is declared, or when
-     * the property has no obvious norm and none was chosen.
+     * The reference value of `comparisonProperty`, such as `NONE` for
+     * `BACKGROUND_NOISE`: shown first in the results. Whether a value did
+     * significantly worse does not depend on it: that is decided against every other
+     * value combined (see `sweepAttribution`). `null` when no comparison is declared,
+     * or when the property has no obvious norm and none was chosen.
      */
     comparisonBaseline: string | null;
 
@@ -1151,9 +1157,11 @@ export namespace SimulationRunPlanGetByIDResponse {
     agentEndpoints: Array<Data.AgentEndpoint>;
 
     /**
-     * The value of `comparisonProperty` every other value is measured against, such as
-     * `NONE` for `BACKGROUND_NOISE`. `null` when no comparison is declared, or when
-     * the property has no obvious norm and none was chosen.
+     * The reference value of `comparisonProperty`, such as `NONE` for
+     * `BACKGROUND_NOISE`: shown first in the results. Whether a value did
+     * significantly worse does not depend on it: that is decided against every other
+     * value combined (see `sweepAttribution`). `null` when no comparison is declared,
+     * or when the property has no obvious norm and none was chosen.
      */
     comparisonBaseline: string | null;
 
@@ -1477,9 +1485,11 @@ export interface SimulationRunPlanCreateParams {
   autoRun?: boolean;
 
   /**
-   * The value of `comparisonProperty` every other value is measured against, for
-   * example `NONE` for `BACKGROUND_NOISE` or `NORMAL` for `SPEECH_PACE`. Must be a
-   * value that property can take.
+   * The reference value of `comparisonProperty`, for example `NONE` for
+   * `BACKGROUND_NOISE` or `NORMAL` for `SPEECH_PACE`: shown first in the results.
+   * Must be a value that property can take. Whether a value did significantly worse
+   * does not depend on it: that is decided against every other value combined (see
+   * `sweepAttribution`).
    *
    * Stored rather than assumed, so the report can say "compared against US accent"
    * instead of implying Roark decided which value is normal. Most properties have an
@@ -1810,7 +1820,7 @@ export interface SimulationRunPlanUpdateParams {
   agentEndpoints?: Array<SimulationRunPlanUpdateParams.AgentEndpoint>;
 
   /**
-   * The value every other value is measured against. See `POST /v1/simulation/plan`.
+   * The reference value, shown first in the results. See `POST /v1/simulation/plan`.
    *
    * A real value cannot be sent on its own: the property it belongs to decides which
    * values are legal, and an omitted property means "leave unchanged", which this

package/src/resources/simulation-template.ts

@@ -210,9 +210,10 @@ export namespace SimulationTemplateListResponse {
      */
     export interface Sweep {
       /**
-       * The value the others are measured against, resolved to what a plan built from
-       * this template will actually record. `null` when the property has no obvious
-       * norm, and the report then compares against the best-performing value instead.
+       * The reference value, shown first in the results, resolved to what a plan built
+       * from this template will actually record. `null` when the property has no obvious
+       * norm. It does not change the verdict: each value is compared with every other
+       * value combined.
        */
       baseline: string | null;
 

package/src/resources/simulation.ts

@@ -276,9 +276,11 @@ export declare namespace SimulationRunParams {
       metrics: Array<Plan.Metric>;
 
       /**
-       * The value of `comparisonProperty` every other value is measured against, for
-       * example `NONE` for `BACKGROUND_NOISE` or `NORMAL` for `SPEECH_PACE`. Must be a
-       * value that property can take.
+       * The reference value of `comparisonProperty`, for example `NONE` for
+       * `BACKGROUND_NOISE` or `NORMAL` for `SPEECH_PACE`: shown first in the results.
+       * Must be a value that property can take. Whether a value did significantly worse
+       * does not depend on it: that is decided against every other value combined (see
+       * `sweepAttribution`).
        *
        * Stored rather than assumed, so the report can say "compared against US accent"
        * instead of implying Roark decided which value is normal. Most properties have an
@@ -734,8 +736,10 @@ export declare namespace SimulationRunParams {
     template: string;
 
     /**
-     * The value of the sweep every other value is measured against. Defaults to the
-     * template's own baseline, as returned by GET /v1/simulation/template.
+     * The sweep's reference value, shown first in the results. Defaults to the
+     * template's own baseline, as returned by GET /v1/simulation/template. Whether a
+     * value did significantly worse does not depend on it: that is decided against
+     * every other value combined.
      *
      * Send it with `comparisonValues` and it must be one of them, or the request is
      * rejected: anchoring every difference to an arm the run never made would measure
@@ -802,7 +806,9 @@ export declare namespace SimulationRunParams {
     flows?: Array<RunSimulationFromTemplate.Flow>;
 
     /**
-     * Number of iterations to run for each test case (1-10000)
+     * Runs per test case (1-10000). Defaults to 1, or to 6 for a template that sweeps
+     * a property. A sweep needs at least 5 calls per value (test cases per value times
+     * iterations) to compare its values, and a lower count is refused with 400.
      */
     iterationCount?: number;
 

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = '4.7.0'; // x-release-please-version
+export const VERSION = '4.8.0'; // x-release-please-version

package/version.d.mts

@@ -1,2 +1,2 @@
-export declare const VERSION = "4.7.0";
+export declare const VERSION = "4.8.0";
 //# sourceMappingURL=version.d.mts.map

package/version.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "4.7.0";
+export declare const VERSION = "4.8.0";
 //# sourceMappingURL=version.d.ts.map

package/version.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '4.7.0'; // x-release-please-version
+exports.VERSION = '4.8.0'; // x-release-please-version
 //# sourceMappingURL=version.js.map

package/version.mjs

@@ -1,2 +1,2 @@
-export const VERSION = '4.7.0'; // x-release-please-version
+export const VERSION = '4.8.0'; // x-release-please-version
 //# sourceMappingURL=version.mjs.map