@roarkanalytics/sdk 4.20.0 → 4.22.0

package/src/resources/simulation-run-plan-job.ts

@@ -212,13 +212,27 @@ export namespace SimulationRunPlanJobGetByIDResponse {
    * Simulation run plan job with all associated simulation jobs
    */
   export interface Data {
+    /**
+     * The run’s simulations against its test cases. Differs from a plain job count
+     * only when the plan retries simulations your agent never spoke on.
+     */
+    attemptSummary: Data.AttemptSummary;
+
     /**
      * When the job was created
      */
     createdAt: string;
 
     /**
-     * List of simulation jobs (calls) in this run plan job
+     * Retries waiting out their backoff, soonest first. While this is not empty the
+     * run is waiting, not stuck: it settles only once every test case has a final
+     * attempt. Empty when nothing is scheduled.
+     */
+    pendingRetries: Array<Data.PendingRetry>;
+
+    /**
+     * List of simulation jobs (calls) in this run plan job, every attempt included: a
+     * simulation your agent never spoke on stays listed after a retry replaces it.
      */
     simulationJobs: Array<Data.SimulationJob>;
 
@@ -280,12 +294,80 @@ export namespace SimulationRunPlanJobGetByIDResponse {
   }
 
   export namespace Data {
+    /**
+     * The run’s simulations against its test cases. Differs from a plain job count
+     * only when the plan retries simulations your agent never spoke on.
+     */
+    export interface AttemptSummary {
+      /**
+       * Every simulation placed, retries included. Each is a separate, billed call.
+       */
+      attemptCount: number;
+
+      /**
+       * Simulations placed as a retry of one your agent never spoke on.
+       */
+      retryCount: number;
+
+      /**
+       * Simulations your agent never spoke on, including the ones a retry replaced.
+       */
+      silentAttemptCount: number;
+
+      /**
+       * Test cases whose final attempt your agent still never spoke on. This is what the
+       * `AGENT_NEVER_SPOKE` verdict failure counts.
+       */
+      stillSilentTestCaseCount: number;
+
+      /**
+       * Test cases in the run. Each one has a single final attempt that its result is
+       * read from.
+       */
+      testCaseCount: number;
+    }
+
+    /**
+     * A retry of a simulation your agent never spoke on, waiting out its backoff
+     * before it dials.
+     */
+    export interface PendingRetry {
+      /**
+       * Its place among its test case’s attempts, 2 for the first retry.
+       */
+      attemptNumber: number;
+
+      /**
+       * The most attempts a test case can get on this run: the plan’s
+       * `maxNoResponseRetries` plus 1.
+       */
+      maxAttempts: number;
+
+      /**
+       * When the retry dials, ISO 8601. It may wait longer behind the run’s concurrency
+       * limit.
+       */
+      scheduledAt: string | null;
+
+      /**
+       * The scheduled retry.
+       */
+      simulationJobId: string;
+    }
+
     export interface SimulationJob {
       /**
        * Agent endpoint used in a simulation
        */
       agentEndpoint: SimulationJob.AgentEndpoint;
 
+      /**
+       * This simulation’s place among its test case’s attempts: 1 for the first, 2 for
+       * the first retry. Above 1 only when the plan retries simulations your agent never
+       * spoke on.
+       */
+      attemptNumber: number;
+
       /**
        * The background noise the call actually ran with. `persona.backgroundNoise` is
        * only what the persona was set to; a flow’s environment overrides it, and on a
@@ -325,18 +407,30 @@ export namespace SimulationRunPlanJobGetByIDResponse {
         | 'COLLECTING_METRICS'
         | 'COMPLETED';
 
+      /**
+       * The simulation this one retries, because your agent never spoke on it. Null on a
+       * test case’s first attempt.
+       */
+      retryOfSimulationJobId: string | null;
+
       /**
        * Scenario used in a simulation
        */
       scenario: SimulationJob.Scenario;
 
+      /**
+       * When a `RETRY_SCHEDULED` retry dials, ISO 8601. Null on a first attempt.
+       */
+      scheduledAt: string | null;
+
       /**
        * Simulation job ID
        */
       simulationJobId: string;
 
       /**
-       * Job status
+       * Job status. `RETRY_SCHEDULED` is a retry of a simulation your agent never spoke
+       * on, waiting out the plan’s `noResponseRetryBackoffSeconds` before it dials.
        */
       status:
         | 'PENDING'
@@ -346,7 +440,8 @@ export namespace SimulationRunPlanJobGetByIDResponse {
         | 'FAILED'
         | 'TIMED_OUT'
         | 'CANCELLED'
-        | 'CANCELLING';
+        | 'CANCELLING'
+        | 'RETRY_SCHEDULED';
 
       /**
        * ID of the call created for this simulation job. Null if the call has not been
@@ -789,6 +884,13 @@ export namespace SimulationRunPlanJobGetByIDResponse {
        */
       minimumDetectableGap: number | null;
 
+      /**
+       * The values your agent never spoke on significantly more often than the rest of
+       * the run, most significant first, tested the same way as `worseValues`. Empty on
+       * a run with no silent simulations.
+       */
+      neverSpokeValues: Array<SweepAttribution.NeverSpokeValue>;
+
       /**
        * How many values had enough counted simulations to be compared.
        */
@@ -882,6 +984,47 @@ export namespace SimulationRunPlanJobGetByIDResponse {
         }
       }
 
+      /**
+       * A value your agent never spoke on significantly more often than the rest of the
+       * run.
+       */
+      export interface NeverSpokeValue {
+        /**
+         * How likely a difference at least this large would be by chance alone, after
+         * correcting across the values (Benjamini-Hochberg).
+         */
+        adjustedPValue: number;
+
+        /**
+         * Simulations run at this value.
+         */
+        attempted: number;
+
+        label: string;
+
+        /**
+         * Simulations run across every other value combined.
+         */
+        restAttempted: number;
+
+        /**
+         * Simulations your agent never spoke on across every other value combined.
+         */
+        restSilentAttempts: number;
+
+        /**
+         * Retries placed at this value because your agent never spoke.
+         */
+        retries: number;
+
+        /**
+         * Simulations at this value your agent never spoke on, retried or not.
+         */
+        silentAttempts: number;
+
+        value: string;
+      }
+
       /**
        * One value of the swept property.
        */
@@ -908,6 +1051,12 @@ export namespace SimulationRunPlanJobGetByIDResponse {
          */
         label: string;
 
+        /**
+         * Of `attempted`, the retries of simulations your agent never spoke on. Zero
+         * unless the plan retries silent simulations.
+         */
+        retries: number;
+
         /**
          * The mean of each check's pass rate at this value, 0-100, the same rule as the
          * run's `score`. Descriptive only: a lower score alone never makes a value worse.
@@ -915,6 +1064,12 @@ export namespace SimulationRunPlanJobGetByIDResponse {
          */
         score: number | null;
 
+        /**
+         * Of `attempted`, the simulations your agent never spoke on, whether or not a
+         * retry followed. They are invalidated, so they are never in `counted`.
+         */
+        silentAttempts: number;
+
         /**
          * Whether this value had at least `minArmCalls` counted simulations. A value below
          * that is "insufficient data": it is reported with its numbers but never compared
@@ -1063,12 +1218,13 @@ export namespace SimulationRunPlanJobGetByIDResponse {
         maxShare: number;
 
         /**
-         * Simulations your agent answered and never spoke on.
+         * Test cases your agent never spoke on, judged on each test case’s last attempt: a
+         * silent simulation a retry later reached your agent on does not count here.
          */
         neverSpokeCalls: number;
 
         /**
-         * Every simulation of the run that reached your agent.
+         * Every test case of the run whose last attempt reached your agent.
          */
         totalCalls: number;
 
@@ -1079,6 +1235,13 @@ export namespace SimulationRunPlanJobGetByIDResponse {
          * agent.
          */
         agentName?: string | null;
+
+        /**
+         * Every simulation your agent never spoke on, including the ones a retry replaced.
+         * Above `neverSpokeCalls` only when the plan retries silent simulations
+         * (`maxNoResponseRetries`).
+         */
+        silentAttempts?: number;
       }
 
       export interface UnionMember3 {

package/src/resources/simulation-run-plan.ts

@@ -238,6 +238,19 @@ export namespace SimulationRunPlanCreateResponse {
        */
       maxConcurrentJobs: number;
 
+      /**
+       * How many more times to run a test case when the agent under test never responds:
+       * it never speaks on a call or never replies in a chat (0-10). 0 turns retries
+       * off. Failed checks and failures on Roark’s side are never retried.
+       *
+       * Each retry is a separate attempt, billed like any other, so a plan retrying N
+       * times can place up to N + 1 calls per test case. Every silent attempt stays on
+       * the run with its own call; the run settles once each test case has a final
+       * attempt, and the agent never spoke verdict is judged on each test case’s last
+       * attempt.
+       */
+      maxNoResponseRetries: number;
+
       /**
        * Maximum duration in seconds for each simulation
        */
@@ -253,6 +266,12 @@ export namespace SimulationRunPlanCreateResponse {
        */
       name: string;
 
+      /**
+       * Seconds a retry waits before it dials (30-600). Only used when
+       * `maxNoResponseRetries` is above 0.
+       */
+      noResponseRetryBackoffSeconds: number;
+
       /**
        * Personas included in this run plan. Only meaningful alongside `scenarios`.
        */
@@ -592,6 +611,19 @@ export namespace SimulationRunPlanUpdateResponse {
      */
     maxConcurrentJobs: number;
 
+    /**
+     * How many more times to run a test case when the agent under test never responds:
+     * it never speaks on a call or never replies in a chat (0-10). 0 turns retries
+     * off. Failed checks and failures on Roark’s side are never retried.
+     *
+     * Each retry is a separate attempt, billed like any other, so a plan retrying N
+     * times can place up to N + 1 calls per test case. Every silent attempt stays on
+     * the run with its own call; the run settles once each test case has a final
+     * attempt, and the agent never spoke verdict is judged on each test case’s last
+     * attempt.
+     */
+    maxNoResponseRetries: number;
+
     /**
      * Maximum duration in seconds for each simulation
      */
@@ -607,6 +639,12 @@ export namespace SimulationRunPlanUpdateResponse {
      */
     name: string;
 
+    /**
+     * Seconds a retry waits before it dials (30-600). Only used when
+     * `maxNoResponseRetries` is above 0.
+     */
+    noResponseRetryBackoffSeconds: number;
+
     /**
      * Personas included in this run plan. Only meaningful alongside `scenarios`.
      */
@@ -910,6 +948,19 @@ export namespace SimulationRunPlanListResponse {
      */
     maxConcurrentJobs: number;
 
+    /**
+     * How many more times to run a test case when the agent under test never responds:
+     * it never speaks on a call or never replies in a chat (0-10). 0 turns retries
+     * off. Failed checks and failures on Roark’s side are never retried.
+     *
+     * Each retry is a separate attempt, billed like any other, so a plan retrying N
+     * times can place up to N + 1 calls per test case. Every silent attempt stays on
+     * the run with its own call; the run settles once each test case has a final
+     * attempt, and the agent never spoke verdict is judged on each test case’s last
+     * attempt.
+     */
+    maxNoResponseRetries: number;
+
     /**
      * Maximum duration in seconds for each simulation
      */
@@ -925,6 +976,12 @@ export namespace SimulationRunPlanListResponse {
      */
     name: string;
 
+    /**
+     * Seconds a retry waits before it dials (30-600). Only used when
+     * `maxNoResponseRetries` is above 0.
+     */
+    noResponseRetryBackoffSeconds: number;
+
     /**
      * Personas included in this run plan. Only meaningful alongside `scenarios`.
      */
@@ -1256,6 +1313,19 @@ export namespace SimulationRunPlanGetByIDResponse {
      */
     maxConcurrentJobs: number;
 
+    /**
+     * How many more times to run a test case when the agent under test never responds:
+     * it never speaks on a call or never replies in a chat (0-10). 0 turns retries
+     * off. Failed checks and failures on Roark’s side are never retried.
+     *
+     * Each retry is a separate attempt, billed like any other, so a plan retrying N
+     * times can place up to N + 1 calls per test case. Every silent attempt stays on
+     * the run with its own call; the run settles once each test case has a final
+     * attempt, and the agent never spoke verdict is judged on each test case’s last
+     * attempt.
+     */
+    maxNoResponseRetries: number;
+
     /**
      * Maximum duration in seconds for each simulation
      */
@@ -1271,6 +1341,12 @@ export namespace SimulationRunPlanGetByIDResponse {
      */
     name: string;
 
+    /**
+     * Seconds a retry waits before it dials (30-600). Only used when
+     * `maxNoResponseRetries` is above 0.
+     */
+    noResponseRetryBackoffSeconds: number;
+
     /**
      * Personas included in this run plan. Only meaningful alongside `scenarios`.
      */
@@ -1619,6 +1695,19 @@ export interface SimulationRunPlanCreateParams {
    */
   maxConcurrentJobs?: number;
 
+  /**
+   * How many more times to run a test case when the agent under test never responds:
+   * it never speaks on a call or never replies in a chat (0-10). 0 turns retries
+   * off. Failed checks and failures on Roark’s side are never retried.
+   *
+   * Each retry is a separate attempt, billed like any other, so a plan retrying N
+   * times can place up to N + 1 calls per test case. Every silent attempt stays on
+   * the run with its own call; the run settles once each test case has a final
+   * attempt, and the agent never spoke verdict is judged on each test case’s last
+   * attempt.
+   */
+  maxNoResponseRetries?: number;
+
   /**
    * Metric definitions to include in this run plan. Reference each by `id` (UUID) or
    * `slug`.
@@ -1630,6 +1719,12 @@ export interface SimulationRunPlanCreateParams {
    */
   metrics?: Array<SimulationRunPlanCreateParams.Metric>;
 
+  /**
+   * Seconds a retry waits before it dials (30-600). Only used when
+   * `maxNoResponseRetries` is above 0.
+   */
+  noResponseRetryBackoffSeconds?: number;
+
   /**
    * Personas to include in this run plan. Required with `scenarios`; ignored with
    * `flows`, where each variant carries its own persona.
@@ -1942,6 +2037,19 @@ export interface SimulationRunPlanUpdateParams {
    */
   maxConcurrentJobs?: number;
 
+  /**
+   * How many more times to run a test case when the agent under test never responds:
+   * it never speaks on a call or never replies in a chat (0-10). 0 turns retries
+   * off. Failed checks and failures on Roark’s side are never retried.
+   *
+   * Each retry is a separate attempt, billed like any other, so a plan retrying N
+   * times can place up to N + 1 calls per test case. Every silent attempt stays on
+   * the run with its own call; the run settles once each test case has a final
+   * attempt, and the agent never spoke verdict is judged on each test case’s last
+   * attempt.
+   */
+  maxNoResponseRetries?: number;
+
   /**
    * Maximum duration in seconds for each simulation
    */
@@ -1958,6 +2066,12 @@ export interface SimulationRunPlanUpdateParams {
    */
   name?: string;
 
+  /**
+   * Seconds a retry waits before it dials (30-600). Only used when
+   * `maxNoResponseRetries` is above 0.
+   */
+  noResponseRetryBackoffSeconds?: number;
+
   /**
    * Personas to include in this run plan
    */

package/src/resources/simulation.ts

@@ -410,6 +410,19 @@ export declare namespace SimulationRunParams {
        */
       maxConcurrentJobs?: number;
 
+      /**
+       * How many more times to run a test case when the agent under test never responds:
+       * it never speaks on a call or never replies in a chat (0-10). 0 turns retries
+       * off. Failed checks and failures on Roark’s side are never retried.
+       *
+       * Each retry is a separate attempt, billed like any other, so a plan retrying N
+       * times can place up to N + 1 calls per test case. Every silent attempt stays on
+       * the run with its own call; the run settles once each test case has a final
+       * attempt, and the agent never spoke verdict is judged on each test case’s last
+       * attempt.
+       */
+      maxNoResponseRetries?: number;
+
       /**
        * Metric definitions to include in this run plan. Reference each by `id` (UUID) or
        * `slug`.
@@ -427,6 +440,12 @@ export declare namespace SimulationRunParams {
        */
       name?: string;
 
+      /**
+       * Seconds a retry waits before it dials (30-600). Only used when
+       * `maxNoResponseRetries` is above 0.
+       */
+      noResponseRetryBackoffSeconds?: number;
+
       /**
        * Personas to include in this run plan. Required with `scenarios`; ignored with
        * `flows`, where each variant carries its own persona.
@@ -826,6 +845,19 @@ export declare namespace SimulationRunParams {
      */
     maxConcurrentJobs?: number;
 
+    /**
+     * How many more times to run a test case when the agent under test never responds:
+     * it never speaks on a call or never replies in a chat (0-10). 0 turns retries
+     * off. Failed checks and failures on Roark’s side are never retried.
+     *
+     * Each retry is a separate attempt, billed like any other, so a plan retrying N
+     * times can place up to N + 1 calls per test case. Every silent attempt stays on
+     * the run with its own call; the run settles once each test case has a final
+     * attempt, and the agent never spoke verdict is judged on each test case’s last
+     * attempt.
+     */
+    maxNoResponseRetries?: number;
+
     /**
      * Defaults to the template's `defaultMaxSimulationDurationSeconds`, as returned by
      * GET /v1/simulation/template.
@@ -838,6 +870,12 @@ export declare namespace SimulationRunParams {
      */
     name?: string;
 
+    /**
+     * Seconds a retry waits before it dials (30-600). Only used when
+     * `maxNoResponseRetries` is above 0.
+     */
+    noResponseRetryBackoffSeconds?: number;
+
     /**
      * For `question-answer-check`: the persona that asks the questions.
      */

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = '4.20.0'; // x-release-please-version
+export const VERSION = '4.22.0'; // x-release-please-version

package/version.d.mts

@@ -1,2 +1,2 @@
-export declare const VERSION = "4.20.0";
+export declare const VERSION = "4.22.0";
 //# sourceMappingURL=version.d.mts.map

package/version.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "4.20.0";
+export declare const VERSION = "4.22.0";
 //# sourceMappingURL=version.d.ts.map

package/version.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '4.20.0'; // x-release-please-version
+exports.VERSION = '4.22.0'; // x-release-please-version
 //# sourceMappingURL=version.js.map

package/version.mjs

@@ -1,2 +1,2 @@
-export const VERSION = '4.20.0'; // x-release-please-version
+export const VERSION = '4.22.0'; // x-release-please-version
 //# sourceMappingURL=version.mjs.map