daishi-sdk 0.2.1 → 0.2.2

package/dist/generated.d.ts

@@ -5,7 +5,7 @@ export type JsonObject = {
 /** The API version these types describe. */
 export declare const API_VERSION = "1.0.0";
 /** Every error code the API documents. */
-export type ErrorCode = 'token_required' | 'scope_required' | 'plan_api' | 'rate_limited' | 'bad_request' | 'bad_scenario' | 'bad_replicates' | 'profile_required' | 'bad_season_ticks' | 'bad_frame' | 'bad_env' | 'too_many_scenarios' | 'unknown_scenario' | 'bad_skill' | 'too_many_skills' | 'unknown_skill' | 'bad_roster' | 'bad_model' | 'bad_name' | 'bad_provider' | 'provider_key_required' | 'openrouter_key_required' | 'openrouter_balance_low' | 'billing_frozen' | 'email_unverified' | 'bad_trials' | 'bad_spend_cap' | 'plan_seats' | 'plan_ticks' | 'plan_series' | 'plan_quota' | 'too_many_runs' | 'run_history_full' | 'seed_not_allowed' | 'unknown_run' | 'unknown_batch' | 'unknown_experiment' | 'experiment_trial' | 'experiment_not_running' | 'experiment_arms' | 'bad_experiment' | 'experiment_scenario' | 'experiment_world_only' | 'experiment_env_seed' | 'experiment_stopping' | 'bad_metric' | 'plan_experiment_arms' | 'unknown_log' | 'plan_invites' | 'invite_world' | 'invite_series' | 'too_many_open_worlds' | 'no_invite' | 'invite_fixed' | 'bad_invite' | 'never_played' | 'run_playing' | 'bad_note' | 'note_too_long' | 'run_launching' | 'internal_error';
+export type ErrorCode = 'token_required' | 'scope_required' | 'plan_api' | 'rate_limited' | 'bad_request' | 'bad_scenario' | 'bad_replicates' | 'profile_required' | 'bad_season_ticks' | 'bad_frame' | 'bad_env' | 'too_many_scenarios' | 'unknown_scenario' | 'bad_skill' | 'too_many_skills' | 'unknown_skill' | 'bad_roster' | 'bad_model' | 'bad_name' | 'bad_provider' | 'provider_key_required' | 'openrouter_key_required' | 'openrouter_balance_low' | 'billing_frozen' | 'email_unverified' | 'bad_trials' | 'bad_spend_cap' | 'plan_seats' | 'plan_ticks' | 'plan_series' | 'plan_quota' | 'too_many_runs' | 'run_history_full' | 'seed_not_allowed' | 'unknown_run' | 'unknown_batch' | 'unknown_experiment' | 'experiment_trial' | 'experiment_not_running' | 'experiment_arms' | 'bad_experiment' | 'experiment_scenario' | 'experiment_world_only' | 'experiment_env_seed' | 'experiment_stopping' | 'bad_metric' | 'experiment_effort_roster' | 'plan_experiment_arms' | 'unknown_log' | 'plan_invites' | 'invite_world' | 'invite_series' | 'too_many_open_worlds' | 'no_invite' | 'invite_fixed' | 'bad_invite' | 'never_played' | 'run_playing' | 'bad_note' | 'note_too_long' | 'run_launching' | 'internal_error';
 export type Method = 'GET' | 'POST' | 'PUT' | 'DELETE';
 export interface RequestParts {
     query?: Record<string, string | number | boolean | undefined | null>;
@@ -124,7 +124,8 @@ export interface Experiment {
     scenario_id: string;
     trials: number;
     seed_mode: 'vary' | 'pinned';
-    metric: 'fitness_index' | 'rank' | 'survived';
+    metric: 'fitness_index' | 'rank' | 'survived' | 'output_tokens_per_turn';
+    secondary_metrics: Array<'fitness_index' | 'rank' | 'survived' | 'output_tokens_per_turn'>;
     rule: StoppingRule;
     arms: ExperimentArmView[];
     created_at_ms: number;
@@ -145,13 +146,14 @@ export interface ExperimentReport {
     scenario_id: string;
     trials: number;
     seed_mode: 'vary' | 'pinned';
-    metric: 'fitness_index' | 'rank' | 'survived';
-    analysis_scale: 'metric' | 'negative_rank';
+    metric: 'fitness_index' | 'rank' | 'survived' | 'output_tokens_per_turn';
+    analysis_scale: 'metric' | 'negative_rank' | 'log';
     rule: StoppingRule;
     rule_text: string;
     arms: ExperimentReportArm[];
     pairs: ExperimentReportPair[];
     comparisons: ExperimentReportComparison[];
+    secondary: ExperimentReportSecondary[];
     detectable_gap: ExperimentReportDetectableGap;
     decision: ExperimentReportDecision;
     verdict: string;
@@ -222,8 +224,10 @@ export interface ExperimentBody {
     name?: string;
     /** vary (default): each trial its own world; pinned: one world for every trial, only the layout varies. A saved scenario that pins its own seed needs pinned; a scenario that draws its worlds from a seed pool refuses it (trial i plays the pool’s world for that trial in every arm, and the pool’s eight worlds come round again from trial nine). A scenario with one fixed world (the lab scenarios) plays it on every trial: its experiment is pinned by default and refuses vary. */
     seed_mode?: 'vary' | 'pinned';
-    /** The per-trial score the arms are compared on, the mean over the arm’s roster seats: the Fitness Index (default), the rank in the field (lower is better, and the report compares on the negative rank so a positive difference always favors the arm), or the share of seats that survived. */
-    metric?: 'fitness_index' | 'rank' | 'survived';
+    /** The per-trial score the arms are compared on, the mean over the arm’s roster seats: the Fitness Index (default), the rank in the field (lower is better, and the report compares on the negative rank so a positive difference always favors the arm), the share of seats that survived, or output tokens per play turn. Output per turn is how much each model generated per turn of play, thinking included (the exit interview and the debrief are not play turns); it compares each model with itself, so every arm must field the same model in the same seat as the control, and it is analyzed on the log scale, the difference read as a percent change. It has no better side: the verdict says which arm generated more. A seat that missed more than a tenth of the rounds is left out of a trial in every arm, and a guest seat is never measured. */
+    metric?: 'fitness_index' | 'rank' | 'survived' | 'output_tokens_per_turn';
+    /** Outcomes reported beside metric, declared now and never changed: any of the metric’s values except the metric itself, each at most once. Each is read on the same pairs with the same rule at the same level, and none stops anything; when a comparison stops, its secondaries take their reading on the pairs it stopped on. Output per turn here holds the arms to the same models in the same seats, as it does as the metric. Default none. */
+    secondary_metrics?: Array<'fitness_index' | 'rank' | 'survived' | 'output_tokens_per_turn'>;
     /** anytime: an anytime-valid 95% confidence sequence on the paired difference, checked after every trial from the tenth on, stopping as soon as it excludes zero or at the last trial; it needs more than 10 trials per arm and is the default above that. fixed: one look at the last trial; the default at 10 or fewer. With more than two arms the level is split across the comparisons against the control. */
     stopping?: 'anytime' | 'fixed';
     /** Anytime rule only: also stop, undecided, once the whole interval lies within plus or minus this on the metric’s scale, so the gap, if any, is smaller than this. */
@@ -506,6 +510,7 @@ export interface ExperimentReportArm {
     agent_turns: number;
     usage: JsonObject;
     awareness: JsonObject;
+    effort?: ExperimentReportArmEffort;
 }
 export interface ExperimentReportArmTrials {
     finished: number;
@@ -521,6 +526,20 @@ export interface ExperimentReportArmScore {
     sd: number | null;
     ci: Interval | null;
 }
+export interface ExperimentReportArmEffort {
+    measured_seats: number;
+    short_seats: number;
+    unmeasured_seats: number;
+    models: ExperimentReportArmEffortModel[];
+}
+export interface ExperimentReportArmEffortModel {
+    model: string;
+    seats: number;
+    output_per_turn: number;
+    thinking_per_turn: number | null;
+    truncated_share: number;
+    served_by: JsonObject;
+}
 export interface ExperimentReportPair {
     trial: number;
     seed: number | null;
@@ -538,8 +557,28 @@ export interface ExperimentReportComparison {
     alpha: number;
     stop: boolean;
     reason: string;
-    verdict: 'arm_better' | 'control_better' | 'undecided' | null;
+    verdict: 'arm_better' | 'control_better' | 'arm_higher' | 'control_higher' | 'undecided' | null;
+    interval: Interval | null;
+    detail: string;
+}
+export interface ExperimentReportSecondary {
+    metric: 'fitness_index' | 'rank' | 'survived' | 'output_tokens_per_turn';
+    analysis_scale: 'metric' | 'negative_rank' | 'log';
+    pairs: ExperimentReportSecondaryPair[];
+    comparisons: ExperimentReportSecondaryComparison[];
+}
+export interface ExperimentReportSecondaryPair {
+    trial: number;
+    values: Array<number | null>;
+    excluded: boolean;
+}
+export interface ExperimentReportSecondaryComparison {
+    arm: number;
+    control: number;
+    n: number;
+    mean_diff: number | null;
     interval: Interval | null;
+    reading: 'arm_better' | 'control_better' | 'arm_higher' | 'control_higher' | 'undecided' | null;
     detail: string;
 }
 export interface ExperimentReportDetectableGap {

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import { Operations, type ErrorCode, type GetBatchResponse, type GetExperimentResponse, type GetRunResponse, type Method, type RequestParts } from './generated.js';
 export * from './generated.js';
-export declare const VERSION = "0.2.1";
+export declare const VERSION = "0.2.2";
 export declare const DEFAULT_BASE_URL = "https://daishi.ai/api/v1";
 export interface DaishiOptions {
     /** A Studio access token (`dsk_...`). Default: the `DAISHI_TOKEN` environment variable. */

package/dist/index.js

@@ -1,6 +1,6 @@
 import { Operations } from './generated.js';
 export * from './generated.js';
-export const VERSION = '0.2.1';
+export const VERSION = '0.2.2';
 export const DEFAULT_BASE_URL = 'https://daishi.ai/api/v1';
 /** A refusal from the API, or a transport failure after the retries ran out. */
 export class DaishiError extends Error {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "daishi-sdk",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "Typed client for the Daishi Studio API: scenarios, runs, series batches and paired experiments, with retries and waiting helpers.",
   "keywords": [
     "daishi",