daishi-sdk 0.2.1 → 0.2.3

package/dist/generated.d.ts

@@ -5,7 +5,7 @@ export type JsonObject = {
 /** The API version these types describe. */
 export declare const API_VERSION = "1.0.0";
 /** Every error code the API documents. */
-export type ErrorCode = 'token_required' | 'scope_required' | 'plan_api' | 'rate_limited' | 'bad_request' | 'bad_scenario' | 'bad_replicates' | 'profile_required' | 'bad_season_ticks' | 'bad_frame' | 'bad_env' | 'too_many_scenarios' | 'unknown_scenario' | 'bad_skill' | 'too_many_skills' | 'unknown_skill' | 'bad_roster' | 'bad_model' | 'bad_name' | 'bad_provider' | 'provider_key_required' | 'openrouter_key_required' | 'openrouter_balance_low' | 'billing_frozen' | 'email_unverified' | 'bad_trials' | 'bad_spend_cap' | 'plan_seats' | 'plan_ticks' | 'plan_series' | 'plan_quota' | 'too_many_runs' | 'run_history_full' | 'seed_not_allowed' | 'unknown_run' | 'unknown_batch' | 'unknown_experiment' | 'experiment_trial' | 'experiment_not_running' | 'experiment_arms' | 'bad_experiment' | 'experiment_scenario' | 'experiment_world_only' | 'experiment_env_seed' | 'experiment_stopping' | 'bad_metric' | 'plan_experiment_arms' | 'unknown_log' | 'plan_invites' | 'invite_world' | 'invite_series' | 'too_many_open_worlds' | 'no_invite' | 'invite_fixed' | 'bad_invite' | 'never_played' | 'run_playing' | 'bad_note' | 'note_too_long' | 'run_launching' | 'internal_error';
+export type ErrorCode = 'token_required' | 'scope_required' | 'plan_api' | 'rate_limited' | 'bad_request' | 'bad_scenario' | 'bad_replicates' | 'profile_required' | 'bad_season_ticks' | 'bad_frame' | 'bad_env' | 'too_many_scenarios' | 'unknown_scenario' | 'bad_skill' | 'too_many_skills' | 'unknown_skill' | 'bad_roster' | 'bad_model' | 'bad_name' | 'bad_provider' | 'provider_key_required' | 'openrouter_key_required' | 'openrouter_balance_low' | 'billing_frozen' | 'email_unverified' | 'bad_trials' | 'bad_spend_cap' | 'plan_seats' | 'plan_ticks' | 'plan_series' | 'plan_quota' | 'too_many_runs' | 'run_history_full' | 'seed_not_allowed' | 'unknown_run' | 'unknown_batch' | 'unknown_experiment' | 'experiment_trial' | 'experiment_not_running' | 'experiment_arms' | 'bad_experiment' | 'experiment_scenario' | 'experiment_world_only' | 'experiment_env_seed' | 'experiment_stopping' | 'bad_metric' | 'experiment_effort_roster' | 'plan_experiment_arms' | 'unknown_log' | 'plan_invites' | 'invite_world' | 'invite_series' | 'too_many_open_worlds' | 'no_invite' | 'invite_fixed' | 'bad_invite' | 'never_played' | 'run_playing' | 'bad_note' | 'note_too_long' | 'run_launching' | 'internal_error';
 export type Method = 'GET' | 'POST' | 'PUT' | 'DELETE';
 export interface RequestParts {
     query?: Record<string, string | number | boolean | undefined | null>;
@@ -124,7 +124,8 @@ export interface Experiment {
     scenario_id: string;
     trials: number;
     seed_mode: 'vary' | 'pinned';
-    metric: 'fitness_index' | 'rank' | 'survived';
+    metric: 'fitness_index' | 'rank' | 'survived' | 'output_tokens_per_turn';
+    secondary_metrics: Array<'fitness_index' | 'rank' | 'survived' | 'output_tokens_per_turn'>;
     rule: StoppingRule;
     arms: ExperimentArmView[];
     created_at_ms: number;
@@ -145,13 +146,14 @@ export interface ExperimentReport {
     scenario_id: string;
     trials: number;
     seed_mode: 'vary' | 'pinned';
-    metric: 'fitness_index' | 'rank' | 'survived';
-    analysis_scale: 'metric' | 'negative_rank';
+    metric: 'fitness_index' | 'rank' | 'survived' | 'output_tokens_per_turn';
+    analysis_scale: 'metric' | 'negative_rank' | 'log';
     rule: StoppingRule;
     rule_text: string;
     arms: ExperimentReportArm[];
     pairs: ExperimentReportPair[];
     comparisons: ExperimentReportComparison[];
+    secondary: ExperimentReportSecondary[];
     detectable_gap: ExperimentReportDetectableGap;
     decision: ExperimentReportDecision;
     verdict: string;
@@ -200,7 +202,7 @@ export interface RosterSeat {
     name?: string;
     /** World runs on Pro and Lab: true makes this a guest seat, played by a peer’s own agent over an invite link on the peer’s keys, inside the world you configured. No model, no key, no skills; the guest’s self-reported identity lands at the claim. The platform caps how many seats a run can invite (the message says the number), a series cannot carry one, and the run waits until its guests arrive. See the runs page. */
     invited?: boolean;
-    /** Reasoning effort where the model supports it. */
+    /** Reasoning effort where the model supports it. A level the model’s route refuses is refused at launch with bad_roster (one short call on the seat’s key checks it). */
     reasoning?: 'off' | 'low' | 'medium' | 'high';
     /** Sampling temperature, passed through to the provider. */
     temperature?: number;
@@ -222,8 +224,10 @@ export interface ExperimentBody {
     name?: string;
     /** vary (default): each trial its own world; pinned: one world for every trial, only the layout varies. A saved scenario that pins its own seed needs pinned; a scenario that draws its worlds from a seed pool refuses it (trial i plays the pool’s world for that trial in every arm, and the pool’s eight worlds come round again from trial nine). A scenario with one fixed world (the lab scenarios) plays it on every trial: its experiment is pinned by default and refuses vary. */
     seed_mode?: 'vary' | 'pinned';
-    /** The per-trial score the arms are compared on, the mean over the arm’s roster seats: the Fitness Index (default), the rank in the field (lower is better, and the report compares on the negative rank so a positive difference always favors the arm), or the share of seats that survived. */
-    metric?: 'fitness_index' | 'rank' | 'survived';
+    /** The per-trial score the arms are compared on, the mean over the arm’s roster seats: the Fitness Index (default), the rank in the field (lower is better, and the report compares on the negative rank so a positive difference always favors the arm), the share of seats that survived, or output tokens per play turn. Output per turn is how much each model generated per turn of play, thinking included (the exit interview and the debrief are not play turns); it compares each model with itself, so every arm must field the same model in the same seat as the control, and it is analyzed on the log scale, the difference read as a percent change. It has no better side: the verdict says which arm generated more. A seat that missed more than a tenth of the rounds is left out of a trial in every arm, and a guest seat is never measured. */
+    metric?: 'fitness_index' | 'rank' | 'survived' | 'output_tokens_per_turn';
+    /** Outcomes reported beside metric, declared now and never changed: any of the metric’s values except the metric itself, each at most once. Each is read on the same pairs with the same rule at the same level, and none stops anything; when a comparison stops, its secondaries take their reading on the pairs it stopped on. Output per turn here holds the arms to the same models in the same seats, as it does as the metric. Default none. */
+    secondary_metrics?: Array<'fitness_index' | 'rank' | 'survived' | 'output_tokens_per_turn'>;
     /** anytime: an anytime-valid 95% confidence sequence on the paired difference, checked after every trial from the tenth on, stopping as soon as it excludes zero or at the last trial; it needs more than 10 trials per arm and is the default above that. fixed: one look at the last trial; the default at 10 or fewer. With more than two arms the level is split across the comparisons against the control. */
     stopping?: 'anytime' | 'fixed';
     /** Anytime rule only: also stop, undecided, once the whole interval lies within plus or minus this on the metric’s scale, so the gap, if any, is smaller than this. */
@@ -506,6 +510,7 @@ export interface ExperimentReportArm {
     agent_turns: number;
     usage: JsonObject;
     awareness: JsonObject;
+    effort?: ExperimentReportArmEffort;
 }
 export interface ExperimentReportArmTrials {
     finished: number;
@@ -521,6 +526,20 @@ export interface ExperimentReportArmScore {
     sd: number | null;
     ci: Interval | null;
 }
+export interface ExperimentReportArmEffort {
+    measured_seats: number;
+    short_seats: number;
+    unmeasured_seats: number;
+    models: ExperimentReportArmEffortModel[];
+}
+export interface ExperimentReportArmEffortModel {
+    model: string;
+    seats: number;
+    output_per_turn: number;
+    thinking_per_turn: number | null;
+    truncated_share: number;
+    served_by: JsonObject;
+}
 export interface ExperimentReportPair {
     trial: number;
     seed: number | null;
@@ -538,8 +557,28 @@ export interface ExperimentReportComparison {
     alpha: number;
     stop: boolean;
     reason: string;
-    verdict: 'arm_better' | 'control_better' | 'undecided' | null;
+    verdict: 'arm_better' | 'control_better' | 'arm_higher' | 'control_higher' | 'undecided' | null;
+    interval: Interval | null;
+    detail: string;
+}
+export interface ExperimentReportSecondary {
+    metric: 'fitness_index' | 'rank' | 'survived' | 'output_tokens_per_turn';
+    analysis_scale: 'metric' | 'negative_rank' | 'log';
+    pairs: ExperimentReportSecondaryPair[];
+    comparisons: ExperimentReportSecondaryComparison[];
+}
+export interface ExperimentReportSecondaryPair {
+    trial: number;
+    values: Array<number | null>;
+    excluded: boolean;
+}
+export interface ExperimentReportSecondaryComparison {
+    arm: number;
+    control: number;
+    n: number;
+    mean_diff: number | null;
     interval: Interval | null;
+    reading: 'arm_better' | 'control_better' | 'arm_higher' | 'control_higher' | 'undecided' | null;
     detail: string;
 }
 export interface ExperimentReportDetectableGap {
@@ -577,7 +616,7 @@ export interface ExperimentArmSeat {
     provider?: string;
     /** Seat name shown in the world. Default: derived from the model. On an invited seat it is the seat’s reserved name until the guest claims it with one of their own. */
     name?: string;
-    /** Reasoning effort where the model supports it. */
+    /** Reasoning effort where the model supports it. A level the model’s route refuses is refused at launch with bad_roster (one short call on the seat’s key checks it). */
     reasoning?: 'off' | 'low' | 'medium' | 'high';
     /** Sampling temperature, passed through to the provider. */
     temperature?: number;

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import { Operations, type ErrorCode, type GetBatchResponse, type GetExperimentResponse, type GetRunResponse, type Method, type RequestParts } from './generated.js';
 export * from './generated.js';
-export declare const VERSION = "0.2.1";
+export declare const VERSION = "0.2.3";
 export declare const DEFAULT_BASE_URL = "https://daishi.ai/api/v1";
 export interface DaishiOptions {
     /** A Studio access token (`dsk_...`). Default: the `DAISHI_TOKEN` environment variable. */

package/dist/index.js

@@ -1,6 +1,6 @@
 import { Operations } from './generated.js';
 export * from './generated.js';
-export const VERSION = '0.2.1';
+export const VERSION = '0.2.3';
 export const DEFAULT_BASE_URL = 'https://daishi.ai/api/v1';
 /** A refusal from the API, or a transport failure after the retries ran out. */
 export class DaishiError extends Error {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "daishi-sdk",
-  "version": "0.2.1",
+  "version": "0.2.3",
   "description": "Typed client for the Daishi Studio API: scenarios, runs, series batches and paired experiments, with retries and waiting helpers.",
   "keywords": [
     "daishi",