@ora-ai/cli 0.24.0 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7169,6 +7169,10 @@ export type StudyListResponse = {
7169
7169
  "nextAttemptAt"?: string;
7170
7170
  "reason"?: string;
7171
7171
  };
7172
+ "stopped"?: {
7173
+ "at": string;
7174
+ "notStarted": number;
7175
+ };
7172
7176
  } | null;
7173
7177
  "pacing"?: {
7174
7178
  "maxInFlight"?: number;
@@ -7622,6 +7626,10 @@ export type StudyCreateResponse = {
7622
7626
  "nextAttemptAt"?: string;
7623
7627
  "reason"?: string;
7624
7628
  };
7629
+ "stopped"?: {
7630
+ "at": string;
7631
+ "notStarted": number;
7632
+ };
7625
7633
  } | null;
7626
7634
  "pacing"?: {
7627
7635
  "maxInFlight"?: number;
@@ -7715,6 +7723,10 @@ export type StudyComparisonModesListResponse = {
7715
7723
  "enabled": boolean;
7716
7724
  "max": number;
7717
7725
  };
7726
+ "runScreens": {
7727
+ "enabled": Array<string>;
7728
+ "defaults": Array<string>;
7729
+ };
7718
7730
  };
7719
7731
  export type StudyConvertWebmcpSuiteRequest = {
7720
7732
  "suite": unknown;
@@ -8516,6 +8528,10 @@ export type StudyGetResponse = {
8516
8528
  "nextAttemptAt"?: string;
8517
8529
  "reason"?: string;
8518
8530
  };
8531
+ "stopped"?: {
8532
+ "at": string;
8533
+ "notStarted": number;
8534
+ };
8519
8535
  } | null;
8520
8536
  "pacing"?: {
8521
8537
  "maxInFlight"?: number;
@@ -10104,6 +10120,10 @@ export type StudyUpdateDetailsResponse = {
10104
10120
  "nextAttemptAt"?: string;
10105
10121
  "reason"?: string;
10106
10122
  };
10123
+ "stopped"?: {
10124
+ "at": string;
10125
+ "notStarted": number;
10126
+ };
10107
10127
  } | null;
10108
10128
  "pacing"?: {
10109
10129
  "maxInFlight"?: number;
@@ -10276,6 +10296,10 @@ export type StudyAddAgentsResponse = {
10276
10296
  "nextAttemptAt"?: string;
10277
10297
  "reason"?: string;
10278
10298
  };
10299
+ "stopped"?: {
10300
+ "at": string;
10301
+ "notStarted": number;
10302
+ };
10279
10303
  } | null;
10280
10304
  "relaunch": {
10281
10305
  "started": boolean;
@@ -10577,6 +10601,10 @@ export type StudyGenerateResponse = {
10577
10601
  "nextAttemptAt"?: string;
10578
10602
  "reason"?: string;
10579
10603
  };
10604
+ "stopped"?: {
10605
+ "at": string;
10606
+ "notStarted": number;
10607
+ };
10580
10608
  } | null;
10581
10609
  "pacing"?: {
10582
10610
  "maxInFlight"?: number;
@@ -10669,10 +10697,20 @@ export type StudyGradingRevisionsResponse = {
10669
10697
  "before": {
10670
10698
  "rubric": Array<string>;
10671
10699
  "expectations": Array<unknown>;
10700
+ "followUps"?: Array<{
10701
+ "prompt": string;
10702
+ "rubric": Array<string>;
10703
+ "expectations": Array<unknown>;
10704
+ }>;
10672
10705
  };
10673
10706
  "after": {
10674
10707
  "rubric": Array<string>;
10675
10708
  "expectations": Array<unknown>;
10709
+ "followUps"?: Array<{
10710
+ "prompt": string;
10711
+ "rubric": Array<string>;
10712
+ "expectations": Array<unknown>;
10713
+ }>;
10676
10714
  };
10677
10715
  "lenses": Array<"judge" | "evals">;
10678
10716
  }>;
@@ -10684,6 +10722,11 @@ export type StudyReviseGradingRequest = {
10684
10722
  "taskId": string;
10685
10723
  "rubric"?: Array<string>;
10686
10724
  "expectations"?: Array<unknown>;
10725
+ "followUps"?: Array<{
10726
+ "step": number;
10727
+ "rubric"?: Array<string>;
10728
+ "expectations"?: Array<unknown>;
10729
+ }>;
10687
10730
  }>;
10688
10731
  "dryRun"?: boolean;
10689
10732
  };
@@ -10702,10 +10745,20 @@ export type StudyReviseGradingResponse = {
10702
10745
  "before": {
10703
10746
  "rubric": Array<string>;
10704
10747
  "expectations": Array<unknown>;
10748
+ "followUps"?: Array<{
10749
+ "prompt": string;
10750
+ "rubric": Array<string>;
10751
+ "expectations": Array<unknown>;
10752
+ }>;
10705
10753
  };
10706
10754
  "after": {
10707
10755
  "rubric": Array<string>;
10708
10756
  "expectations": Array<unknown>;
10757
+ "followUps"?: Array<{
10758
+ "prompt": string;
10759
+ "rubric": Array<string>;
10760
+ "expectations": Array<unknown>;
10761
+ }>;
10709
10762
  };
10710
10763
  "lenses": Array<"judge" | "evals">;
10711
10764
  }>;
@@ -10847,6 +10900,10 @@ export type StudyLaunchResponse = {
10847
10900
  "nextAttemptAt"?: string;
10848
10901
  "reason"?: string;
10849
10902
  };
10903
+ "stopped"?: {
10904
+ "at": string;
10905
+ "notStarted": number;
10906
+ };
10850
10907
  } | null;
10851
10908
  "pacing"?: {
10852
10909
  "maxInFlight"?: number;
@@ -11058,6 +11115,10 @@ export type StudyUpdateLensesResponse = {
11058
11115
  "nextAttemptAt"?: string;
11059
11116
  "reason"?: string;
11060
11117
  };
11118
+ "stopped"?: {
11119
+ "at": string;
11120
+ "notStarted": number;
11121
+ };
11061
11122
  } | null;
11062
11123
  "pacing"?: {
11063
11124
  "maxInFlight"?: number;
@@ -12629,6 +12690,10 @@ export type StudyPublishResponse = {
12629
12690
  "nextAttemptAt"?: string;
12630
12691
  "reason"?: string;
12631
12692
  };
12693
+ "stopped"?: {
12694
+ "at": string;
12695
+ "notStarted": number;
12696
+ };
12632
12697
  } | null;
12633
12698
  "pacing"?: {
12634
12699
  "maxInFlight"?: number;
@@ -12879,6 +12944,10 @@ export type StudyRegradeResponse = {
12879
12944
  "nextAttemptAt"?: string;
12880
12945
  "reason"?: string;
12881
12946
  };
12947
+ "stopped"?: {
12948
+ "at": string;
12949
+ "notStarted": number;
12950
+ };
12882
12951
  } | null;
12883
12952
  "pacing"?: {
12884
12953
  "maxInFlight"?: number;
@@ -13172,6 +13241,10 @@ export type StudyRestoreResponse = {
13172
13241
  "nextAttemptAt"?: string;
13173
13242
  "reason"?: string;
13174
13243
  };
13244
+ "stopped"?: {
13245
+ "at": string;
13246
+ "notStarted": number;
13247
+ };
13175
13248
  } | null;
13176
13249
  "pacing"?: {
13177
13250
  "maxInFlight"?: number;
@@ -13614,6 +13687,10 @@ export type StudyRotateShareTokenResponse = {
13614
13687
  "nextAttemptAt"?: string;
13615
13688
  "reason"?: string;
13616
13689
  };
13690
+ "stopped"?: {
13691
+ "at": string;
13692
+ "notStarted": number;
13693
+ };
13617
13694
  } | null;
13618
13695
  "pacing"?: {
13619
13696
  "maxInFlight"?: number;
@@ -13687,6 +13764,216 @@ export type StudyRotateShareTokenResponse = {
13687
13764
  "shareUrl": string | null;
13688
13765
  "shareLinkKept": boolean;
13689
13766
  };
13767
+ export type StudyStopResponse = {
13768
+ "study": {
13769
+ "id": string;
13770
+ "executiveSummary"?: string | null;
13771
+ "executiveSummaryDraft"?: string | null;
13772
+ "executiveSummaryRequest"?: "live" | "draft" | null;
13773
+ "executiveSummaryRequestedAt"?: string | null;
13774
+ "executiveSummaryUnavailableAt"?: string | null;
13775
+ "plannedRunCount": number;
13776
+ "organizationId": string;
13777
+ "createdBy": string | null;
13778
+ "name": string;
13779
+ "slug": string;
13780
+ "description": string | null;
13781
+ "skillRef": string | null;
13782
+ "skillSlug": string | null;
13783
+ "skillName": string | null;
13784
+ "skillPromise": string;
13785
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
13786
+ "subjectName": string | null;
13787
+ "subjectLabel": string;
13788
+ "armConfig": {
13789
+ "baseline"?: {
13790
+ "tools"?: "default" | "none" | {
13791
+ "tools": Array<string>;
13792
+ };
13793
+ "addons"?: Record<string, unknown>;
13794
+ };
13795
+ "treatment"?: {
13796
+ "tools"?: "default" | "none" | {
13797
+ "tools": Array<string>;
13798
+ };
13799
+ "addons"?: Record<string, unknown>;
13800
+ };
13801
+ };
13802
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
13803
+ "subjectBackend": "model" | "harness" | null;
13804
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
13805
+ "importedFrom"?: string;
13806
+ "baselineLabel": string | null;
13807
+ "treatmentLabel": string | null;
13808
+ "promiseMetrics": Array<{
13809
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
13810
+ "direction": "down" | "up";
13811
+ "label": string;
13812
+ }>;
13813
+ "matrix": Array<{
13814
+ "harness": string;
13815
+ "model": string;
13816
+ "provider"?: string;
13817
+ "params"?: {
13818
+ "temperature"?: number;
13819
+ "maxOutputTokens"?: number;
13820
+ "reasoningEffort"?: "low" | "medium" | "high";
13821
+ "thinkingBudgetTokens"?: number;
13822
+ "codeMode"?: "auto" | "on" | "off";
13823
+ "toolSearch"?: "on" | "off" | "directory";
13824
+ };
13825
+ }>;
13826
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
13827
+ "judgeSupplementary"?: boolean;
13828
+ "taskCount": number;
13829
+ "variantsPerTask": number;
13830
+ "repeats": number;
13831
+ "subset": {
13832
+ "count": number;
13833
+ "seed": string;
13834
+ "of": number;
13835
+ } | null;
13836
+ "status": string;
13837
+ "error": string | null;
13838
+ "preparation"?: {
13839
+ "since": string;
13840
+ "progressAt"?: string;
13841
+ "launchedBy": {
13842
+ "userId"?: string;
13843
+ "role"?: string;
13844
+ };
13845
+ "waitingOn": Array<{
13846
+ "imageKey": string;
13847
+ "buildId": string;
13848
+ "stage"?: "image" | "snapshot";
13849
+ "retry"?: {
13850
+ "failedAttempts": number;
13851
+ "maxAttempts": number;
13852
+ "notBefore": string;
13853
+ "lastFailure": string;
13854
+ };
13855
+ }>;
13856
+ } | null;
13857
+ "visibility": "private" | "org" | "link" | "public";
13858
+ "visibilityChangedAt"?: string | null;
13859
+ "visibilityChangedBy"?: string | null;
13860
+ "shareToken"?: string | null;
13861
+ "lowSampleOverride": boolean;
13862
+ "publishedAt": string | null;
13863
+ "verdict": unknown;
13864
+ "definitionVersionId": string | null;
13865
+ "runnerKind": "ora-runs" | "batch-job";
13866
+ "createdAt": string;
13867
+ "updatedAt": string;
13868
+ "liveUntil": string | null;
13869
+ "launch"?: {
13870
+ "attempt": number;
13871
+ "state": "fanning_out" | "deferred" | "settled";
13872
+ "startedAt": string;
13873
+ "progressAt": string;
13874
+ "planned"?: number;
13875
+ "created"?: number;
13876
+ "missing"?: {
13877
+ "retryable": number;
13878
+ "refused": number;
13879
+ };
13880
+ "notes"?: Array<string>;
13881
+ "settledAt"?: string;
13882
+ "pacing"?: {
13883
+ "waiting": number;
13884
+ "nextUnit"?: number;
13885
+ "nextWaveAt"?: string;
13886
+ "emptyWaves"?: number;
13887
+ "stopped"?: string;
13888
+ };
13889
+ "recovery"?: {
13890
+ "state": "retrying" | "needs_attention";
13891
+ "attempts": number;
13892
+ "lastAttemptAt"?: string;
13893
+ "nextAttemptAt"?: string;
13894
+ "reason"?: string;
13895
+ };
13896
+ "stopped"?: {
13897
+ "at": string;
13898
+ "notStarted": number;
13899
+ };
13900
+ } | null;
13901
+ "pacing"?: {
13902
+ "maxInFlight"?: number;
13903
+ "startsPerMinute"?: number;
13904
+ } | null;
13905
+ "relaunchable"?: boolean;
13906
+ "archivedAt"?: string | null;
13907
+ "archivedBy"?: string | null;
13908
+ "arms"?: Array<{
13909
+ "key": string;
13910
+ "position": number;
13911
+ "label": string;
13912
+ "role": "reference" | "peer";
13913
+ "payload": {
13914
+ "tools"?: "default" | "none" | {
13915
+ "tools": Array<string>;
13916
+ };
13917
+ "addons"?: Record<string, unknown>;
13918
+ };
13919
+ "cells"?: Array<{
13920
+ "harness": string;
13921
+ "model": string;
13922
+ "provider"?: string;
13923
+ "params"?: {
13924
+ "temperature"?: number;
13925
+ "maxOutputTokens"?: number;
13926
+ "reasoningEffort"?: "low" | "medium" | "high";
13927
+ "thinkingBudgetTokens"?: number;
13928
+ "codeMode"?: "auto" | "on" | "off";
13929
+ "toolSearch"?: "on" | "off" | "directory";
13930
+ };
13931
+ }>;
13932
+ "skills"?: Array<{
13933
+ "ref": string;
13934
+ "name": string;
13935
+ "changedSinceCreate"?: boolean;
13936
+ }>;
13937
+ "addedAgents"?: Array<{
13938
+ "position": number;
13939
+ "addedAt": string;
13940
+ }>;
13941
+ }>;
13942
+ "contrastPlan"?: "vs_reference" | "all_pairs";
13943
+ "cellPlan"?: "crossed" | "per_setup";
13944
+ "skillDisclosure"?: "withheld" | "named";
13945
+ "modeOptions"?: Record<string, unknown>;
13946
+ "screens"?: Array<"test_awareness">;
13947
+ "runCounting"?: {
13948
+ "policy": "hold_back_screened";
13949
+ "screen": "test_awareness";
13950
+ "levels": Array<"aware" | "influenced">;
13951
+ } | null;
13952
+ "seriesKey"?: string | null;
13953
+ "ref"?: {
13954
+ "commit"?: string;
13955
+ "branch"?: string;
13956
+ "pr"?: number;
13957
+ "url"?: string;
13958
+ } | null;
13959
+ "gradingRevision"?: number;
13960
+ "gradingRevisedAt"?: string | null;
13961
+ "rehearsal"?: true;
13962
+ "rehearsalMode"?: string;
13963
+ "behaviors"?: Array<{
13964
+ "key": string;
13965
+ "name": string;
13966
+ "question": string;
13967
+ "version": number;
13968
+ }>;
13969
+ };
13970
+ "stop": {
13971
+ "alreadyStopped": boolean;
13972
+ "stoppedAt": string | null;
13973
+ "cancelled": number;
13974
+ "notStarted": number;
13975
+ };
13976
+ };
13690
13977
  export type StudyStreamResponse = unknown;
13691
13978
  export type StudyUpdateTaskRequest = {
13692
13979
  "prompt": string;
@@ -13908,6 +14195,10 @@ export type StudyUnpublishResponse = {
13908
14195
  "nextAttemptAt"?: string;
13909
14196
  "reason"?: string;
13910
14197
  };
14198
+ "stopped"?: {
14199
+ "at": string;
14200
+ "notStarted": number;
14201
+ };
13911
14202
  } | null;
13912
14203
  "pacing"?: {
13913
14204
  "maxInFlight"?: number;
@@ -14173,6 +14464,10 @@ export type StudySetVisibilityResponse = {
14173
14464
  "nextAttemptAt"?: string;
14174
14465
  "reason"?: string;
14175
14466
  };
14467
+ "stopped"?: {
14468
+ "at": string;
14469
+ "notStarted": number;
14470
+ };
14176
14471
  } | null;
14177
14472
  "pacing"?: {
14178
14473
  "maxInFlight"?: number;