@ora-ai/cli 0.21.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/README.md +96 -1
  2. package/dist/auth-commands.d.ts.map +1 -1
  3. package/dist/auth-commands.js +12 -1
  4. package/dist/auth-commands.js.map +1 -1
  5. package/dist/core/auth.d.ts +24 -13
  6. package/dist/core/auth.d.ts.map +1 -1
  7. package/dist/core/auth.js +26 -15
  8. package/dist/core/auth.js.map +1 -1
  9. package/dist/core/credential-sources.d.ts +45 -0
  10. package/dist/core/credential-sources.d.ts.map +1 -0
  11. package/dist/core/credential-sources.js +65 -0
  12. package/dist/core/credential-sources.js.map +1 -0
  13. package/dist/core/github-actions-sign-in.d.ts +33 -0
  14. package/dist/core/github-actions-sign-in.d.ts.map +1 -0
  15. package/dist/core/github-actions-sign-in.js +54 -0
  16. package/dist/core/github-actions-sign-in.js.map +1 -0
  17. package/dist/core/output.js +14 -1
  18. package/dist/core/output.js.map +1 -1
  19. package/dist/core/run-operation.d.ts +11 -4
  20. package/dist/core/run-operation.d.ts.map +1 -1
  21. package/dist/core/run-operation.js +14 -9
  22. package/dist/core/run-operation.js.map +1 -1
  23. package/dist/core/sign-in.d.ts +34 -0
  24. package/dist/core/sign-in.d.ts.map +1 -0
  25. package/dist/core/sign-in.js +21 -0
  26. package/dist/core/sign-in.js.map +1 -0
  27. package/dist/generated/operations.d.ts +1 -1
  28. package/dist/generated/operations.d.ts.map +1 -1
  29. package/dist/generated/operations.js +1274 -73
  30. package/dist/generated/operations.js.map +1 -1
  31. package/dist/generated/types.d.ts +1445 -10
  32. package/dist/generated/types.d.ts.map +1 -1
  33. package/dist/generated/types.js +1 -1
  34. package/dist/mcp/mcp-call.d.ts +25 -0
  35. package/dist/mcp/mcp-call.d.ts.map +1 -0
  36. package/dist/mcp/mcp-call.js +44 -0
  37. package/dist/mcp/mcp-call.js.map +1 -0
  38. package/dist/mcp/mcp-command.d.ts +26 -0
  39. package/dist/mcp/mcp-command.d.ts.map +1 -0
  40. package/dist/mcp/mcp-command.js +91 -0
  41. package/dist/mcp/mcp-command.js.map +1 -0
  42. package/dist/mcp/mcp-install.d.ts +50 -0
  43. package/dist/mcp/mcp-install.d.ts.map +1 -0
  44. package/dist/mcp/mcp-install.js +71 -0
  45. package/dist/mcp/mcp-install.js.map +1 -0
  46. package/dist/mcp/mcp-protocol.d.ts +69 -0
  47. package/dist/mcp/mcp-protocol.d.ts.map +1 -0
  48. package/dist/mcp/mcp-protocol.js +96 -0
  49. package/dist/mcp/mcp-protocol.js.map +1 -0
  50. package/dist/mcp/mcp-stdio.d.ts +17 -0
  51. package/dist/mcp/mcp-stdio.d.ts.map +1 -0
  52. package/dist/mcp/mcp-stdio.js +39 -0
  53. package/dist/mcp/mcp-stdio.js.map +1 -0
  54. package/dist/mcp/mcp-tools.d.ts +81 -0
  55. package/dist/mcp/mcp-tools.d.ts.map +1 -0
  56. package/dist/mcp/mcp-tools.js +134 -0
  57. package/dist/mcp/mcp-tools.js.map +1 -0
  58. package/dist/program.d.ts +4 -9
  59. package/dist/program.d.ts.map +1 -1
  60. package/dist/program.js +9 -3
  61. package/dist/program.js.map +1 -1
  62. package/dist/render/index.js +6 -0
  63. package/dist/render/index.js.map +1 -1
  64. package/dist/render/study/counting-views.d.ts +46 -0
  65. package/dist/render/study/counting-views.d.ts.map +1 -0
  66. package/dist/render/study/counting-views.js +49 -0
  67. package/dist/render/study/counting-views.js.map +1 -0
  68. package/dist/render/study/gate-views.d.ts +56 -0
  69. package/dist/render/study/gate-views.d.ts.map +1 -0
  70. package/dist/render/study/gate-views.js +64 -0
  71. package/dist/render/study/gate-views.js.map +1 -0
  72. package/dist/render/study/renderers.d.ts +8 -0
  73. package/dist/render/study/renderers.d.ts.map +1 -0
  74. package/dist/render/study/renderers.js +31 -0
  75. package/dist/render/study/renderers.js.map +1 -0
  76. package/dist/render/study/series-views.d.ts +65 -0
  77. package/dist/render/study/series-views.d.ts.map +1 -0
  78. package/dist/render/study/series-views.js +116 -0
  79. package/dist/render/study/series-views.js.map +1 -0
  80. package/dist/render/study/study-format.d.ts +24 -0
  81. package/dist/render/study/study-format.d.ts.map +1 -0
  82. package/dist/render/study/study-format.js +42 -0
  83. package/dist/render/study/study-format.js.map +1 -0
  84. package/dist/study-ci/ci-command.d.ts +40 -0
  85. package/dist/study-ci/ci-command.d.ts.map +1 -0
  86. package/dist/study-ci/ci-command.js +151 -0
  87. package/dist/study-ci/ci-command.js.map +1 -0
  88. package/dist/study-ci/ci-rehearse-command.d.ts +11 -0
  89. package/dist/study-ci/ci-rehearse-command.d.ts.map +1 -0
  90. package/dist/study-ci/ci-rehearse-command.js +83 -0
  91. package/dist/study-ci/ci-rehearse-command.js.map +1 -0
  92. package/dist/study-ci/ci-rehearse.d.ts +62 -0
  93. package/dist/study-ci/ci-rehearse.d.ts.map +1 -0
  94. package/dist/study-ci/ci-rehearse.js +49 -0
  95. package/dist/study-ci/ci-rehearse.js.map +1 -0
  96. package/dist/study-ci/ci-run.d.ts +87 -0
  97. package/dist/study-ci/ci-run.d.ts.map +1 -0
  98. package/dist/study-ci/ci-run.js +131 -0
  99. package/dist/study-ci/ci-run.js.map +1 -0
  100. package/dist/study-ci/gate-report.d.ts +64 -0
  101. package/dist/study-ci/gate-report.d.ts.map +1 -0
  102. package/dist/study-ci/gate-report.js +80 -0
  103. package/dist/study-ci/gate-report.js.map +1 -0
  104. package/dist/study-ci/params.d.ts +8 -0
  105. package/dist/study-ci/params.d.ts.map +1 -0
  106. package/dist/study-ci/params.js +72 -0
  107. package/dist/study-ci/params.js.map +1 -0
  108. package/package.json +1 -1
@@ -7174,6 +7174,8 @@ export type StudyListResponse = {
7174
7174
  "startsPerMinute"?: number;
7175
7175
  } | null;
7176
7176
  "relaunchable"?: boolean;
7177
+ "archivedAt"?: string | null;
7178
+ "archivedBy"?: string | null;
7177
7179
  "arms"?: Array<{
7178
7180
  "key": string;
7179
7181
  "position": number;
@@ -7212,6 +7214,29 @@ export type StudyListResponse = {
7212
7214
  "cellPlan"?: "crossed" | "per_setup";
7213
7215
  "skillDisclosure"?: "withheld" | "named";
7214
7216
  "modeOptions"?: Record<string, unknown>;
7217
+ "screens"?: Array<"test_awareness">;
7218
+ "runCounting"?: {
7219
+ "policy": "hold_back_screened";
7220
+ "screen": "test_awareness";
7221
+ "levels": Array<"aware" | "influenced">;
7222
+ } | null;
7223
+ "seriesKey"?: string | null;
7224
+ "ref"?: {
7225
+ "commit"?: string;
7226
+ "branch"?: string;
7227
+ "pr"?: number;
7228
+ "url"?: string;
7229
+ } | null;
7230
+ "gradingRevision"?: number;
7231
+ "gradingRevisedAt"?: string | null;
7232
+ "rehearsal"?: true;
7233
+ "rehearsalMode"?: string;
7234
+ "behaviors"?: Array<{
7235
+ "key": string;
7236
+ "name": string;
7237
+ "question": string;
7238
+ "version": number;
7239
+ }>;
7215
7240
  }>;
7216
7241
  };
7217
7242
  export type StudyCreateRequest = {
@@ -7357,6 +7382,28 @@ export type StudyCreateRequest = {
7357
7382
  "maxInFlight"?: number;
7358
7383
  "startsPerMinute"?: number;
7359
7384
  } | null;
7385
+ "screens"?: Array<"test_awareness">;
7386
+ "series"?: string;
7387
+ "ref"?: {
7388
+ "commit"?: string;
7389
+ "branch"?: string;
7390
+ "pr"?: number;
7391
+ "url"?: string;
7392
+ };
7393
+ "carry"?: {
7394
+ "from": "series";
7395
+ "maxAgeDays"?: number;
7396
+ };
7397
+ "runCounting"?: {
7398
+ "policy": "hold_back_screened";
7399
+ "screen": "test_awareness";
7400
+ "levels": Array<"aware" | "influenced">;
7401
+ } | null;
7402
+ "behaviors"?: Array<{
7403
+ "key": string;
7404
+ "name": string;
7405
+ "question": string;
7406
+ }>;
7360
7407
  "matrix": Array<{
7361
7408
  "harness": string;
7362
7409
  "model": string;
@@ -7507,6 +7554,8 @@ export type StudyCreateResponse = {
7507
7554
  "startsPerMinute"?: number;
7508
7555
  } | null;
7509
7556
  "relaunchable"?: boolean;
7557
+ "archivedAt"?: string | null;
7558
+ "archivedBy"?: string | null;
7510
7559
  "arms"?: Array<{
7511
7560
  "key": string;
7512
7561
  "position": number;
@@ -7545,6 +7594,29 @@ export type StudyCreateResponse = {
7545
7594
  "cellPlan"?: "crossed" | "per_setup";
7546
7595
  "skillDisclosure"?: "withheld" | "named";
7547
7596
  "modeOptions"?: Record<string, unknown>;
7597
+ "screens"?: Array<"test_awareness">;
7598
+ "runCounting"?: {
7599
+ "policy": "hold_back_screened";
7600
+ "screen": "test_awareness";
7601
+ "levels": Array<"aware" | "influenced">;
7602
+ } | null;
7603
+ "seriesKey"?: string | null;
7604
+ "ref"?: {
7605
+ "commit"?: string;
7606
+ "branch"?: string;
7607
+ "pr"?: number;
7608
+ "url"?: string;
7609
+ } | null;
7610
+ "gradingRevision"?: number;
7611
+ "gradingRevisedAt"?: string | null;
7612
+ "rehearsal"?: true;
7613
+ "rehearsalMode"?: string;
7614
+ "behaviors"?: Array<{
7615
+ "key": string;
7616
+ "name": string;
7617
+ "question": string;
7618
+ "version": number;
7619
+ }>;
7548
7620
  };
7549
7621
  "streamUrl": string;
7550
7622
  "warnings"?: Array<{
@@ -7565,6 +7637,10 @@ export type StudyComparisonModesListResponse = {
7565
7637
  "id": "crossed" | "per_setup";
7566
7638
  "enabled": boolean;
7567
7639
  }>;
7640
+ "customBehaviors": {
7641
+ "enabled": boolean;
7642
+ "max": number;
7643
+ };
7568
7644
  };
7569
7645
  export type StudyConvertWebmcpSuiteRequest = {
7570
7646
  "suite": unknown;
@@ -7721,6 +7797,28 @@ export type StudyPreflightRequest = {
7721
7797
  "maxInFlight"?: number;
7722
7798
  "startsPerMinute"?: number;
7723
7799
  } | null;
7800
+ "screens"?: Array<"test_awareness">;
7801
+ "series"?: string;
7802
+ "ref"?: {
7803
+ "commit"?: string;
7804
+ "branch"?: string;
7805
+ "pr"?: number;
7806
+ "url"?: string;
7807
+ };
7808
+ "carry"?: {
7809
+ "from": "series";
7810
+ "maxAgeDays"?: number;
7811
+ };
7812
+ "runCounting"?: {
7813
+ "policy": "hold_back_screened";
7814
+ "screen": "test_awareness";
7815
+ "levels": Array<"aware" | "influenced">;
7816
+ } | null;
7817
+ "behaviors"?: Array<{
7818
+ "key": string;
7819
+ "name": string;
7820
+ "question": string;
7821
+ }>;
7724
7822
  "matrix": Array<{
7725
7823
  "harness": string;
7726
7824
  "model": string;
@@ -7743,6 +7841,14 @@ export type StudyPreflightResponse = {
7743
7841
  "label": string;
7744
7842
  "plannedRunCount": number;
7745
7843
  "plannedJudgeComparisons": number;
7844
+ "screens"?: {
7845
+ "ids": Array<string>;
7846
+ "plannedReads": number;
7847
+ };
7848
+ "behaviors"?: {
7849
+ "keys": Array<string>;
7850
+ "plannedReads": number;
7851
+ };
7746
7852
  "runs": {
7747
7853
  "tasks": number;
7748
7854
  "variants": number;
@@ -7813,6 +7919,248 @@ export type StudyPreflightResponse = {
7813
7919
  "allowed": boolean;
7814
7920
  } | null;
7815
7921
  };
7922
+ export type StudyRehearseRequest = {
7923
+ "name": string;
7924
+ "description"?: string;
7925
+ "subjectKind"?: "skill" | "tools" | "addon" | "custom" | "benchmark";
7926
+ "skillRef"?: string;
7927
+ "definitionVersionId"?: string;
7928
+ "subjectName"?: string;
7929
+ "armConfig"?: {
7930
+ "baseline"?: {
7931
+ "tools"?: "default" | "none" | {
7932
+ "tools": Array<string>;
7933
+ };
7934
+ "addons"?: Record<string, unknown>;
7935
+ };
7936
+ "treatment"?: {
7937
+ "tools"?: "default" | "none" | {
7938
+ "tools": Array<string>;
7939
+ };
7940
+ "addons"?: Record<string, unknown>;
7941
+ };
7942
+ };
7943
+ "lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
7944
+ "judgeSupplementary"?: boolean;
7945
+ "proposeExpectations"?: boolean;
7946
+ "comparisonMode"?: "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
7947
+ "subjectBackend"?: "model" | "harness";
7948
+ "baselineLabel"?: string;
7949
+ "treatmentLabel"?: string;
7950
+ "arms"?: Array<{
7951
+ "key"?: string;
7952
+ "label": string;
7953
+ "role"?: "reference" | "peer";
7954
+ "config": {
7955
+ "tools"?: "default" | "none" | {
7956
+ "tools": Array<string>;
7957
+ };
7958
+ "addons"?: Record<string, unknown>;
7959
+ };
7960
+ "cells"?: Array<{
7961
+ "harness": string;
7962
+ "model": string;
7963
+ "provider"?: string;
7964
+ "params"?: {
7965
+ "temperature"?: number;
7966
+ "maxOutputTokens"?: number;
7967
+ "reasoningEffort"?: "low" | "medium" | "high";
7968
+ "thinkingBudgetTokens"?: number;
7969
+ "codeMode"?: "auto" | "on" | "off";
7970
+ "toolSearch"?: "on" | "off" | "directory";
7971
+ };
7972
+ }>;
7973
+ }>;
7974
+ "contrasts"?: "vs_reference" | "all_pairs";
7975
+ "modeOptions"?: Record<string, unknown>;
7976
+ "taskCount"?: number;
7977
+ "variantsPerTask"?: number;
7978
+ "repeats"?: number;
7979
+ "tasks"?: Array<{
7980
+ "prompt": string;
7981
+ "rationale"?: string;
7982
+ "category"?: string;
7983
+ "requiresWeb"?: boolean;
7984
+ "rubric"?: Array<string>;
7985
+ "variants"?: Array<string>;
7986
+ "expectations"?: Array<{
7987
+ "label"?: string;
7988
+ "arms"?: Array<"baseline" | "skill" | string>;
7989
+ "kind": "mcp_tool_called";
7990
+ "server"?: string;
7991
+ "tool": string;
7992
+ "arguments"?: Record<string, unknown>;
7993
+ "minCalls"?: number;
7994
+ } | {
7995
+ "label"?: string;
7996
+ "arms"?: Array<"baseline" | "skill" | string>;
7997
+ "kind": "tool_not_called";
7998
+ "tool"?: string;
7999
+ "mcp"?: {
8000
+ "server"?: string;
8001
+ "tool": string;
8002
+ };
8003
+ "pattern"?: string;
8004
+ } | {
8005
+ "label"?: string;
8006
+ "arms"?: Array<"baseline" | "skill" | string>;
8007
+ "kind": "command_ran";
8008
+ "pattern": string;
8009
+ } | {
8010
+ "label"?: string;
8011
+ "arms"?: Array<"baseline" | "skill" | string>;
8012
+ "kind": "skill_loaded";
8013
+ "slug": string;
8014
+ } | {
8015
+ "label"?: string;
8016
+ "arms"?: Array<"baseline" | "skill" | string>;
8017
+ "kind": "file_operation";
8018
+ "op"?: "read" | "write" | "edit";
8019
+ "path": unknown;
8020
+ } | {
8021
+ "label"?: string;
8022
+ "arms"?: Array<"baseline" | "skill" | string>;
8023
+ "kind": "web_fetched";
8024
+ "urlHost"?: string;
8025
+ "urlPattern"?: string;
8026
+ } | {
8027
+ "label"?: string;
8028
+ "arms"?: Array<"baseline" | "skill" | string>;
8029
+ "kind": "response_contains";
8030
+ "text": string;
8031
+ "normalize"?: "prose";
8032
+ } | {
8033
+ "label"?: string;
8034
+ "arms"?: Array<"baseline" | "skill" | string>;
8035
+ "kind": "response_matches";
8036
+ "pattern": string;
8037
+ "ignoreCase"?: boolean;
8038
+ } | {
8039
+ "label"?: string;
8040
+ "arms"?: Array<"baseline" | "skill" | string>;
8041
+ "kind": "token_budget";
8042
+ "maxOutputTokens"?: number;
8043
+ "maxTotalTokens"?: number;
8044
+ } | {
8045
+ "label"?: string;
8046
+ "arms"?: Array<"baseline" | "skill" | string>;
8047
+ "kind": "sequence";
8048
+ "of": Array<unknown>;
8049
+ } | {
8050
+ "label"?: string;
8051
+ "arms"?: Array<"baseline" | "skill" | string>;
8052
+ "kind": "any_of";
8053
+ "of": Array<unknown>;
8054
+ }>;
8055
+ "private"?: boolean;
8056
+ }>;
8057
+ "subset"?: {
8058
+ "count": number;
8059
+ "seed"?: string;
8060
+ };
8061
+ "pacing"?: {
8062
+ "maxInFlight"?: number;
8063
+ "startsPerMinute"?: number;
8064
+ } | null;
8065
+ "screens"?: Array<"test_awareness">;
8066
+ "series"?: string;
8067
+ "ref"?: {
8068
+ "commit"?: string;
8069
+ "branch"?: string;
8070
+ "pr"?: number;
8071
+ "url"?: string;
8072
+ };
8073
+ "carry"?: {
8074
+ "from": "series";
8075
+ "maxAgeDays"?: number;
8076
+ };
8077
+ "runCounting"?: {
8078
+ "policy": "hold_back_screened";
8079
+ "screen": "test_awareness";
8080
+ "levels": Array<"aware" | "influenced">;
8081
+ } | null;
8082
+ "behaviors"?: Array<{
8083
+ "key": string;
8084
+ "name": string;
8085
+ "question": string;
8086
+ }>;
8087
+ "matrix": Array<{
8088
+ "harness": string;
8089
+ "model": string;
8090
+ "provider"?: string;
8091
+ "params"?: {
8092
+ "temperature"?: number;
8093
+ "maxOutputTokens"?: number;
8094
+ "reasoningEffort"?: "low" | "medium" | "high";
8095
+ "thinkingBudgetTokens"?: number;
8096
+ "codeMode"?: "auto" | "on" | "off";
8097
+ "toolSearch"?: "on" | "off" | "directory";
8098
+ };
8099
+ }>;
8100
+ "promise"?: string;
8101
+ };
8102
+ export type StudyRehearseResponse = {
8103
+ "rehearsal": {
8104
+ "studyId": string;
8105
+ "name": string;
8106
+ "status": string;
8107
+ "tasks": number;
8108
+ "mode": string;
8109
+ };
8110
+ };
8111
+ export type StudySeriesResponse = {
8112
+ "series": Array<{
8113
+ "key": string;
8114
+ "studies": number;
8115
+ "newestAt": string;
8116
+ }>;
8117
+ "capped": boolean;
8118
+ };
8119
+ export type StudySeriesTrendResponse = {
8120
+ "series": string;
8121
+ "metric": {
8122
+ "key": string;
8123
+ "label": string;
8124
+ "family": string;
8125
+ };
8126
+ "countingPolicy": string;
8127
+ "studies": Array<{
8128
+ "studyId": string;
8129
+ "name": string;
8130
+ "createdAt": string;
8131
+ "ref": {
8132
+ "commit"?: string;
8133
+ "branch"?: string;
8134
+ "pr"?: number;
8135
+ "url"?: string;
8136
+ } | null;
8137
+ "state": "settled" | "pending";
8138
+ "owed": Array<string>;
8139
+ "segment": number | null;
8140
+ "heldBackRuns"?: number;
8141
+ "groups": Array<{
8142
+ "key": string;
8143
+ "label": string;
8144
+ "value": number | null;
8145
+ "interval": Array<unknown> | null;
8146
+ "tasks": number;
8147
+ "runs": number;
8148
+ "change": {
8149
+ "effect": number;
8150
+ "unit": string;
8151
+ "interval": Array<unknown>;
8152
+ "tasks": number;
8153
+ } | null;
8154
+ }>;
8155
+ }>;
8156
+ "segments": Array<{
8157
+ "index": number;
8158
+ "comparabilityKey": string;
8159
+ "studyIds": Array<string>;
8160
+ "differsFromPrevious": Array<"corpus" | "lenses" | "graders" | "classifiers" | "countingPolicy" | "agents">;
8161
+ }>;
8162
+ "truncated": boolean;
8163
+ };
7816
8164
  export type StudyGetResponse = {
7817
8165
  "study": {
7818
8166
  "id": string;
@@ -7948,6 +8296,8 @@ export type StudyGetResponse = {
7948
8296
  "startsPerMinute"?: number;
7949
8297
  } | null;
7950
8298
  "relaunchable"?: boolean;
8299
+ "archivedAt"?: string | null;
8300
+ "archivedBy"?: string | null;
7951
8301
  "arms"?: Array<{
7952
8302
  "key": string;
7953
8303
  "position": number;
@@ -7986,6 +8336,29 @@ export type StudyGetResponse = {
7986
8336
  "cellPlan"?: "crossed" | "per_setup";
7987
8337
  "skillDisclosure"?: "withheld" | "named";
7988
8338
  "modeOptions"?: Record<string, unknown>;
8339
+ "screens"?: Array<"test_awareness">;
8340
+ "runCounting"?: {
8341
+ "policy": "hold_back_screened";
8342
+ "screen": "test_awareness";
8343
+ "levels": Array<"aware" | "influenced">;
8344
+ } | null;
8345
+ "seriesKey"?: string | null;
8346
+ "ref"?: {
8347
+ "commit"?: string;
8348
+ "branch"?: string;
8349
+ "pr"?: number;
8350
+ "url"?: string;
8351
+ } | null;
8352
+ "gradingRevision"?: number;
8353
+ "gradingRevisedAt"?: string | null;
8354
+ "rehearsal"?: true;
8355
+ "rehearsalMode"?: string;
8356
+ "behaviors"?: Array<{
8357
+ "key": string;
8358
+ "name": string;
8359
+ "question": string;
8360
+ "version": number;
8361
+ }>;
7989
8362
  };
7990
8363
  "tasks": Array<{
7991
8364
  "id": string;
@@ -8134,6 +8507,12 @@ export type StudyGetResponse = {
8134
8507
  }>;
8135
8508
  "modelCalls": number | null;
8136
8509
  } | null;
8510
+ "screens"?: Record<string, unknown>;
8511
+ "carried"?: {
8512
+ "fromRunId": string;
8513
+ "fromStudyId": string | null;
8514
+ "at": string;
8515
+ } | null;
8137
8516
  }>;
8138
8517
  "aggregate": {
8139
8518
  "summary": {
@@ -8397,6 +8776,18 @@ export type StudyGetResponse = {
8397
8776
  "byArm": Record<string, unknown>;
8398
8777
  "phasesByArm": Record<string, unknown>;
8399
8778
  };
8779
+ "runCounting"?: {
8780
+ "policy": string;
8781
+ "screen": string;
8782
+ "levels": Array<string>;
8783
+ "heldBack": number;
8784
+ "groups": Array<{
8785
+ "group": string;
8786
+ "heldBack": number;
8787
+ "runs": number;
8788
+ }>;
8789
+ "imbalanced": boolean;
8790
+ };
8400
8791
  } | {
8401
8792
  "shape": "multi-arm";
8402
8793
  "summary": {
@@ -9272,6 +9663,18 @@ export type StudyGetResponse = {
9272
9663
  "byArm": Record<string, unknown>;
9273
9664
  "phasesByArm": Record<string, unknown>;
9274
9665
  };
9666
+ "runCounting"?: {
9667
+ "policy": string;
9668
+ "screen": string;
9669
+ "levels": Array<string>;
9670
+ "heldBack": number;
9671
+ "groups": Array<{
9672
+ "group": string;
9673
+ "heldBack": number;
9674
+ "runs": number;
9675
+ }>;
9676
+ "imbalanced": boolean;
9677
+ };
9275
9678
  };
9276
9679
  "warnings": Array<{
9277
9680
  "type": "eval_capability_gap";
@@ -9281,6 +9684,11 @@ export type StudyGetResponse = {
9281
9684
  "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
9282
9685
  "detail": string;
9283
9686
  "taskIds"?: Array<string>;
9687
+ } | {
9688
+ "type": "design_lint";
9689
+ "lint": string;
9690
+ "detail": string;
9691
+ "taskIds": Array<string>;
9284
9692
  } | {
9285
9693
  "type": "design_aggregate_budget";
9286
9694
  "detail": string;
@@ -9447,6 +9855,8 @@ export type StudyUpdateDetailsResponse = {
9447
9855
  "startsPerMinute"?: number;
9448
9856
  } | null;
9449
9857
  "relaunchable"?: boolean;
9858
+ "archivedAt"?: string | null;
9859
+ "archivedBy"?: string | null;
9450
9860
  "arms"?: Array<{
9451
9861
  "key": string;
9452
9862
  "position": number;
@@ -9485,6 +9895,29 @@ export type StudyUpdateDetailsResponse = {
9485
9895
  "cellPlan"?: "crossed" | "per_setup";
9486
9896
  "skillDisclosure"?: "withheld" | "named";
9487
9897
  "modeOptions"?: Record<string, unknown>;
9898
+ "screens"?: Array<"test_awareness">;
9899
+ "runCounting"?: {
9900
+ "policy": "hold_back_screened";
9901
+ "screen": "test_awareness";
9902
+ "levels": Array<"aware" | "influenced">;
9903
+ } | null;
9904
+ "seriesKey"?: string | null;
9905
+ "ref"?: {
9906
+ "commit"?: string;
9907
+ "branch"?: string;
9908
+ "pr"?: number;
9909
+ "url"?: string;
9910
+ } | null;
9911
+ "gradingRevision"?: number;
9912
+ "gradingRevisedAt"?: string | null;
9913
+ "rehearsal"?: true;
9914
+ "rehearsalMode"?: string;
9915
+ "behaviors"?: Array<{
9916
+ "key": string;
9917
+ "name": string;
9918
+ "question": string;
9919
+ "version": number;
9920
+ }>;
9488
9921
  };
9489
9922
  };
9490
9923
  export type StudyDeleteResponse = unknown;
@@ -9594,16 +10027,175 @@ export type StudyAddAgentsResponse = {
9594
10027
  "reason": string | null;
9595
10028
  };
9596
10029
  };
9597
- export type StudyRegenerateSummaryResponse = {
9598
- "status": "queued";
9599
- "target": "live" | "draft";
9600
- };
9601
- export type StudyGenerateRequest = {
9602
- "proposeExpectations"?: boolean;
9603
- };
9604
- export type StudyGenerateResponse = {
9605
- "study": {
9606
- "id": string;
10030
+ export type StudyBehaviorsResponse = {
10031
+ "behaviors": Array<{
10032
+ "key": string;
10033
+ "name": string;
10034
+ "question": string;
10035
+ "version": number;
10036
+ "screenable": number;
10037
+ "judged": number;
10038
+ "measurement": {
10039
+ "key": string;
10040
+ "version": number;
10041
+ "verdicts": {
10042
+ "present": number;
10043
+ "absent": number;
10044
+ "unknown": number;
10045
+ };
10046
+ "present": {
10047
+ "reviewed": number;
10048
+ "confirmed": number;
10049
+ "precision": {
10050
+ "rate": number;
10051
+ "lower": number;
10052
+ "upper": number;
10053
+ "successes": number;
10054
+ "n": number;
10055
+ } | null;
10056
+ };
10057
+ "absent": {
10058
+ "reviewed": number;
10059
+ "confirmed": number;
10060
+ "precision": {
10061
+ "rate": number;
10062
+ "lower": number;
10063
+ "upper": number;
10064
+ "successes": number;
10065
+ "n": number;
10066
+ } | null;
10067
+ };
10068
+ "contested": number;
10069
+ "agreement": {
10070
+ "runs": number;
10071
+ "agreed": number;
10072
+ "kappa": number | null;
10073
+ } | null;
10074
+ };
10075
+ }>;
10076
+ "reviews": number;
10077
+ "reviewsTruncated": boolean;
10078
+ };
10079
+ export type StudySetCarryRequest = {
10080
+ "carry": {
10081
+ "from": "series";
10082
+ "maxAgeDays"?: number;
10083
+ } | null;
10084
+ };
10085
+ export type StudySetCarryResponse = {
10086
+ "policy": {
10087
+ "from": "series";
10088
+ "maxAgeDays": number;
10089
+ } | null;
10090
+ "sourceStudyId": string | null;
10091
+ "planned": number;
10092
+ "carried": number;
10093
+ "fresh": Record<string, unknown>;
10094
+ "differences": Array<string>;
10095
+ "carriedRuns": number;
10096
+ "changed": boolean;
10097
+ };
10098
+ export type StudyCarryPlanResponse = {
10099
+ "policy": {
10100
+ "from": "series";
10101
+ "maxAgeDays": number;
10102
+ } | null;
10103
+ "sourceStudyId": string | null;
10104
+ "planned": number;
10105
+ "carried": number;
10106
+ "fresh": Record<string, unknown>;
10107
+ "differences": Array<string>;
10108
+ "carriedRuns": number;
10109
+ };
10110
+ export type StudyRegenerateSummaryResponse = {
10111
+ "status": "queued";
10112
+ "target": "live" | "draft";
10113
+ };
10114
+ export type StudyGateRequest = {
10115
+ "metric"?: "benchmark" | "goal" | "evals" | "quality" | "rubric" | "completion";
10116
+ "group"?: string;
10117
+ "floor"?: {
10118
+ "rate": number;
10119
+ } | {
10120
+ "points": number;
10121
+ };
10122
+ "maxDrop"?: {
10123
+ "pp": number;
10124
+ } | {
10125
+ "points": number;
10126
+ };
10127
+ "baseline"?: {
10128
+ "studyId": string;
10129
+ } | {
10130
+ "series": {
10131
+ "branch"?: string;
10132
+ };
10133
+ };
10134
+ "minTasks"?: number;
10135
+ "infraCeiling"?: number;
10136
+ "counting"?: "policy" | "all";
10137
+ };
10138
+ export type StudyGateResponse = {
10139
+ "state": "settled" | "pending";
10140
+ "owed": Array<string>;
10141
+ "countingPolicy": string;
10142
+ "metric": {
10143
+ "key": string;
10144
+ "label": string;
10145
+ "family": "rate" | "score";
10146
+ };
10147
+ "group": string;
10148
+ "groups": Array<{
10149
+ "key": string;
10150
+ "label": string;
10151
+ "value": number | null;
10152
+ "runs": number;
10153
+ "tasks": number;
10154
+ "interval": Array<unknown> | null;
10155
+ "carriedRuns"?: number;
10156
+ "heldBackRuns"?: number;
10157
+ }>;
10158
+ "baseline": {
10159
+ "studyId": string;
10160
+ "name": string;
10161
+ "comparable": boolean;
10162
+ "differs": Array<string>;
10163
+ "value": number | null;
10164
+ "interval": Array<unknown> | null;
10165
+ "difference": {
10166
+ "effect": number;
10167
+ "unit": "pp" | "pts";
10168
+ "interval": Array<unknown>;
10169
+ "tasks": number;
10170
+ } | null;
10171
+ } | null;
10172
+ "comparabilityKey": string;
10173
+ "series": {
10174
+ "key": string;
10175
+ "ref": {
10176
+ "commit"?: string;
10177
+ "branch"?: string;
10178
+ "pr"?: number;
10179
+ "url"?: string;
10180
+ } | null;
10181
+ } | null;
10182
+ "gate": {
10183
+ "decision": "pass" | "fail" | "undecided";
10184
+ "refusedBy": string | null;
10185
+ "reason": string;
10186
+ "rules": Array<{
10187
+ "rule": string;
10188
+ "decision": "pass" | "fail" | "undecided";
10189
+ "reason": string;
10190
+ }>;
10191
+ };
10192
+ };
10193
+ export type StudyGenerateRequest = {
10194
+ "proposeExpectations"?: boolean;
10195
+ };
10196
+ export type StudyGenerateResponse = {
10197
+ "study": {
10198
+ "id": string;
9607
10199
  "executiveSummary"?: string | null;
9608
10200
  "executiveSummaryDraft"?: string | null;
9609
10201
  "executiveSummaryRequest"?: "live" | "draft" | null;
@@ -9736,6 +10328,8 @@ export type StudyGenerateResponse = {
9736
10328
  "startsPerMinute"?: number;
9737
10329
  } | null;
9738
10330
  "relaunchable"?: boolean;
10331
+ "archivedAt"?: string | null;
10332
+ "archivedBy"?: string | null;
9739
10333
  "arms"?: Array<{
9740
10334
  "key": string;
9741
10335
  "position": number;
@@ -9774,6 +10368,29 @@ export type StudyGenerateResponse = {
9774
10368
  "cellPlan"?: "crossed" | "per_setup";
9775
10369
  "skillDisclosure"?: "withheld" | "named";
9776
10370
  "modeOptions"?: Record<string, unknown>;
10371
+ "screens"?: Array<"test_awareness">;
10372
+ "runCounting"?: {
10373
+ "policy": "hold_back_screened";
10374
+ "screen": "test_awareness";
10375
+ "levels": Array<"aware" | "influenced">;
10376
+ } | null;
10377
+ "seriesKey"?: string | null;
10378
+ "ref"?: {
10379
+ "commit"?: string;
10380
+ "branch"?: string;
10381
+ "pr"?: number;
10382
+ "url"?: string;
10383
+ } | null;
10384
+ "gradingRevision"?: number;
10385
+ "gradingRevisedAt"?: string | null;
10386
+ "rehearsal"?: true;
10387
+ "rehearsalMode"?: string;
10388
+ "behaviors"?: Array<{
10389
+ "key": string;
10390
+ "name": string;
10391
+ "question": string;
10392
+ "version": number;
10393
+ }>;
9777
10394
  };
9778
10395
  "streamUrl": string;
9779
10396
  "warnings"?: Array<{
@@ -9784,6 +10401,68 @@ export type StudyGenerateResponse = {
9784
10401
  "detail": string;
9785
10402
  }>;
9786
10403
  };
10404
+ export type StudyGradingRevisionsResponse = {
10405
+ "revision": number;
10406
+ "revisions": Array<{
10407
+ "revision": number;
10408
+ "createdBy": string | null;
10409
+ "createdAt": string;
10410
+ "reason": string;
10411
+ "changes": Array<{
10412
+ "taskId": string;
10413
+ "position": number;
10414
+ "before": {
10415
+ "rubric": Array<string>;
10416
+ "expectations": Array<unknown>;
10417
+ };
10418
+ "after": {
10419
+ "rubric": Array<string>;
10420
+ "expectations": Array<unknown>;
10421
+ };
10422
+ "lenses": Array<"judge" | "evals">;
10423
+ }>;
10424
+ }>;
10425
+ };
10426
+ export type StudyReviseGradingRequest = {
10427
+ "reason": string;
10428
+ "tasks": Array<{
10429
+ "taskId": string;
10430
+ "rubric"?: Array<string>;
10431
+ "expectations"?: Array<unknown>;
10432
+ }>;
10433
+ "dryRun"?: boolean;
10434
+ };
10435
+ export type StudyReviseGradingResponse = {
10436
+ "dryRun": boolean;
10437
+ "plan": {
10438
+ "revision": number;
10439
+ "lenses": Array<"judge" | "evals">;
10440
+ "tasks": number;
10441
+ "evalRegrades": number;
10442
+ "judgeReads": number;
10443
+ };
10444
+ "changes": Array<{
10445
+ "taskId": string;
10446
+ "position": number;
10447
+ "before": {
10448
+ "rubric": Array<string>;
10449
+ "expectations": Array<unknown>;
10450
+ };
10451
+ "after": {
10452
+ "rubric": Array<string>;
10453
+ "expectations": Array<unknown>;
10454
+ };
10455
+ "lenses": Array<"judge" | "evals">;
10456
+ }>;
10457
+ "flips"?: Array<{
10458
+ "runId": string;
10459
+ "taskId": string;
10460
+ "arm": string;
10461
+ "before": number | null;
10462
+ "after": number | null;
10463
+ }>;
10464
+ "flipCount"?: number;
10465
+ };
9787
10466
  export type StudyLaunchResponse = {
9788
10467
  "study": {
9789
10468
  "id": string;
@@ -9919,6 +10598,8 @@ export type StudyLaunchResponse = {
9919
10598
  "startsPerMinute"?: number;
9920
10599
  } | null;
9921
10600
  "relaunchable"?: boolean;
10601
+ "archivedAt"?: string | null;
10602
+ "archivedBy"?: string | null;
9922
10603
  "arms"?: Array<{
9923
10604
  "key": string;
9924
10605
  "position": number;
@@ -9957,6 +10638,29 @@ export type StudyLaunchResponse = {
9957
10638
  "cellPlan"?: "crossed" | "per_setup";
9958
10639
  "skillDisclosure"?: "withheld" | "named";
9959
10640
  "modeOptions"?: Record<string, unknown>;
10641
+ "screens"?: Array<"test_awareness">;
10642
+ "runCounting"?: {
10643
+ "policy": "hold_back_screened";
10644
+ "screen": "test_awareness";
10645
+ "levels": Array<"aware" | "influenced">;
10646
+ } | null;
10647
+ "seriesKey"?: string | null;
10648
+ "ref"?: {
10649
+ "commit"?: string;
10650
+ "branch"?: string;
10651
+ "pr"?: number;
10652
+ "url"?: string;
10653
+ } | null;
10654
+ "gradingRevision"?: number;
10655
+ "gradingRevisedAt"?: string | null;
10656
+ "rehearsal"?: true;
10657
+ "rehearsalMode"?: string;
10658
+ "behaviors"?: Array<{
10659
+ "key": string;
10660
+ "name": string;
10661
+ "question": string;
10662
+ "version": number;
10663
+ }>;
9960
10664
  };
9961
10665
  "streamUrl": string;
9962
10666
  "warnings"?: Array<{
@@ -10105,6 +10809,8 @@ export type StudyUpdateLensesResponse = {
10105
10809
  "startsPerMinute"?: number;
10106
10810
  } | null;
10107
10811
  "relaunchable"?: boolean;
10812
+ "archivedAt"?: string | null;
10813
+ "archivedBy"?: string | null;
10108
10814
  "arms"?: Array<{
10109
10815
  "key": string;
10110
10816
  "position": number;
@@ -10143,6 +10849,29 @@ export type StudyUpdateLensesResponse = {
10143
10849
  "cellPlan"?: "crossed" | "per_setup";
10144
10850
  "skillDisclosure"?: "withheld" | "named";
10145
10851
  "modeOptions"?: Record<string, unknown>;
10852
+ "screens"?: Array<"test_awareness">;
10853
+ "runCounting"?: {
10854
+ "policy": "hold_back_screened";
10855
+ "screen": "test_awareness";
10856
+ "levels": Array<"aware" | "influenced">;
10857
+ } | null;
10858
+ "seriesKey"?: string | null;
10859
+ "ref"?: {
10860
+ "commit"?: string;
10861
+ "branch"?: string;
10862
+ "pr"?: number;
10863
+ "url"?: string;
10864
+ } | null;
10865
+ "gradingRevision"?: number;
10866
+ "gradingRevisedAt"?: string | null;
10867
+ "rehearsal"?: true;
10868
+ "rehearsalMode"?: string;
10869
+ "behaviors"?: Array<{
10870
+ "key": string;
10871
+ "name": string;
10872
+ "question": string;
10873
+ "version": number;
10874
+ }>;
10146
10875
  };
10147
10876
  "warnings": Array<{
10148
10877
  "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
@@ -10246,6 +10975,14 @@ export type StudyPreviewResponse = {
10246
10975
  "label": string;
10247
10976
  "skills": Array<string>;
10248
10977
  }>;
10978
+ "gradingRevision"?: number;
10979
+ "gradingRevisedAt"?: string | null;
10980
+ "behaviors"?: Array<{
10981
+ "key": string;
10982
+ "name": string;
10983
+ "question": string;
10984
+ "version": number;
10985
+ }>;
10249
10986
  };
10250
10987
  "tasks": Array<{
10251
10988
  "id": string;
@@ -10306,6 +11043,8 @@ export type StudyPreviewResponse = {
10306
11043
  "paramsKey"?: string;
10307
11044
  "textWithheld"?: boolean;
10308
11045
  "runTimeMs"?: number | null;
11046
+ "screens"?: Record<string, unknown>;
11047
+ "carriedRun"?: true;
10309
11048
  }>;
10310
11049
  "aggregate": {
10311
11050
  "summary": {
@@ -10569,6 +11308,18 @@ export type StudyPreviewResponse = {
10569
11308
  "byArm": Record<string, unknown>;
10570
11309
  "phasesByArm": Record<string, unknown>;
10571
11310
  };
11311
+ "runCounting"?: {
11312
+ "policy": string;
11313
+ "screen": string;
11314
+ "levels": Array<string>;
11315
+ "heldBack": number;
11316
+ "groups": Array<{
11317
+ "group": string;
11318
+ "heldBack": number;
11319
+ "runs": number;
11320
+ }>;
11321
+ "imbalanced": boolean;
11322
+ };
10572
11323
  } | {
10573
11324
  "shape": "multi-arm";
10574
11325
  "summary": {
@@ -11444,6 +12195,18 @@ export type StudyPreviewResponse = {
11444
12195
  "byArm": Record<string, unknown>;
11445
12196
  "phasesByArm": Record<string, unknown>;
11446
12197
  };
12198
+ "runCounting"?: {
12199
+ "policy": string;
12200
+ "screen": string;
12201
+ "levels": Array<string>;
12202
+ "heldBack": number;
12203
+ "groups": Array<{
12204
+ "group": string;
12205
+ "heldBack": number;
12206
+ "runs": number;
12207
+ }>;
12208
+ "imbalanced": boolean;
12209
+ };
11447
12210
  };
11448
12211
  "textWithheldReason"?: "sign_in_required" | "terms_required" | "licence";
11449
12212
  };
@@ -11588,6 +12351,8 @@ export type StudyPublishResponse = {
11588
12351
  "startsPerMinute"?: number;
11589
12352
  } | null;
11590
12353
  "relaunchable"?: boolean;
12354
+ "archivedAt"?: string | null;
12355
+ "archivedBy"?: string | null;
11591
12356
  "arms"?: Array<{
11592
12357
  "key": string;
11593
12358
  "position": number;
@@ -11626,8 +12391,72 @@ export type StudyPublishResponse = {
11626
12391
  "cellPlan"?: "crossed" | "per_setup";
11627
12392
  "skillDisclosure"?: "withheld" | "named";
11628
12393
  "modeOptions"?: Record<string, unknown>;
12394
+ "screens"?: Array<"test_awareness">;
12395
+ "runCounting"?: {
12396
+ "policy": "hold_back_screened";
12397
+ "screen": "test_awareness";
12398
+ "levels": Array<"aware" | "influenced">;
12399
+ } | null;
12400
+ "seriesKey"?: string | null;
12401
+ "ref"?: {
12402
+ "commit"?: string;
12403
+ "branch"?: string;
12404
+ "pr"?: number;
12405
+ "url"?: string;
12406
+ } | null;
12407
+ "gradingRevision"?: number;
12408
+ "gradingRevisedAt"?: string | null;
12409
+ "rehearsal"?: true;
12410
+ "rehearsalMode"?: string;
12411
+ "behaviors"?: Array<{
12412
+ "key": string;
12413
+ "name": string;
12414
+ "question": string;
12415
+ "version": number;
12416
+ }>;
11629
12417
  };
11630
12418
  };
12419
+ export type StudyQueryRequest = {
12420
+ "metrics": Array<string>;
12421
+ "groupBy"?: Array<string>;
12422
+ "filters"?: Array<{
12423
+ "field": string;
12424
+ "in": Array<string>;
12425
+ }>;
12426
+ "limit"?: number;
12427
+ };
12428
+ export type StudyQueryResponse = {
12429
+ "basis": "runs";
12430
+ "matchedRuns": number;
12431
+ "groupBy": Array<{
12432
+ "id": string;
12433
+ "label": string;
12434
+ }>;
12435
+ "metrics": Array<{
12436
+ "id": string;
12437
+ "label": string;
12438
+ "unit": "count" | "rate" | "points" | "usd" | "tokens" | "ms";
12439
+ "fold": string;
12440
+ }>;
12441
+ "rows": Array<{
12442
+ "key": Record<string, unknown>;
12443
+ "runs": number;
12444
+ "cells": Record<string, unknown>;
12445
+ }>;
12446
+ "truncated": boolean;
12447
+ };
12448
+ export type StudyQueryVocabularyResponse = {
12449
+ "dimensions": Array<{
12450
+ "id": string;
12451
+ "label": string;
12452
+ }>;
12453
+ "metrics": Array<{
12454
+ "id": string;
12455
+ "label": string;
12456
+ "unit": "count" | "rate" | "points" | "usd" | "tokens" | "ms";
12457
+ "fold": string;
12458
+ }>;
12459
+ };
11631
12460
  export type StudyReconcileRunsResponse = {
11632
12461
  "studyId": string;
11633
12462
  "laggingRuns": number;
@@ -11772,6 +12601,8 @@ export type StudyRegradeResponse = {
11772
12601
  "startsPerMinute"?: number;
11773
12602
  } | null;
11774
12603
  "relaunchable"?: boolean;
12604
+ "archivedAt"?: string | null;
12605
+ "archivedBy"?: string | null;
11775
12606
  "arms"?: Array<{
11776
12607
  "key": string;
11777
12608
  "position": number;
@@ -11810,6 +12641,29 @@ export type StudyRegradeResponse = {
11810
12641
  "cellPlan"?: "crossed" | "per_setup";
11811
12642
  "skillDisclosure"?: "withheld" | "named";
11812
12643
  "modeOptions"?: Record<string, unknown>;
12644
+ "screens"?: Array<"test_awareness">;
12645
+ "runCounting"?: {
12646
+ "policy": "hold_back_screened";
12647
+ "screen": "test_awareness";
12648
+ "levels": Array<"aware" | "influenced">;
12649
+ } | null;
12650
+ "seriesKey"?: string | null;
12651
+ "ref"?: {
12652
+ "commit"?: string;
12653
+ "branch"?: string;
12654
+ "pr"?: number;
12655
+ "url"?: string;
12656
+ } | null;
12657
+ "gradingRevision"?: number;
12658
+ "gradingRevisedAt"?: string | null;
12659
+ "rehearsal"?: true;
12660
+ "rehearsalMode"?: string;
12661
+ "behaviors"?: Array<{
12662
+ "key": string;
12663
+ "name": string;
12664
+ "question": string;
12665
+ "version": number;
12666
+ }>;
11813
12667
  };
11814
12668
  "streamUrl": string;
11815
12669
  "warnings"?: Array<{
@@ -11820,6 +12674,23 @@ export type StudyRegradeResponse = {
11820
12674
  "detail": string;
11821
12675
  }>;
11822
12676
  };
12677
+ export type StudyRehearsalReportResponse = {
12678
+ "studyId": string;
12679
+ "name": string;
12680
+ "status": string;
12681
+ "mode": string;
12682
+ "state": "pending" | "ready" | "not_ready";
12683
+ "groups": Array<{
12684
+ "key": string;
12685
+ "label": string;
12686
+ "state": "pending" | "ready" | "not_ready";
12687
+ "checks": Array<{
12688
+ "check": string;
12689
+ "answer": "pass" | "fail" | "pending" | "unknown";
12690
+ "detail": string;
12691
+ }>;
12692
+ }>;
12693
+ };
11823
12694
  export type StudyRerunAttemptsResponse = {
11824
12695
  "studyId": string;
11825
12696
  "attempts": Array<{
@@ -11888,6 +12759,405 @@ export type StudyRerunInfraFailuresResponse = {
11888
12759
  "reason": string;
11889
12760
  }>;
11890
12761
  };
12762
+ export type StudyRestoreResponse = {
12763
+ "study": {
12764
+ "id": string;
12765
+ "executiveSummary"?: string | null;
12766
+ "executiveSummaryDraft"?: string | null;
12767
+ "executiveSummaryRequest"?: "live" | "draft" | null;
12768
+ "executiveSummaryRequestedAt"?: string | null;
12769
+ "executiveSummaryUnavailableAt"?: string | null;
12770
+ "plannedRunCount": number;
12771
+ "organizationId": string;
12772
+ "createdBy": string | null;
12773
+ "name": string;
12774
+ "slug": string;
12775
+ "description": string | null;
12776
+ "skillRef": string | null;
12777
+ "skillSlug": string | null;
12778
+ "skillName": string | null;
12779
+ "skillPromise": string;
12780
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
12781
+ "subjectName": string | null;
12782
+ "subjectLabel": string;
12783
+ "armConfig": {
12784
+ "baseline"?: {
12785
+ "tools"?: "default" | "none" | {
12786
+ "tools": Array<string>;
12787
+ };
12788
+ "addons"?: Record<string, unknown>;
12789
+ };
12790
+ "treatment"?: {
12791
+ "tools"?: "default" | "none" | {
12792
+ "tools": Array<string>;
12793
+ };
12794
+ "addons"?: Record<string, unknown>;
12795
+ };
12796
+ };
12797
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
12798
+ "subjectBackend": "model" | "harness" | null;
12799
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
12800
+ "importedFrom"?: string;
12801
+ "baselineLabel": string | null;
12802
+ "treatmentLabel": string | null;
12803
+ "promiseMetrics": Array<{
12804
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
12805
+ "direction": "down" | "up";
12806
+ "label": string;
12807
+ }>;
12808
+ "matrix": Array<{
12809
+ "harness": string;
12810
+ "model": string;
12811
+ "provider"?: string;
12812
+ "params"?: {
12813
+ "temperature"?: number;
12814
+ "maxOutputTokens"?: number;
12815
+ "reasoningEffort"?: "low" | "medium" | "high";
12816
+ "thinkingBudgetTokens"?: number;
12817
+ "codeMode"?: "auto" | "on" | "off";
12818
+ "toolSearch"?: "on" | "off" | "directory";
12819
+ };
12820
+ }>;
12821
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
12822
+ "judgeSupplementary"?: boolean;
12823
+ "taskCount": number;
12824
+ "variantsPerTask": number;
12825
+ "repeats": number;
12826
+ "subset": {
12827
+ "count": number;
12828
+ "seed": string;
12829
+ "of": number;
12830
+ } | null;
12831
+ "status": string;
12832
+ "error": string | null;
12833
+ "preparation"?: {
12834
+ "since": string;
12835
+ "progressAt"?: string;
12836
+ "launchedBy": {
12837
+ "userId"?: string;
12838
+ "role"?: string;
12839
+ };
12840
+ "waitingOn": Array<{
12841
+ "imageKey": string;
12842
+ "buildId": string;
12843
+ "stage"?: "image" | "snapshot";
12844
+ "retry"?: {
12845
+ "failedAttempts": number;
12846
+ "maxAttempts": number;
12847
+ "notBefore": string;
12848
+ "lastFailure": string;
12849
+ };
12850
+ }>;
12851
+ } | null;
12852
+ "visibility": "private" | "org" | "link" | "public";
12853
+ "visibilityChangedAt"?: string | null;
12854
+ "visibilityChangedBy"?: string | null;
12855
+ "shareToken"?: string | null;
12856
+ "lowSampleOverride": boolean;
12857
+ "publishedAt": string | null;
12858
+ "verdict": unknown;
12859
+ "definitionVersionId": string | null;
12860
+ "runnerKind": "ora-runs" | "batch-job";
12861
+ "createdAt": string;
12862
+ "updatedAt": string;
12863
+ "liveUntil": string | null;
12864
+ "launch"?: {
12865
+ "attempt": number;
12866
+ "state": "fanning_out" | "deferred" | "settled";
12867
+ "startedAt": string;
12868
+ "progressAt": string;
12869
+ "planned"?: number;
12870
+ "created"?: number;
12871
+ "missing"?: {
12872
+ "retryable": number;
12873
+ "refused": number;
12874
+ };
12875
+ "notes"?: Array<string>;
12876
+ "settledAt"?: string;
12877
+ "pacing"?: {
12878
+ "waiting": number;
12879
+ "nextUnit"?: number;
12880
+ "nextWaveAt"?: string;
12881
+ "emptyWaves"?: number;
12882
+ "stopped"?: string;
12883
+ };
12884
+ "recovery"?: {
12885
+ "state": "retrying" | "needs_attention";
12886
+ "attempts": number;
12887
+ "lastAttemptAt"?: string;
12888
+ "nextAttemptAt"?: string;
12889
+ "reason"?: string;
12890
+ };
12891
+ } | null;
12892
+ "pacing"?: {
12893
+ "maxInFlight"?: number;
12894
+ "startsPerMinute"?: number;
12895
+ } | null;
12896
+ "relaunchable"?: boolean;
12897
+ "archivedAt"?: string | null;
12898
+ "archivedBy"?: string | null;
12899
+ "arms"?: Array<{
12900
+ "key": string;
12901
+ "position": number;
12902
+ "label": string;
12903
+ "role": "reference" | "peer";
12904
+ "payload": {
12905
+ "tools"?: "default" | "none" | {
12906
+ "tools": Array<string>;
12907
+ };
12908
+ "addons"?: Record<string, unknown>;
12909
+ };
12910
+ "cells"?: Array<{
12911
+ "harness": string;
12912
+ "model": string;
12913
+ "provider"?: string;
12914
+ "params"?: {
12915
+ "temperature"?: number;
12916
+ "maxOutputTokens"?: number;
12917
+ "reasoningEffort"?: "low" | "medium" | "high";
12918
+ "thinkingBudgetTokens"?: number;
12919
+ "codeMode"?: "auto" | "on" | "off";
12920
+ "toolSearch"?: "on" | "off" | "directory";
12921
+ };
12922
+ }>;
12923
+ "skills"?: Array<{
12924
+ "ref": string;
12925
+ "name": string;
12926
+ "changedSinceCreate"?: boolean;
12927
+ }>;
12928
+ "addedAgents"?: Array<{
12929
+ "position": number;
12930
+ "addedAt": string;
12931
+ }>;
12932
+ }>;
12933
+ "contrastPlan"?: "vs_reference" | "all_pairs";
12934
+ "cellPlan"?: "crossed" | "per_setup";
12935
+ "skillDisclosure"?: "withheld" | "named";
12936
+ "modeOptions"?: Record<string, unknown>;
12937
+ "screens"?: Array<"test_awareness">;
12938
+ "runCounting"?: {
12939
+ "policy": "hold_back_screened";
12940
+ "screen": "test_awareness";
12941
+ "levels": Array<"aware" | "influenced">;
12942
+ } | null;
12943
+ "seriesKey"?: string | null;
12944
+ "ref"?: {
12945
+ "commit"?: string;
12946
+ "branch"?: string;
12947
+ "pr"?: number;
12948
+ "url"?: string;
12949
+ } | null;
12950
+ "gradingRevision"?: number;
12951
+ "gradingRevisedAt"?: string | null;
12952
+ "rehearsal"?: true;
12953
+ "rehearsalMode"?: string;
12954
+ "behaviors"?: Array<{
12955
+ "key": string;
12956
+ "name": string;
12957
+ "question": string;
12958
+ "version": number;
12959
+ }>;
12960
+ };
12961
+ };
12962
+ export type StudyRetestsResponse = {
12963
+ "retests": Array<{
12964
+ "id": string;
12965
+ "kind": "fill" | "audit";
12966
+ "reason": string | null;
12967
+ "requestedBy": string | null;
12968
+ "createdAt": string;
12969
+ "settledAt": string | null;
12970
+ "summary": {
12971
+ "rechecked": number;
12972
+ "recorded": number;
12973
+ "filled": number;
12974
+ "agreed": number;
12975
+ "disagreements": Array<{
12976
+ "runId": string;
12977
+ "retestRunId": string;
12978
+ "run": {
12979
+ "pass": boolean | null;
12980
+ "status": "passed" | "failed" | "timeout" | "not-run" | "unavailable";
12981
+ "reason"?: string;
12982
+ };
12983
+ "recheck": {
12984
+ "pass": boolean | null;
12985
+ "status": "passed" | "failed" | "timeout" | "not-run" | "unavailable";
12986
+ "reason"?: string;
12987
+ };
12988
+ }>;
12989
+ };
12990
+ "runs": Array<{
12991
+ "runId": string;
12992
+ "retestRunId": string;
12993
+ "runStatus": string | null;
12994
+ "finishedAt": string | null;
12995
+ "verdict": {
12996
+ "pass": boolean | null;
12997
+ "status": "passed" | "failed" | "timeout" | "not-run" | "unavailable";
12998
+ "reason"?: string;
12999
+ } | null;
13000
+ }>;
13001
+ }>;
13002
+ };
13003
+ export type StudyRetestRequest = {
13004
+ "kind"?: "fill" | "audit";
13005
+ "runIds"?: Array<string>;
13006
+ "reason"?: string;
13007
+ "dryRun"?: boolean;
13008
+ };
13009
+ export type StudyRetestResponse = {
13010
+ "dryRun": boolean;
13011
+ "kind": "fill" | "audit";
13012
+ "retestId": string | null;
13013
+ "runs": Array<{
13014
+ "runId": string;
13015
+ "outcome"?: "dispatched" | "failed";
13016
+ "retestRunId"?: string | null;
13017
+ "error"?: string | null;
13018
+ }>;
13019
+ "skipped": Array<{
13020
+ "runId": string;
13021
+ "reason": "not_in_study" | "not_succeeded" | "no_verdict" | "no_saved_inputs" | "saved_files_missing";
13022
+ "detail": string;
13023
+ }>;
13024
+ };
13025
+ export type StudyRunCountingResponse = {
13026
+ "runCounting": {
13027
+ "policy": "hold_back_screened";
13028
+ "screen": "test_awareness";
13029
+ "levels": Array<"aware" | "influenced">;
13030
+ } | null;
13031
+ "policy": string;
13032
+ "offered": boolean;
13033
+ "refusal": {
13034
+ "code": "run_counting_not_enabled" | "run_counting_screen_not_enabled" | "run_counting_screen_undeclared" | "run_counting_pairs_unnamed" | "run_counting_platform_discloses_test" | "run_counting_benchmark_at_create";
13035
+ "message": string;
13036
+ } | null;
13037
+ "screen": string;
13038
+ "screenedRuns": number;
13039
+ "unscreenedRuns": number;
13040
+ "flagged": Record<string, unknown>;
13041
+ "heldBack": Array<{
13042
+ "runId": string;
13043
+ "group": string;
13044
+ "taskId": string;
13045
+ "level": string;
13046
+ }>;
13047
+ "groups": Array<{
13048
+ "key": string;
13049
+ "label": string;
13050
+ "runs": number;
13051
+ "heldBack": number;
13052
+ "counted": {
13053
+ "value": number | null;
13054
+ "interval": Array<unknown> | null;
13055
+ "runs": number;
13056
+ "tasks": number;
13057
+ } | null;
13058
+ "all": {
13059
+ "value": number | null;
13060
+ "interval": Array<unknown> | null;
13061
+ "runs": number;
13062
+ "tasks": number;
13063
+ } | null;
13064
+ }>;
13065
+ "imbalanced": boolean;
13066
+ "metric": {
13067
+ "key": string;
13068
+ "label": string;
13069
+ "family": string;
13070
+ };
13071
+ "changes": Array<{
13072
+ "change": number;
13073
+ "createdBy": string | null;
13074
+ "createdAt": string;
13075
+ "previous": {
13076
+ "policy": "hold_back_screened";
13077
+ "screen": "test_awareness";
13078
+ "levels": Array<"aware" | "influenced">;
13079
+ } | null;
13080
+ "runCounting": {
13081
+ "policy": "hold_back_screened";
13082
+ "screen": "test_awareness";
13083
+ "levels": Array<"aware" | "influenced">;
13084
+ } | null;
13085
+ "classifierVersion": string | null;
13086
+ }>;
13087
+ };
13088
+ export type StudySetRunCountingRequest = {
13089
+ "runCounting": {
13090
+ "policy": "hold_back_screened";
13091
+ "screen": "test_awareness";
13092
+ "levels": Array<"aware" | "influenced">;
13093
+ } | null;
13094
+ "dryRun"?: boolean;
13095
+ };
13096
+ export type StudySetRunCountingResponse = {
13097
+ "runCounting": {
13098
+ "policy": "hold_back_screened";
13099
+ "screen": "test_awareness";
13100
+ "levels": Array<"aware" | "influenced">;
13101
+ } | null;
13102
+ "policy": string;
13103
+ "offered": boolean;
13104
+ "refusal": {
13105
+ "code": "run_counting_not_enabled" | "run_counting_screen_not_enabled" | "run_counting_screen_undeclared" | "run_counting_pairs_unnamed" | "run_counting_platform_discloses_test" | "run_counting_benchmark_at_create";
13106
+ "message": string;
13107
+ } | null;
13108
+ "screen": string;
13109
+ "screenedRuns": number;
13110
+ "unscreenedRuns": number;
13111
+ "flagged": Record<string, unknown>;
13112
+ "heldBack": Array<{
13113
+ "runId": string;
13114
+ "group": string;
13115
+ "taskId": string;
13116
+ "level": string;
13117
+ }>;
13118
+ "groups": Array<{
13119
+ "key": string;
13120
+ "label": string;
13121
+ "runs": number;
13122
+ "heldBack": number;
13123
+ "counted": {
13124
+ "value": number | null;
13125
+ "interval": Array<unknown> | null;
13126
+ "runs": number;
13127
+ "tasks": number;
13128
+ } | null;
13129
+ "all": {
13130
+ "value": number | null;
13131
+ "interval": Array<unknown> | null;
13132
+ "runs": number;
13133
+ "tasks": number;
13134
+ } | null;
13135
+ }>;
13136
+ "imbalanced": boolean;
13137
+ "metric": {
13138
+ "key": string;
13139
+ "label": string;
13140
+ "family": string;
13141
+ };
13142
+ "changes": Array<{
13143
+ "change": number;
13144
+ "createdBy": string | null;
13145
+ "createdAt": string;
13146
+ "previous": {
13147
+ "policy": "hold_back_screened";
13148
+ "screen": "test_awareness";
13149
+ "levels": Array<"aware" | "influenced">;
13150
+ } | null;
13151
+ "runCounting": {
13152
+ "policy": "hold_back_screened";
13153
+ "screen": "test_awareness";
13154
+ "levels": Array<"aware" | "influenced">;
13155
+ } | null;
13156
+ "classifierVersion": string | null;
13157
+ }>;
13158
+ "dryRun": boolean;
13159
+ "changed": boolean;
13160
+ };
11891
13161
  export type StudyListRunAnswersResponse = {
11892
13162
  "coordinatorRunId": string;
11893
13163
  "answers": Array<{
@@ -11901,7 +13171,35 @@ export type StudyListRunAnswersResponse = {
11901
13171
  "finishedAt": string | null;
11902
13172
  }>;
11903
13173
  };
13174
+ export type StudyRunDeliveredResponse = {
13175
+ "takenAt": string | null;
13176
+ "items": Array<{
13177
+ "id": string;
13178
+ "kind": "file" | "image";
13179
+ "origin": string;
13180
+ "url"?: string;
13181
+ "path"?: string;
13182
+ "status": string;
13183
+ "reason"?: string;
13184
+ "contentType"?: string;
13185
+ "bytes"?: number;
13186
+ "sha256"?: string;
13187
+ "readable"?: boolean;
13188
+ "redactions"?: Array<string>;
13189
+ "truncated"?: boolean;
13190
+ "downloadable": boolean;
13191
+ "filename": string;
13192
+ }>;
13193
+ "workspaceSkipped"?: {
13194
+ "dependencyOrDot": number;
13195
+ "overCap": number;
13196
+ };
13197
+ };
11904
13198
  export type StudyRunInsightResponse = unknown;
13199
+ export type StudyRunScreensResponse = {
13200
+ "screens": Record<string, unknown>;
13201
+ };
13202
+ export type StudyRunSpansResponse = unknown;
11905
13203
  export type StudyRunTrajectoryResponse = unknown;
11906
13204
  export type StudyRotateShareTokenResponse = {
11907
13205
  "study": {
@@ -12038,6 +13336,8 @@ export type StudyRotateShareTokenResponse = {
12038
13336
  "startsPerMinute"?: number;
12039
13337
  } | null;
12040
13338
  "relaunchable"?: boolean;
13339
+ "archivedAt"?: string | null;
13340
+ "archivedBy"?: string | null;
12041
13341
  "arms"?: Array<{
12042
13342
  "key": string;
12043
13343
  "position": number;
@@ -12076,6 +13376,29 @@ export type StudyRotateShareTokenResponse = {
12076
13376
  "cellPlan"?: "crossed" | "per_setup";
12077
13377
  "skillDisclosure"?: "withheld" | "named";
12078
13378
  "modeOptions"?: Record<string, unknown>;
13379
+ "screens"?: Array<"test_awareness">;
13380
+ "runCounting"?: {
13381
+ "policy": "hold_back_screened";
13382
+ "screen": "test_awareness";
13383
+ "levels": Array<"aware" | "influenced">;
13384
+ } | null;
13385
+ "seriesKey"?: string | null;
13386
+ "ref"?: {
13387
+ "commit"?: string;
13388
+ "branch"?: string;
13389
+ "pr"?: number;
13390
+ "url"?: string;
13391
+ } | null;
13392
+ "gradingRevision"?: number;
13393
+ "gradingRevisedAt"?: string | null;
13394
+ "rehearsal"?: true;
13395
+ "rehearsalMode"?: string;
13396
+ "behaviors"?: Array<{
13397
+ "key": string;
13398
+ "name": string;
13399
+ "question": string;
13400
+ "version": number;
13401
+ }>;
12079
13402
  };
12080
13403
  "shareUrl": string | null;
12081
13404
  "shareLinkKept": boolean;
@@ -12302,6 +13625,8 @@ export type StudyUnpublishResponse = {
12302
13625
  "startsPerMinute"?: number;
12303
13626
  } | null;
12304
13627
  "relaunchable"?: boolean;
13628
+ "archivedAt"?: string | null;
13629
+ "archivedBy"?: string | null;
12305
13630
  "arms"?: Array<{
12306
13631
  "key": string;
12307
13632
  "position": number;
@@ -12340,8 +13665,93 @@ export type StudyUnpublishResponse = {
12340
13665
  "cellPlan"?: "crossed" | "per_setup";
12341
13666
  "skillDisclosure"?: "withheld" | "named";
12342
13667
  "modeOptions"?: Record<string, unknown>;
13668
+ "screens"?: Array<"test_awareness">;
13669
+ "runCounting"?: {
13670
+ "policy": "hold_back_screened";
13671
+ "screen": "test_awareness";
13672
+ "levels": Array<"aware" | "influenced">;
13673
+ } | null;
13674
+ "seriesKey"?: string | null;
13675
+ "ref"?: {
13676
+ "commit"?: string;
13677
+ "branch"?: string;
13678
+ "pr"?: number;
13679
+ "url"?: string;
13680
+ } | null;
13681
+ "gradingRevision"?: number;
13682
+ "gradingRevisedAt"?: string | null;
13683
+ "rehearsal"?: true;
13684
+ "rehearsalMode"?: string;
13685
+ "behaviors"?: Array<{
13686
+ "key": string;
13687
+ "name": string;
13688
+ "question": string;
13689
+ "version": number;
13690
+ }>;
13691
+ };
13692
+ };
13693
+ export type StudyViewsResponse = {
13694
+ "views": Array<{
13695
+ "id": string;
13696
+ "name": string;
13697
+ "query": unknown;
13698
+ "createdBy": string | null;
13699
+ "createdAt": string;
13700
+ }>;
13701
+ };
13702
+ export type StudySaveViewRequest = {
13703
+ "name": string;
13704
+ "query": {
13705
+ "metrics": Array<string>;
13706
+ "groupBy"?: Array<string>;
13707
+ "filters"?: Array<{
13708
+ "field": string;
13709
+ "in": Array<string>;
13710
+ }>;
13711
+ "limit"?: number;
13712
+ };
13713
+ };
13714
+ export type StudySaveViewResponse = {
13715
+ "view": {
13716
+ "id": string;
13717
+ "name": string;
13718
+ "query": unknown;
13719
+ "createdBy": string | null;
13720
+ "createdAt": string;
13721
+ };
13722
+ };
13723
+ export type StudyRunViewResponse = {
13724
+ "view": {
13725
+ "id": string;
13726
+ "name": string;
13727
+ "query": unknown;
13728
+ "createdBy": string | null;
13729
+ "createdAt": string;
13730
+ };
13731
+ "result": {
13732
+ "basis": "runs";
13733
+ "matchedRuns": number;
13734
+ "groupBy": Array<{
13735
+ "id": string;
13736
+ "label": string;
13737
+ }>;
13738
+ "metrics": Array<{
13739
+ "id": string;
13740
+ "label": string;
13741
+ "unit": "count" | "rate" | "points" | "usd" | "tokens" | "ms";
13742
+ "fold": string;
13743
+ }>;
13744
+ "rows": Array<{
13745
+ "key": Record<string, unknown>;
13746
+ "runs": number;
13747
+ "cells": Record<string, unknown>;
13748
+ }>;
13749
+ "truncated": boolean;
12343
13750
  };
12344
13751
  };
13752
+ export type StudyDeleteViewResponse = {
13753
+ "deleted": true;
13754
+ };
12345
13755
  export type StudySetVisibilityRequest = {
12346
13756
  "visibility": "private" | "org" | "link" | "public";
12347
13757
  };
@@ -12480,6 +13890,8 @@ export type StudySetVisibilityResponse = {
12480
13890
  "startsPerMinute"?: number;
12481
13891
  } | null;
12482
13892
  "relaunchable"?: boolean;
13893
+ "archivedAt"?: string | null;
13894
+ "archivedBy"?: string | null;
12483
13895
  "arms"?: Array<{
12484
13896
  "key": string;
12485
13897
  "position": number;
@@ -12518,6 +13930,29 @@ export type StudySetVisibilityResponse = {
12518
13930
  "cellPlan"?: "crossed" | "per_setup";
12519
13931
  "skillDisclosure"?: "withheld" | "named";
12520
13932
  "modeOptions"?: Record<string, unknown>;
13933
+ "screens"?: Array<"test_awareness">;
13934
+ "runCounting"?: {
13935
+ "policy": "hold_back_screened";
13936
+ "screen": "test_awareness";
13937
+ "levels": Array<"aware" | "influenced">;
13938
+ } | null;
13939
+ "seriesKey"?: string | null;
13940
+ "ref"?: {
13941
+ "commit"?: string;
13942
+ "branch"?: string;
13943
+ "pr"?: number;
13944
+ "url"?: string;
13945
+ } | null;
13946
+ "gradingRevision"?: number;
13947
+ "gradingRevisedAt"?: string | null;
13948
+ "rehearsal"?: true;
13949
+ "rehearsalMode"?: string;
13950
+ "behaviors"?: Array<{
13951
+ "key": string;
13952
+ "name": string;
13953
+ "question": string;
13954
+ "version": number;
13955
+ }>;
12521
13956
  };
12522
13957
  "shareUrl": string | null;
12523
13958
  "shareLinkKept": boolean;