@ora-ai/cli 0.21.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/README.md +93 -1
  2. package/dist/auth-commands.d.ts.map +1 -1
  3. package/dist/auth-commands.js +12 -1
  4. package/dist/auth-commands.js.map +1 -1
  5. package/dist/core/auth.d.ts +24 -13
  6. package/dist/core/auth.d.ts.map +1 -1
  7. package/dist/core/auth.js +26 -15
  8. package/dist/core/auth.js.map +1 -1
  9. package/dist/core/credential-sources.d.ts +45 -0
  10. package/dist/core/credential-sources.d.ts.map +1 -0
  11. package/dist/core/credential-sources.js +65 -0
  12. package/dist/core/credential-sources.js.map +1 -0
  13. package/dist/core/github-actions-sign-in.d.ts +33 -0
  14. package/dist/core/github-actions-sign-in.d.ts.map +1 -0
  15. package/dist/core/github-actions-sign-in.js +54 -0
  16. package/dist/core/github-actions-sign-in.js.map +1 -0
  17. package/dist/core/output.js +14 -1
  18. package/dist/core/output.js.map +1 -1
  19. package/dist/core/run-operation.d.ts +11 -4
  20. package/dist/core/run-operation.d.ts.map +1 -1
  21. package/dist/core/run-operation.js +14 -9
  22. package/dist/core/run-operation.js.map +1 -1
  23. package/dist/core/sign-in.d.ts +34 -0
  24. package/dist/core/sign-in.d.ts.map +1 -0
  25. package/dist/core/sign-in.js +21 -0
  26. package/dist/core/sign-in.js.map +1 -0
  27. package/dist/generated/operations.d.ts +1 -1
  28. package/dist/generated/operations.d.ts.map +1 -1
  29. package/dist/generated/operations.js +1236 -70
  30. package/dist/generated/operations.js.map +1 -1
  31. package/dist/generated/types.d.ts +1426 -10
  32. package/dist/generated/types.d.ts.map +1 -1
  33. package/dist/generated/types.js +1 -1
  34. package/dist/mcp/mcp-call.d.ts +25 -0
  35. package/dist/mcp/mcp-call.d.ts.map +1 -0
  36. package/dist/mcp/mcp-call.js +44 -0
  37. package/dist/mcp/mcp-call.js.map +1 -0
  38. package/dist/mcp/mcp-command.d.ts +26 -0
  39. package/dist/mcp/mcp-command.d.ts.map +1 -0
  40. package/dist/mcp/mcp-command.js +91 -0
  41. package/dist/mcp/mcp-command.js.map +1 -0
  42. package/dist/mcp/mcp-install.d.ts +50 -0
  43. package/dist/mcp/mcp-install.d.ts.map +1 -0
  44. package/dist/mcp/mcp-install.js +71 -0
  45. package/dist/mcp/mcp-install.js.map +1 -0
  46. package/dist/mcp/mcp-protocol.d.ts +69 -0
  47. package/dist/mcp/mcp-protocol.d.ts.map +1 -0
  48. package/dist/mcp/mcp-protocol.js +96 -0
  49. package/dist/mcp/mcp-protocol.js.map +1 -0
  50. package/dist/mcp/mcp-stdio.d.ts +17 -0
  51. package/dist/mcp/mcp-stdio.d.ts.map +1 -0
  52. package/dist/mcp/mcp-stdio.js +39 -0
  53. package/dist/mcp/mcp-stdio.js.map +1 -0
  54. package/dist/mcp/mcp-tools.d.ts +81 -0
  55. package/dist/mcp/mcp-tools.d.ts.map +1 -0
  56. package/dist/mcp/mcp-tools.js +134 -0
  57. package/dist/mcp/mcp-tools.js.map +1 -0
  58. package/dist/program.d.ts +4 -9
  59. package/dist/program.d.ts.map +1 -1
  60. package/dist/program.js +9 -3
  61. package/dist/program.js.map +1 -1
  62. package/dist/render/index.js +6 -0
  63. package/dist/render/index.js.map +1 -1
  64. package/dist/render/study/counting-views.d.ts +46 -0
  65. package/dist/render/study/counting-views.d.ts.map +1 -0
  66. package/dist/render/study/counting-views.js +49 -0
  67. package/dist/render/study/counting-views.js.map +1 -0
  68. package/dist/render/study/gate-views.d.ts +56 -0
  69. package/dist/render/study/gate-views.d.ts.map +1 -0
  70. package/dist/render/study/gate-views.js +64 -0
  71. package/dist/render/study/gate-views.js.map +1 -0
  72. package/dist/render/study/renderers.d.ts +8 -0
  73. package/dist/render/study/renderers.d.ts.map +1 -0
  74. package/dist/render/study/renderers.js +31 -0
  75. package/dist/render/study/renderers.js.map +1 -0
  76. package/dist/render/study/series-views.d.ts +65 -0
  77. package/dist/render/study/series-views.d.ts.map +1 -0
  78. package/dist/render/study/series-views.js +116 -0
  79. package/dist/render/study/series-views.js.map +1 -0
  80. package/dist/render/study/study-format.d.ts +24 -0
  81. package/dist/render/study/study-format.d.ts.map +1 -0
  82. package/dist/render/study/study-format.js +42 -0
  83. package/dist/render/study/study-format.js.map +1 -0
  84. package/dist/study-ci/ci-command.d.ts +31 -0
  85. package/dist/study-ci/ci-command.d.ts.map +1 -0
  86. package/dist/study-ci/ci-command.js +134 -0
  87. package/dist/study-ci/ci-command.js.map +1 -0
  88. package/dist/study-ci/ci-rehearse-command.d.ts +11 -0
  89. package/dist/study-ci/ci-rehearse-command.d.ts.map +1 -0
  90. package/dist/study-ci/ci-rehearse-command.js +83 -0
  91. package/dist/study-ci/ci-rehearse-command.js.map +1 -0
  92. package/dist/study-ci/ci-rehearse.d.ts +62 -0
  93. package/dist/study-ci/ci-rehearse.d.ts.map +1 -0
  94. package/dist/study-ci/ci-rehearse.js +49 -0
  95. package/dist/study-ci/ci-rehearse.js.map +1 -0
  96. package/dist/study-ci/ci-run.d.ts +87 -0
  97. package/dist/study-ci/ci-run.d.ts.map +1 -0
  98. package/dist/study-ci/ci-run.js +131 -0
  99. package/dist/study-ci/ci-run.js.map +1 -0
  100. package/dist/study-ci/gate-report.d.ts +64 -0
  101. package/dist/study-ci/gate-report.d.ts.map +1 -0
  102. package/dist/study-ci/gate-report.js +80 -0
  103. package/dist/study-ci/gate-report.js.map +1 -0
  104. package/dist/study-ci/params.d.ts +8 -0
  105. package/dist/study-ci/params.d.ts.map +1 -0
  106. package/dist/study-ci/params.js +72 -0
  107. package/dist/study-ci/params.js.map +1 -0
  108. package/package.json +1 -1
@@ -7174,6 +7174,8 @@ export type StudyListResponse = {
7174
7174
  "startsPerMinute"?: number;
7175
7175
  } | null;
7176
7176
  "relaunchable"?: boolean;
7177
+ "archivedAt"?: string | null;
7178
+ "archivedBy"?: string | null;
7177
7179
  "arms"?: Array<{
7178
7180
  "key": string;
7179
7181
  "position": number;
@@ -7212,6 +7214,29 @@ export type StudyListResponse = {
7212
7214
  "cellPlan"?: "crossed" | "per_setup";
7213
7215
  "skillDisclosure"?: "withheld" | "named";
7214
7216
  "modeOptions"?: Record<string, unknown>;
7217
+ "screens"?: Array<"test_awareness">;
7218
+ "runCounting"?: {
7219
+ "policy": "hold_back_screened";
7220
+ "screen": "test_awareness";
7221
+ "levels": Array<"aware" | "influenced">;
7222
+ } | null;
7223
+ "seriesKey"?: string | null;
7224
+ "ref"?: {
7225
+ "commit"?: string;
7226
+ "branch"?: string;
7227
+ "pr"?: number;
7228
+ "url"?: string;
7229
+ } | null;
7230
+ "gradingRevision"?: number;
7231
+ "gradingRevisedAt"?: string | null;
7232
+ "rehearsal"?: true;
7233
+ "rehearsalMode"?: string;
7234
+ "behaviors"?: Array<{
7235
+ "key": string;
7236
+ "name": string;
7237
+ "question": string;
7238
+ "version": number;
7239
+ }>;
7215
7240
  }>;
7216
7241
  };
7217
7242
  export type StudyCreateRequest = {
@@ -7357,6 +7382,28 @@ export type StudyCreateRequest = {
7357
7382
  "maxInFlight"?: number;
7358
7383
  "startsPerMinute"?: number;
7359
7384
  } | null;
7385
+ "screens"?: Array<"test_awareness">;
7386
+ "series"?: string;
7387
+ "ref"?: {
7388
+ "commit"?: string;
7389
+ "branch"?: string;
7390
+ "pr"?: number;
7391
+ "url"?: string;
7392
+ };
7393
+ "carry"?: {
7394
+ "from": "series";
7395
+ "maxAgeDays"?: number;
7396
+ };
7397
+ "runCounting"?: {
7398
+ "policy": "hold_back_screened";
7399
+ "screen": "test_awareness";
7400
+ "levels": Array<"aware" | "influenced">;
7401
+ } | null;
7402
+ "behaviors"?: Array<{
7403
+ "key": string;
7404
+ "name": string;
7405
+ "question": string;
7406
+ }>;
7360
7407
  "matrix": Array<{
7361
7408
  "harness": string;
7362
7409
  "model": string;
@@ -7507,6 +7554,8 @@ export type StudyCreateResponse = {
7507
7554
  "startsPerMinute"?: number;
7508
7555
  } | null;
7509
7556
  "relaunchable"?: boolean;
7557
+ "archivedAt"?: string | null;
7558
+ "archivedBy"?: string | null;
7510
7559
  "arms"?: Array<{
7511
7560
  "key": string;
7512
7561
  "position": number;
@@ -7545,6 +7594,29 @@ export type StudyCreateResponse = {
7545
7594
  "cellPlan"?: "crossed" | "per_setup";
7546
7595
  "skillDisclosure"?: "withheld" | "named";
7547
7596
  "modeOptions"?: Record<string, unknown>;
7597
+ "screens"?: Array<"test_awareness">;
7598
+ "runCounting"?: {
7599
+ "policy": "hold_back_screened";
7600
+ "screen": "test_awareness";
7601
+ "levels": Array<"aware" | "influenced">;
7602
+ } | null;
7603
+ "seriesKey"?: string | null;
7604
+ "ref"?: {
7605
+ "commit"?: string;
7606
+ "branch"?: string;
7607
+ "pr"?: number;
7608
+ "url"?: string;
7609
+ } | null;
7610
+ "gradingRevision"?: number;
7611
+ "gradingRevisedAt"?: string | null;
7612
+ "rehearsal"?: true;
7613
+ "rehearsalMode"?: string;
7614
+ "behaviors"?: Array<{
7615
+ "key": string;
7616
+ "name": string;
7617
+ "question": string;
7618
+ "version": number;
7619
+ }>;
7548
7620
  };
7549
7621
  "streamUrl": string;
7550
7622
  "warnings"?: Array<{
@@ -7565,6 +7637,10 @@ export type StudyComparisonModesListResponse = {
7565
7637
  "id": "crossed" | "per_setup";
7566
7638
  "enabled": boolean;
7567
7639
  }>;
7640
+ "customBehaviors": {
7641
+ "enabled": boolean;
7642
+ "max": number;
7643
+ };
7568
7644
  };
7569
7645
  export type StudyConvertWebmcpSuiteRequest = {
7570
7646
  "suite": unknown;
@@ -7721,6 +7797,28 @@ export type StudyPreflightRequest = {
7721
7797
  "maxInFlight"?: number;
7722
7798
  "startsPerMinute"?: number;
7723
7799
  } | null;
7800
+ "screens"?: Array<"test_awareness">;
7801
+ "series"?: string;
7802
+ "ref"?: {
7803
+ "commit"?: string;
7804
+ "branch"?: string;
7805
+ "pr"?: number;
7806
+ "url"?: string;
7807
+ };
7808
+ "carry"?: {
7809
+ "from": "series";
7810
+ "maxAgeDays"?: number;
7811
+ };
7812
+ "runCounting"?: {
7813
+ "policy": "hold_back_screened";
7814
+ "screen": "test_awareness";
7815
+ "levels": Array<"aware" | "influenced">;
7816
+ } | null;
7817
+ "behaviors"?: Array<{
7818
+ "key": string;
7819
+ "name": string;
7820
+ "question": string;
7821
+ }>;
7724
7822
  "matrix": Array<{
7725
7823
  "harness": string;
7726
7824
  "model": string;
@@ -7743,6 +7841,14 @@ export type StudyPreflightResponse = {
7743
7841
  "label": string;
7744
7842
  "plannedRunCount": number;
7745
7843
  "plannedJudgeComparisons": number;
7844
+ "screens"?: {
7845
+ "ids": Array<string>;
7846
+ "plannedReads": number;
7847
+ };
7848
+ "behaviors"?: {
7849
+ "keys": Array<string>;
7850
+ "plannedReads": number;
7851
+ };
7746
7852
  "runs": {
7747
7853
  "tasks": number;
7748
7854
  "variants": number;
@@ -7813,6 +7919,248 @@ export type StudyPreflightResponse = {
7813
7919
  "allowed": boolean;
7814
7920
  } | null;
7815
7921
  };
7922
+ export type StudyRehearseRequest = {
7923
+ "name": string;
7924
+ "description"?: string;
7925
+ "subjectKind"?: "skill" | "tools" | "addon" | "custom" | "benchmark";
7926
+ "skillRef"?: string;
7927
+ "definitionVersionId"?: string;
7928
+ "subjectName"?: string;
7929
+ "armConfig"?: {
7930
+ "baseline"?: {
7931
+ "tools"?: "default" | "none" | {
7932
+ "tools": Array<string>;
7933
+ };
7934
+ "addons"?: Record<string, unknown>;
7935
+ };
7936
+ "treatment"?: {
7937
+ "tools"?: "default" | "none" | {
7938
+ "tools": Array<string>;
7939
+ };
7940
+ "addons"?: Record<string, unknown>;
7941
+ };
7942
+ };
7943
+ "lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
7944
+ "judgeSupplementary"?: boolean;
7945
+ "proposeExpectations"?: boolean;
7946
+ "comparisonMode"?: "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
7947
+ "subjectBackend"?: "model" | "harness";
7948
+ "baselineLabel"?: string;
7949
+ "treatmentLabel"?: string;
7950
+ "arms"?: Array<{
7951
+ "key"?: string;
7952
+ "label": string;
7953
+ "role"?: "reference" | "peer";
7954
+ "config": {
7955
+ "tools"?: "default" | "none" | {
7956
+ "tools": Array<string>;
7957
+ };
7958
+ "addons"?: Record<string, unknown>;
7959
+ };
7960
+ "cells"?: Array<{
7961
+ "harness": string;
7962
+ "model": string;
7963
+ "provider"?: string;
7964
+ "params"?: {
7965
+ "temperature"?: number;
7966
+ "maxOutputTokens"?: number;
7967
+ "reasoningEffort"?: "low" | "medium" | "high";
7968
+ "thinkingBudgetTokens"?: number;
7969
+ "codeMode"?: "auto" | "on" | "off";
7970
+ "toolSearch"?: "on" | "off" | "directory";
7971
+ };
7972
+ }>;
7973
+ }>;
7974
+ "contrasts"?: "vs_reference" | "all_pairs";
7975
+ "modeOptions"?: Record<string, unknown>;
7976
+ "taskCount"?: number;
7977
+ "variantsPerTask"?: number;
7978
+ "repeats"?: number;
7979
+ "tasks"?: Array<{
7980
+ "prompt": string;
7981
+ "rationale"?: string;
7982
+ "category"?: string;
7983
+ "requiresWeb"?: boolean;
7984
+ "rubric"?: Array<string>;
7985
+ "variants"?: Array<string>;
7986
+ "expectations"?: Array<{
7987
+ "label"?: string;
7988
+ "arms"?: Array<"baseline" | "skill" | string>;
7989
+ "kind": "mcp_tool_called";
7990
+ "server"?: string;
7991
+ "tool": string;
7992
+ "arguments"?: Record<string, unknown>;
7993
+ "minCalls"?: number;
7994
+ } | {
7995
+ "label"?: string;
7996
+ "arms"?: Array<"baseline" | "skill" | string>;
7997
+ "kind": "tool_not_called";
7998
+ "tool"?: string;
7999
+ "mcp"?: {
8000
+ "server"?: string;
8001
+ "tool": string;
8002
+ };
8003
+ "pattern"?: string;
8004
+ } | {
8005
+ "label"?: string;
8006
+ "arms"?: Array<"baseline" | "skill" | string>;
8007
+ "kind": "command_ran";
8008
+ "pattern": string;
8009
+ } | {
8010
+ "label"?: string;
8011
+ "arms"?: Array<"baseline" | "skill" | string>;
8012
+ "kind": "skill_loaded";
8013
+ "slug": string;
8014
+ } | {
8015
+ "label"?: string;
8016
+ "arms"?: Array<"baseline" | "skill" | string>;
8017
+ "kind": "file_operation";
8018
+ "op"?: "read" | "write" | "edit";
8019
+ "path": unknown;
8020
+ } | {
8021
+ "label"?: string;
8022
+ "arms"?: Array<"baseline" | "skill" | string>;
8023
+ "kind": "web_fetched";
8024
+ "urlHost"?: string;
8025
+ "urlPattern"?: string;
8026
+ } | {
8027
+ "label"?: string;
8028
+ "arms"?: Array<"baseline" | "skill" | string>;
8029
+ "kind": "response_contains";
8030
+ "text": string;
8031
+ "normalize"?: "prose";
8032
+ } | {
8033
+ "label"?: string;
8034
+ "arms"?: Array<"baseline" | "skill" | string>;
8035
+ "kind": "response_matches";
8036
+ "pattern": string;
8037
+ "ignoreCase"?: boolean;
8038
+ } | {
8039
+ "label"?: string;
8040
+ "arms"?: Array<"baseline" | "skill" | string>;
8041
+ "kind": "token_budget";
8042
+ "maxOutputTokens"?: number;
8043
+ "maxTotalTokens"?: number;
8044
+ } | {
8045
+ "label"?: string;
8046
+ "arms"?: Array<"baseline" | "skill" | string>;
8047
+ "kind": "sequence";
8048
+ "of": Array<unknown>;
8049
+ } | {
8050
+ "label"?: string;
8051
+ "arms"?: Array<"baseline" | "skill" | string>;
8052
+ "kind": "any_of";
8053
+ "of": Array<unknown>;
8054
+ }>;
8055
+ "private"?: boolean;
8056
+ }>;
8057
+ "subset"?: {
8058
+ "count": number;
8059
+ "seed"?: string;
8060
+ };
8061
+ "pacing"?: {
8062
+ "maxInFlight"?: number;
8063
+ "startsPerMinute"?: number;
8064
+ } | null;
8065
+ "screens"?: Array<"test_awareness">;
8066
+ "series"?: string;
8067
+ "ref"?: {
8068
+ "commit"?: string;
8069
+ "branch"?: string;
8070
+ "pr"?: number;
8071
+ "url"?: string;
8072
+ };
8073
+ "carry"?: {
8074
+ "from": "series";
8075
+ "maxAgeDays"?: number;
8076
+ };
8077
+ "runCounting"?: {
8078
+ "policy": "hold_back_screened";
8079
+ "screen": "test_awareness";
8080
+ "levels": Array<"aware" | "influenced">;
8081
+ } | null;
8082
+ "behaviors"?: Array<{
8083
+ "key": string;
8084
+ "name": string;
8085
+ "question": string;
8086
+ }>;
8087
+ "matrix": Array<{
8088
+ "harness": string;
8089
+ "model": string;
8090
+ "provider"?: string;
8091
+ "params"?: {
8092
+ "temperature"?: number;
8093
+ "maxOutputTokens"?: number;
8094
+ "reasoningEffort"?: "low" | "medium" | "high";
8095
+ "thinkingBudgetTokens"?: number;
8096
+ "codeMode"?: "auto" | "on" | "off";
8097
+ "toolSearch"?: "on" | "off" | "directory";
8098
+ };
8099
+ }>;
8100
+ "promise"?: string;
8101
+ };
8102
+ export type StudyRehearseResponse = {
8103
+ "rehearsal": {
8104
+ "studyId": string;
8105
+ "name": string;
8106
+ "status": string;
8107
+ "tasks": number;
8108
+ "mode": string;
8109
+ };
8110
+ };
8111
+ export type StudySeriesResponse = {
8112
+ "series": Array<{
8113
+ "key": string;
8114
+ "studies": number;
8115
+ "newestAt": string;
8116
+ }>;
8117
+ "capped": boolean;
8118
+ };
8119
+ export type StudySeriesTrendResponse = {
8120
+ "series": string;
8121
+ "metric": {
8122
+ "key": string;
8123
+ "label": string;
8124
+ "family": string;
8125
+ };
8126
+ "countingPolicy": string;
8127
+ "studies": Array<{
8128
+ "studyId": string;
8129
+ "name": string;
8130
+ "createdAt": string;
8131
+ "ref": {
8132
+ "commit"?: string;
8133
+ "branch"?: string;
8134
+ "pr"?: number;
8135
+ "url"?: string;
8136
+ } | null;
8137
+ "state": "settled" | "pending";
8138
+ "owed": Array<string>;
8139
+ "segment": number | null;
8140
+ "heldBackRuns"?: number;
8141
+ "groups": Array<{
8142
+ "key": string;
8143
+ "label": string;
8144
+ "value": number | null;
8145
+ "interval": Array<unknown> | null;
8146
+ "tasks": number;
8147
+ "runs": number;
8148
+ "change": {
8149
+ "effect": number;
8150
+ "unit": string;
8151
+ "interval": Array<unknown>;
8152
+ "tasks": number;
8153
+ } | null;
8154
+ }>;
8155
+ }>;
8156
+ "segments": Array<{
8157
+ "index": number;
8158
+ "comparabilityKey": string;
8159
+ "studyIds": Array<string>;
8160
+ "differsFromPrevious": Array<"corpus" | "lenses" | "graders" | "classifiers" | "countingPolicy" | "agents">;
8161
+ }>;
8162
+ "truncated": boolean;
8163
+ };
7816
8164
  export type StudyGetResponse = {
7817
8165
  "study": {
7818
8166
  "id": string;
@@ -7948,6 +8296,8 @@ export type StudyGetResponse = {
7948
8296
  "startsPerMinute"?: number;
7949
8297
  } | null;
7950
8298
  "relaunchable"?: boolean;
8299
+ "archivedAt"?: string | null;
8300
+ "archivedBy"?: string | null;
7951
8301
  "arms"?: Array<{
7952
8302
  "key": string;
7953
8303
  "position": number;
@@ -7986,6 +8336,29 @@ export type StudyGetResponse = {
7986
8336
  "cellPlan"?: "crossed" | "per_setup";
7987
8337
  "skillDisclosure"?: "withheld" | "named";
7988
8338
  "modeOptions"?: Record<string, unknown>;
8339
+ "screens"?: Array<"test_awareness">;
8340
+ "runCounting"?: {
8341
+ "policy": "hold_back_screened";
8342
+ "screen": "test_awareness";
8343
+ "levels": Array<"aware" | "influenced">;
8344
+ } | null;
8345
+ "seriesKey"?: string | null;
8346
+ "ref"?: {
8347
+ "commit"?: string;
8348
+ "branch"?: string;
8349
+ "pr"?: number;
8350
+ "url"?: string;
8351
+ } | null;
8352
+ "gradingRevision"?: number;
8353
+ "gradingRevisedAt"?: string | null;
8354
+ "rehearsal"?: true;
8355
+ "rehearsalMode"?: string;
8356
+ "behaviors"?: Array<{
8357
+ "key": string;
8358
+ "name": string;
8359
+ "question": string;
8360
+ "version": number;
8361
+ }>;
7989
8362
  };
7990
8363
  "tasks": Array<{
7991
8364
  "id": string;
@@ -8134,6 +8507,12 @@ export type StudyGetResponse = {
8134
8507
  }>;
8135
8508
  "modelCalls": number | null;
8136
8509
  } | null;
8510
+ "screens"?: Record<string, unknown>;
8511
+ "carried"?: {
8512
+ "fromRunId": string;
8513
+ "fromStudyId": string | null;
8514
+ "at": string;
8515
+ } | null;
8137
8516
  }>;
8138
8517
  "aggregate": {
8139
8518
  "summary": {
@@ -8397,6 +8776,18 @@ export type StudyGetResponse = {
8397
8776
  "byArm": Record<string, unknown>;
8398
8777
  "phasesByArm": Record<string, unknown>;
8399
8778
  };
8779
+ "runCounting"?: {
8780
+ "policy": string;
8781
+ "screen": string;
8782
+ "levels": Array<string>;
8783
+ "heldBack": number;
8784
+ "groups": Array<{
8785
+ "group": string;
8786
+ "heldBack": number;
8787
+ "runs": number;
8788
+ }>;
8789
+ "imbalanced": boolean;
8790
+ };
8400
8791
  } | {
8401
8792
  "shape": "multi-arm";
8402
8793
  "summary": {
@@ -9272,6 +9663,18 @@ export type StudyGetResponse = {
9272
9663
  "byArm": Record<string, unknown>;
9273
9664
  "phasesByArm": Record<string, unknown>;
9274
9665
  };
9666
+ "runCounting"?: {
9667
+ "policy": string;
9668
+ "screen": string;
9669
+ "levels": Array<string>;
9670
+ "heldBack": number;
9671
+ "groups": Array<{
9672
+ "group": string;
9673
+ "heldBack": number;
9674
+ "runs": number;
9675
+ }>;
9676
+ "imbalanced": boolean;
9677
+ };
9275
9678
  };
9276
9679
  "warnings": Array<{
9277
9680
  "type": "eval_capability_gap";
@@ -9281,6 +9684,11 @@ export type StudyGetResponse = {
9281
9684
  "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
9282
9685
  "detail": string;
9283
9686
  "taskIds"?: Array<string>;
9687
+ } | {
9688
+ "type": "design_lint";
9689
+ "lint": string;
9690
+ "detail": string;
9691
+ "taskIds": Array<string>;
9284
9692
  } | {
9285
9693
  "type": "design_aggregate_budget";
9286
9694
  "detail": string;
@@ -9447,6 +9855,8 @@ export type StudyUpdateDetailsResponse = {
9447
9855
  "startsPerMinute"?: number;
9448
9856
  } | null;
9449
9857
  "relaunchable"?: boolean;
9858
+ "archivedAt"?: string | null;
9859
+ "archivedBy"?: string | null;
9450
9860
  "arms"?: Array<{
9451
9861
  "key": string;
9452
9862
  "position": number;
@@ -9485,6 +9895,29 @@ export type StudyUpdateDetailsResponse = {
9485
9895
  "cellPlan"?: "crossed" | "per_setup";
9486
9896
  "skillDisclosure"?: "withheld" | "named";
9487
9897
  "modeOptions"?: Record<string, unknown>;
9898
+ "screens"?: Array<"test_awareness">;
9899
+ "runCounting"?: {
9900
+ "policy": "hold_back_screened";
9901
+ "screen": "test_awareness";
9902
+ "levels": Array<"aware" | "influenced">;
9903
+ } | null;
9904
+ "seriesKey"?: string | null;
9905
+ "ref"?: {
9906
+ "commit"?: string;
9907
+ "branch"?: string;
9908
+ "pr"?: number;
9909
+ "url"?: string;
9910
+ } | null;
9911
+ "gradingRevision"?: number;
9912
+ "gradingRevisedAt"?: string | null;
9913
+ "rehearsal"?: true;
9914
+ "rehearsalMode"?: string;
9915
+ "behaviors"?: Array<{
9916
+ "key": string;
9917
+ "name": string;
9918
+ "question": string;
9919
+ "version": number;
9920
+ }>;
9488
9921
  };
9489
9922
  };
9490
9923
  export type StudyDeleteResponse = unknown;
@@ -9594,16 +10027,156 @@ export type StudyAddAgentsResponse = {
9594
10027
  "reason": string | null;
9595
10028
  };
9596
10029
  };
9597
- export type StudyRegenerateSummaryResponse = {
9598
- "status": "queued";
9599
- "target": "live" | "draft";
9600
- };
9601
- export type StudyGenerateRequest = {
9602
- "proposeExpectations"?: boolean;
9603
- };
9604
- export type StudyGenerateResponse = {
9605
- "study": {
9606
- "id": string;
10030
+ export type StudyBehaviorsResponse = {
10031
+ "behaviors": Array<{
10032
+ "key": string;
10033
+ "name": string;
10034
+ "question": string;
10035
+ "version": number;
10036
+ "screenable": number;
10037
+ "judged": number;
10038
+ "measurement": {
10039
+ "key": string;
10040
+ "version": number;
10041
+ "verdicts": {
10042
+ "present": number;
10043
+ "absent": number;
10044
+ "unknown": number;
10045
+ };
10046
+ "present": {
10047
+ "reviewed": number;
10048
+ "confirmed": number;
10049
+ "precision": {
10050
+ "rate": number;
10051
+ "lower": number;
10052
+ "upper": number;
10053
+ "successes": number;
10054
+ "n": number;
10055
+ } | null;
10056
+ };
10057
+ "absent": {
10058
+ "reviewed": number;
10059
+ "confirmed": number;
10060
+ "precision": {
10061
+ "rate": number;
10062
+ "lower": number;
10063
+ "upper": number;
10064
+ "successes": number;
10065
+ "n": number;
10066
+ } | null;
10067
+ };
10068
+ "contested": number;
10069
+ "agreement": {
10070
+ "runs": number;
10071
+ "agreed": number;
10072
+ "kappa": number | null;
10073
+ } | null;
10074
+ };
10075
+ }>;
10076
+ "reviews": number;
10077
+ "reviewsTruncated": boolean;
10078
+ };
10079
+ export type StudyCarryPlanResponse = {
10080
+ "policy": {
10081
+ "from": "series";
10082
+ "maxAgeDays": number;
10083
+ } | null;
10084
+ "sourceStudyId": string | null;
10085
+ "planned": number;
10086
+ "carried": number;
10087
+ "fresh": Record<string, unknown>;
10088
+ "differences": Array<string>;
10089
+ "carriedRuns": number;
10090
+ };
10091
+ export type StudyRegenerateSummaryResponse = {
10092
+ "status": "queued";
10093
+ "target": "live" | "draft";
10094
+ };
10095
+ export type StudyGateRequest = {
10096
+ "metric"?: "benchmark" | "goal" | "evals" | "quality" | "rubric" | "completion";
10097
+ "group"?: string;
10098
+ "floor"?: {
10099
+ "rate": number;
10100
+ } | {
10101
+ "points": number;
10102
+ };
10103
+ "maxDrop"?: {
10104
+ "pp": number;
10105
+ } | {
10106
+ "points": number;
10107
+ };
10108
+ "baseline"?: {
10109
+ "studyId": string;
10110
+ } | {
10111
+ "series": {
10112
+ "branch"?: string;
10113
+ };
10114
+ };
10115
+ "minTasks"?: number;
10116
+ "infraCeiling"?: number;
10117
+ "counting"?: "policy" | "all";
10118
+ };
10119
+ export type StudyGateResponse = {
10120
+ "state": "settled" | "pending";
10121
+ "owed": Array<string>;
10122
+ "countingPolicy": string;
10123
+ "metric": {
10124
+ "key": string;
10125
+ "label": string;
10126
+ "family": "rate" | "score";
10127
+ };
10128
+ "group": string;
10129
+ "groups": Array<{
10130
+ "key": string;
10131
+ "label": string;
10132
+ "value": number | null;
10133
+ "runs": number;
10134
+ "tasks": number;
10135
+ "interval": Array<unknown> | null;
10136
+ "carriedRuns"?: number;
10137
+ "heldBackRuns"?: number;
10138
+ }>;
10139
+ "baseline": {
10140
+ "studyId": string;
10141
+ "name": string;
10142
+ "comparable": boolean;
10143
+ "differs": Array<string>;
10144
+ "value": number | null;
10145
+ "interval": Array<unknown> | null;
10146
+ "difference": {
10147
+ "effect": number;
10148
+ "unit": "pp" | "pts";
10149
+ "interval": Array<unknown>;
10150
+ "tasks": number;
10151
+ } | null;
10152
+ } | null;
10153
+ "comparabilityKey": string;
10154
+ "series": {
10155
+ "key": string;
10156
+ "ref": {
10157
+ "commit"?: string;
10158
+ "branch"?: string;
10159
+ "pr"?: number;
10160
+ "url"?: string;
10161
+ } | null;
10162
+ } | null;
10163
+ "gate": {
10164
+ "decision": "pass" | "fail" | "undecided";
10165
+ "refusedBy": string | null;
10166
+ "reason": string;
10167
+ "rules": Array<{
10168
+ "rule": string;
10169
+ "decision": "pass" | "fail" | "undecided";
10170
+ "reason": string;
10171
+ }>;
10172
+ };
10173
+ };
10174
+ export type StudyGenerateRequest = {
10175
+ "proposeExpectations"?: boolean;
10176
+ };
10177
+ export type StudyGenerateResponse = {
10178
+ "study": {
10179
+ "id": string;
9607
10180
  "executiveSummary"?: string | null;
9608
10181
  "executiveSummaryDraft"?: string | null;
9609
10182
  "executiveSummaryRequest"?: "live" | "draft" | null;
@@ -9736,6 +10309,8 @@ export type StudyGenerateResponse = {
9736
10309
  "startsPerMinute"?: number;
9737
10310
  } | null;
9738
10311
  "relaunchable"?: boolean;
10312
+ "archivedAt"?: string | null;
10313
+ "archivedBy"?: string | null;
9739
10314
  "arms"?: Array<{
9740
10315
  "key": string;
9741
10316
  "position": number;
@@ -9774,6 +10349,29 @@ export type StudyGenerateResponse = {
9774
10349
  "cellPlan"?: "crossed" | "per_setup";
9775
10350
  "skillDisclosure"?: "withheld" | "named";
9776
10351
  "modeOptions"?: Record<string, unknown>;
10352
+ "screens"?: Array<"test_awareness">;
10353
+ "runCounting"?: {
10354
+ "policy": "hold_back_screened";
10355
+ "screen": "test_awareness";
10356
+ "levels": Array<"aware" | "influenced">;
10357
+ } | null;
10358
+ "seriesKey"?: string | null;
10359
+ "ref"?: {
10360
+ "commit"?: string;
10361
+ "branch"?: string;
10362
+ "pr"?: number;
10363
+ "url"?: string;
10364
+ } | null;
10365
+ "gradingRevision"?: number;
10366
+ "gradingRevisedAt"?: string | null;
10367
+ "rehearsal"?: true;
10368
+ "rehearsalMode"?: string;
10369
+ "behaviors"?: Array<{
10370
+ "key": string;
10371
+ "name": string;
10372
+ "question": string;
10373
+ "version": number;
10374
+ }>;
9777
10375
  };
9778
10376
  "streamUrl": string;
9779
10377
  "warnings"?: Array<{
@@ -9784,6 +10382,68 @@ export type StudyGenerateResponse = {
9784
10382
  "detail": string;
9785
10383
  }>;
9786
10384
  };
10385
+ export type StudyGradingRevisionsResponse = {
10386
+ "revision": number;
10387
+ "revisions": Array<{
10388
+ "revision": number;
10389
+ "createdBy": string | null;
10390
+ "createdAt": string;
10391
+ "reason": string;
10392
+ "changes": Array<{
10393
+ "taskId": string;
10394
+ "position": number;
10395
+ "before": {
10396
+ "rubric": Array<string>;
10397
+ "expectations": Array<unknown>;
10398
+ };
10399
+ "after": {
10400
+ "rubric": Array<string>;
10401
+ "expectations": Array<unknown>;
10402
+ };
10403
+ "lenses": Array<"judge" | "evals">;
10404
+ }>;
10405
+ }>;
10406
+ };
10407
+ export type StudyReviseGradingRequest = {
10408
+ "reason": string;
10409
+ "tasks": Array<{
10410
+ "taskId": string;
10411
+ "rubric"?: Array<string>;
10412
+ "expectations"?: Array<unknown>;
10413
+ }>;
10414
+ "dryRun"?: boolean;
10415
+ };
10416
+ export type StudyReviseGradingResponse = {
10417
+ "dryRun": boolean;
10418
+ "plan": {
10419
+ "revision": number;
10420
+ "lenses": Array<"judge" | "evals">;
10421
+ "tasks": number;
10422
+ "evalRegrades": number;
10423
+ "judgeReads": number;
10424
+ };
10425
+ "changes": Array<{
10426
+ "taskId": string;
10427
+ "position": number;
10428
+ "before": {
10429
+ "rubric": Array<string>;
10430
+ "expectations": Array<unknown>;
10431
+ };
10432
+ "after": {
10433
+ "rubric": Array<string>;
10434
+ "expectations": Array<unknown>;
10435
+ };
10436
+ "lenses": Array<"judge" | "evals">;
10437
+ }>;
10438
+ "flips"?: Array<{
10439
+ "runId": string;
10440
+ "taskId": string;
10441
+ "arm": string;
10442
+ "before": number | null;
10443
+ "after": number | null;
10444
+ }>;
10445
+ "flipCount"?: number;
10446
+ };
9787
10447
  export type StudyLaunchResponse = {
9788
10448
  "study": {
9789
10449
  "id": string;
@@ -9919,6 +10579,8 @@ export type StudyLaunchResponse = {
9919
10579
  "startsPerMinute"?: number;
9920
10580
  } | null;
9921
10581
  "relaunchable"?: boolean;
10582
+ "archivedAt"?: string | null;
10583
+ "archivedBy"?: string | null;
9922
10584
  "arms"?: Array<{
9923
10585
  "key": string;
9924
10586
  "position": number;
@@ -9957,6 +10619,29 @@ export type StudyLaunchResponse = {
9957
10619
  "cellPlan"?: "crossed" | "per_setup";
9958
10620
  "skillDisclosure"?: "withheld" | "named";
9959
10621
  "modeOptions"?: Record<string, unknown>;
10622
+ "screens"?: Array<"test_awareness">;
10623
+ "runCounting"?: {
10624
+ "policy": "hold_back_screened";
10625
+ "screen": "test_awareness";
10626
+ "levels": Array<"aware" | "influenced">;
10627
+ } | null;
10628
+ "seriesKey"?: string | null;
10629
+ "ref"?: {
10630
+ "commit"?: string;
10631
+ "branch"?: string;
10632
+ "pr"?: number;
10633
+ "url"?: string;
10634
+ } | null;
10635
+ "gradingRevision"?: number;
10636
+ "gradingRevisedAt"?: string | null;
10637
+ "rehearsal"?: true;
10638
+ "rehearsalMode"?: string;
10639
+ "behaviors"?: Array<{
10640
+ "key": string;
10641
+ "name": string;
10642
+ "question": string;
10643
+ "version": number;
10644
+ }>;
9960
10645
  };
9961
10646
  "streamUrl": string;
9962
10647
  "warnings"?: Array<{
@@ -10105,6 +10790,8 @@ export type StudyUpdateLensesResponse = {
10105
10790
  "startsPerMinute"?: number;
10106
10791
  } | null;
10107
10792
  "relaunchable"?: boolean;
10793
+ "archivedAt"?: string | null;
10794
+ "archivedBy"?: string | null;
10108
10795
  "arms"?: Array<{
10109
10796
  "key": string;
10110
10797
  "position": number;
@@ -10143,6 +10830,29 @@ export type StudyUpdateLensesResponse = {
10143
10830
  "cellPlan"?: "crossed" | "per_setup";
10144
10831
  "skillDisclosure"?: "withheld" | "named";
10145
10832
  "modeOptions"?: Record<string, unknown>;
10833
+ "screens"?: Array<"test_awareness">;
10834
+ "runCounting"?: {
10835
+ "policy": "hold_back_screened";
10836
+ "screen": "test_awareness";
10837
+ "levels": Array<"aware" | "influenced">;
10838
+ } | null;
10839
+ "seriesKey"?: string | null;
10840
+ "ref"?: {
10841
+ "commit"?: string;
10842
+ "branch"?: string;
10843
+ "pr"?: number;
10844
+ "url"?: string;
10845
+ } | null;
10846
+ "gradingRevision"?: number;
10847
+ "gradingRevisedAt"?: string | null;
10848
+ "rehearsal"?: true;
10849
+ "rehearsalMode"?: string;
10850
+ "behaviors"?: Array<{
10851
+ "key": string;
10852
+ "name": string;
10853
+ "question": string;
10854
+ "version": number;
10855
+ }>;
10146
10856
  };
10147
10857
  "warnings": Array<{
10148
10858
  "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
@@ -10246,6 +10956,14 @@ export type StudyPreviewResponse = {
10246
10956
  "label": string;
10247
10957
  "skills": Array<string>;
10248
10958
  }>;
10959
+ "gradingRevision"?: number;
10960
+ "gradingRevisedAt"?: string | null;
10961
+ "behaviors"?: Array<{
10962
+ "key": string;
10963
+ "name": string;
10964
+ "question": string;
10965
+ "version": number;
10966
+ }>;
10249
10967
  };
10250
10968
  "tasks": Array<{
10251
10969
  "id": string;
@@ -10306,6 +11024,8 @@ export type StudyPreviewResponse = {
10306
11024
  "paramsKey"?: string;
10307
11025
  "textWithheld"?: boolean;
10308
11026
  "runTimeMs"?: number | null;
11027
+ "screens"?: Record<string, unknown>;
11028
+ "carriedRun"?: true;
10309
11029
  }>;
10310
11030
  "aggregate": {
10311
11031
  "summary": {
@@ -10569,6 +11289,18 @@ export type StudyPreviewResponse = {
10569
11289
  "byArm": Record<string, unknown>;
10570
11290
  "phasesByArm": Record<string, unknown>;
10571
11291
  };
11292
+ "runCounting"?: {
11293
+ "policy": string;
11294
+ "screen": string;
11295
+ "levels": Array<string>;
11296
+ "heldBack": number;
11297
+ "groups": Array<{
11298
+ "group": string;
11299
+ "heldBack": number;
11300
+ "runs": number;
11301
+ }>;
11302
+ "imbalanced": boolean;
11303
+ };
10572
11304
  } | {
10573
11305
  "shape": "multi-arm";
10574
11306
  "summary": {
@@ -11444,6 +12176,18 @@ export type StudyPreviewResponse = {
11444
12176
  "byArm": Record<string, unknown>;
11445
12177
  "phasesByArm": Record<string, unknown>;
11446
12178
  };
12179
+ "runCounting"?: {
12180
+ "policy": string;
12181
+ "screen": string;
12182
+ "levels": Array<string>;
12183
+ "heldBack": number;
12184
+ "groups": Array<{
12185
+ "group": string;
12186
+ "heldBack": number;
12187
+ "runs": number;
12188
+ }>;
12189
+ "imbalanced": boolean;
12190
+ };
11447
12191
  };
11448
12192
  "textWithheldReason"?: "sign_in_required" | "terms_required" | "licence";
11449
12193
  };
@@ -11588,6 +12332,8 @@ export type StudyPublishResponse = {
11588
12332
  "startsPerMinute"?: number;
11589
12333
  } | null;
11590
12334
  "relaunchable"?: boolean;
12335
+ "archivedAt"?: string | null;
12336
+ "archivedBy"?: string | null;
11591
12337
  "arms"?: Array<{
11592
12338
  "key": string;
11593
12339
  "position": number;
@@ -11626,8 +12372,72 @@ export type StudyPublishResponse = {
11626
12372
  "cellPlan"?: "crossed" | "per_setup";
11627
12373
  "skillDisclosure"?: "withheld" | "named";
11628
12374
  "modeOptions"?: Record<string, unknown>;
12375
+ "screens"?: Array<"test_awareness">;
12376
+ "runCounting"?: {
12377
+ "policy": "hold_back_screened";
12378
+ "screen": "test_awareness";
12379
+ "levels": Array<"aware" | "influenced">;
12380
+ } | null;
12381
+ "seriesKey"?: string | null;
12382
+ "ref"?: {
12383
+ "commit"?: string;
12384
+ "branch"?: string;
12385
+ "pr"?: number;
12386
+ "url"?: string;
12387
+ } | null;
12388
+ "gradingRevision"?: number;
12389
+ "gradingRevisedAt"?: string | null;
12390
+ "rehearsal"?: true;
12391
+ "rehearsalMode"?: string;
12392
+ "behaviors"?: Array<{
12393
+ "key": string;
12394
+ "name": string;
12395
+ "question": string;
12396
+ "version": number;
12397
+ }>;
11629
12398
  };
11630
12399
  };
12400
+ export type StudyQueryRequest = {
12401
+ "metrics": Array<string>;
12402
+ "groupBy"?: Array<string>;
12403
+ "filters"?: Array<{
12404
+ "field": string;
12405
+ "in": Array<string>;
12406
+ }>;
12407
+ "limit"?: number;
12408
+ };
12409
+ export type StudyQueryResponse = {
12410
+ "basis": "runs";
12411
+ "matchedRuns": number;
12412
+ "groupBy": Array<{
12413
+ "id": string;
12414
+ "label": string;
12415
+ }>;
12416
+ "metrics": Array<{
12417
+ "id": string;
12418
+ "label": string;
12419
+ "unit": "count" | "rate" | "points" | "usd" | "tokens" | "ms";
12420
+ "fold": string;
12421
+ }>;
12422
+ "rows": Array<{
12423
+ "key": Record<string, unknown>;
12424
+ "runs": number;
12425
+ "cells": Record<string, unknown>;
12426
+ }>;
12427
+ "truncated": boolean;
12428
+ };
12429
+ export type StudyQueryVocabularyResponse = {
12430
+ "dimensions": Array<{
12431
+ "id": string;
12432
+ "label": string;
12433
+ }>;
12434
+ "metrics": Array<{
12435
+ "id": string;
12436
+ "label": string;
12437
+ "unit": "count" | "rate" | "points" | "usd" | "tokens" | "ms";
12438
+ "fold": string;
12439
+ }>;
12440
+ };
11631
12441
  export type StudyReconcileRunsResponse = {
11632
12442
  "studyId": string;
11633
12443
  "laggingRuns": number;
@@ -11772,6 +12582,8 @@ export type StudyRegradeResponse = {
11772
12582
  "startsPerMinute"?: number;
11773
12583
  } | null;
11774
12584
  "relaunchable"?: boolean;
12585
+ "archivedAt"?: string | null;
12586
+ "archivedBy"?: string | null;
11775
12587
  "arms"?: Array<{
11776
12588
  "key": string;
11777
12589
  "position": number;
@@ -11810,6 +12622,29 @@ export type StudyRegradeResponse = {
11810
12622
  "cellPlan"?: "crossed" | "per_setup";
11811
12623
  "skillDisclosure"?: "withheld" | "named";
11812
12624
  "modeOptions"?: Record<string, unknown>;
12625
+ "screens"?: Array<"test_awareness">;
12626
+ "runCounting"?: {
12627
+ "policy": "hold_back_screened";
12628
+ "screen": "test_awareness";
12629
+ "levels": Array<"aware" | "influenced">;
12630
+ } | null;
12631
+ "seriesKey"?: string | null;
12632
+ "ref"?: {
12633
+ "commit"?: string;
12634
+ "branch"?: string;
12635
+ "pr"?: number;
12636
+ "url"?: string;
12637
+ } | null;
12638
+ "gradingRevision"?: number;
12639
+ "gradingRevisedAt"?: string | null;
12640
+ "rehearsal"?: true;
12641
+ "rehearsalMode"?: string;
12642
+ "behaviors"?: Array<{
12643
+ "key": string;
12644
+ "name": string;
12645
+ "question": string;
12646
+ "version": number;
12647
+ }>;
11813
12648
  };
11814
12649
  "streamUrl": string;
11815
12650
  "warnings"?: Array<{
@@ -11820,6 +12655,23 @@ export type StudyRegradeResponse = {
11820
12655
  "detail": string;
11821
12656
  }>;
11822
12657
  };
12658
+ export type StudyRehearsalReportResponse = {
12659
+ "studyId": string;
12660
+ "name": string;
12661
+ "status": string;
12662
+ "mode": string;
12663
+ "state": "pending" | "ready" | "not_ready";
12664
+ "groups": Array<{
12665
+ "key": string;
12666
+ "label": string;
12667
+ "state": "pending" | "ready" | "not_ready";
12668
+ "checks": Array<{
12669
+ "check": string;
12670
+ "answer": "pass" | "fail" | "pending" | "unknown";
12671
+ "detail": string;
12672
+ }>;
12673
+ }>;
12674
+ };
11823
12675
  export type StudyRerunAttemptsResponse = {
11824
12676
  "studyId": string;
11825
12677
  "attempts": Array<{
@@ -11888,6 +12740,405 @@ export type StudyRerunInfraFailuresResponse = {
11888
12740
  "reason": string;
11889
12741
  }>;
11890
12742
  };
12743
+ export type StudyRestoreResponse = {
12744
+ "study": {
12745
+ "id": string;
12746
+ "executiveSummary"?: string | null;
12747
+ "executiveSummaryDraft"?: string | null;
12748
+ "executiveSummaryRequest"?: "live" | "draft" | null;
12749
+ "executiveSummaryRequestedAt"?: string | null;
12750
+ "executiveSummaryUnavailableAt"?: string | null;
12751
+ "plannedRunCount": number;
12752
+ "organizationId": string;
12753
+ "createdBy": string | null;
12754
+ "name": string;
12755
+ "slug": string;
12756
+ "description": string | null;
12757
+ "skillRef": string | null;
12758
+ "skillSlug": string | null;
12759
+ "skillName": string | null;
12760
+ "skillPromise": string;
12761
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
12762
+ "subjectName": string | null;
12763
+ "subjectLabel": string;
12764
+ "armConfig": {
12765
+ "baseline"?: {
12766
+ "tools"?: "default" | "none" | {
12767
+ "tools": Array<string>;
12768
+ };
12769
+ "addons"?: Record<string, unknown>;
12770
+ };
12771
+ "treatment"?: {
12772
+ "tools"?: "default" | "none" | {
12773
+ "tools": Array<string>;
12774
+ };
12775
+ "addons"?: Record<string, unknown>;
12776
+ };
12777
+ };
12778
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
12779
+ "subjectBackend": "model" | "harness" | null;
12780
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
12781
+ "importedFrom"?: string;
12782
+ "baselineLabel": string | null;
12783
+ "treatmentLabel": string | null;
12784
+ "promiseMetrics": Array<{
12785
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
12786
+ "direction": "down" | "up";
12787
+ "label": string;
12788
+ }>;
12789
+ "matrix": Array<{
12790
+ "harness": string;
12791
+ "model": string;
12792
+ "provider"?: string;
12793
+ "params"?: {
12794
+ "temperature"?: number;
12795
+ "maxOutputTokens"?: number;
12796
+ "reasoningEffort"?: "low" | "medium" | "high";
12797
+ "thinkingBudgetTokens"?: number;
12798
+ "codeMode"?: "auto" | "on" | "off";
12799
+ "toolSearch"?: "on" | "off" | "directory";
12800
+ };
12801
+ }>;
12802
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
12803
+ "judgeSupplementary"?: boolean;
12804
+ "taskCount": number;
12805
+ "variantsPerTask": number;
12806
+ "repeats": number;
12807
+ "subset": {
12808
+ "count": number;
12809
+ "seed": string;
12810
+ "of": number;
12811
+ } | null;
12812
+ "status": string;
12813
+ "error": string | null;
12814
+ "preparation"?: {
12815
+ "since": string;
12816
+ "progressAt"?: string;
12817
+ "launchedBy": {
12818
+ "userId"?: string;
12819
+ "role"?: string;
12820
+ };
12821
+ "waitingOn": Array<{
12822
+ "imageKey": string;
12823
+ "buildId": string;
12824
+ "stage"?: "image" | "snapshot";
12825
+ "retry"?: {
12826
+ "failedAttempts": number;
12827
+ "maxAttempts": number;
12828
+ "notBefore": string;
12829
+ "lastFailure": string;
12830
+ };
12831
+ }>;
12832
+ } | null;
12833
+ "visibility": "private" | "org" | "link" | "public";
12834
+ "visibilityChangedAt"?: string | null;
12835
+ "visibilityChangedBy"?: string | null;
12836
+ "shareToken"?: string | null;
12837
+ "lowSampleOverride": boolean;
12838
+ "publishedAt": string | null;
12839
+ "verdict": unknown;
12840
+ "definitionVersionId": string | null;
12841
+ "runnerKind": "ora-runs" | "batch-job";
12842
+ "createdAt": string;
12843
+ "updatedAt": string;
12844
+ "liveUntil": string | null;
12845
+ "launch"?: {
12846
+ "attempt": number;
12847
+ "state": "fanning_out" | "deferred" | "settled";
12848
+ "startedAt": string;
12849
+ "progressAt": string;
12850
+ "planned"?: number;
12851
+ "created"?: number;
12852
+ "missing"?: {
12853
+ "retryable": number;
12854
+ "refused": number;
12855
+ };
12856
+ "notes"?: Array<string>;
12857
+ "settledAt"?: string;
12858
+ "pacing"?: {
12859
+ "waiting": number;
12860
+ "nextUnit"?: number;
12861
+ "nextWaveAt"?: string;
12862
+ "emptyWaves"?: number;
12863
+ "stopped"?: string;
12864
+ };
12865
+ "recovery"?: {
12866
+ "state": "retrying" | "needs_attention";
12867
+ "attempts": number;
12868
+ "lastAttemptAt"?: string;
12869
+ "nextAttemptAt"?: string;
12870
+ "reason"?: string;
12871
+ };
12872
+ } | null;
12873
+ "pacing"?: {
12874
+ "maxInFlight"?: number;
12875
+ "startsPerMinute"?: number;
12876
+ } | null;
12877
+ "relaunchable"?: boolean;
12878
+ "archivedAt"?: string | null;
12879
+ "archivedBy"?: string | null;
12880
+ "arms"?: Array<{
12881
+ "key": string;
12882
+ "position": number;
12883
+ "label": string;
12884
+ "role": "reference" | "peer";
12885
+ "payload": {
12886
+ "tools"?: "default" | "none" | {
12887
+ "tools": Array<string>;
12888
+ };
12889
+ "addons"?: Record<string, unknown>;
12890
+ };
12891
+ "cells"?: Array<{
12892
+ "harness": string;
12893
+ "model": string;
12894
+ "provider"?: string;
12895
+ "params"?: {
12896
+ "temperature"?: number;
12897
+ "maxOutputTokens"?: number;
12898
+ "reasoningEffort"?: "low" | "medium" | "high";
12899
+ "thinkingBudgetTokens"?: number;
12900
+ "codeMode"?: "auto" | "on" | "off";
12901
+ "toolSearch"?: "on" | "off" | "directory";
12902
+ };
12903
+ }>;
12904
+ "skills"?: Array<{
12905
+ "ref": string;
12906
+ "name": string;
12907
+ "changedSinceCreate"?: boolean;
12908
+ }>;
12909
+ "addedAgents"?: Array<{
12910
+ "position": number;
12911
+ "addedAt": string;
12912
+ }>;
12913
+ }>;
12914
+ "contrastPlan"?: "vs_reference" | "all_pairs";
12915
+ "cellPlan"?: "crossed" | "per_setup";
12916
+ "skillDisclosure"?: "withheld" | "named";
12917
+ "modeOptions"?: Record<string, unknown>;
12918
+ "screens"?: Array<"test_awareness">;
12919
+ "runCounting"?: {
12920
+ "policy": "hold_back_screened";
12921
+ "screen": "test_awareness";
12922
+ "levels": Array<"aware" | "influenced">;
12923
+ } | null;
12924
+ "seriesKey"?: string | null;
12925
+ "ref"?: {
12926
+ "commit"?: string;
12927
+ "branch"?: string;
12928
+ "pr"?: number;
12929
+ "url"?: string;
12930
+ } | null;
12931
+ "gradingRevision"?: number;
12932
+ "gradingRevisedAt"?: string | null;
12933
+ "rehearsal"?: true;
12934
+ "rehearsalMode"?: string;
12935
+ "behaviors"?: Array<{
12936
+ "key": string;
12937
+ "name": string;
12938
+ "question": string;
12939
+ "version": number;
12940
+ }>;
12941
+ };
12942
+ };
12943
+ export type StudyRetestsResponse = {
12944
+ "retests": Array<{
12945
+ "id": string;
12946
+ "kind": "fill" | "audit";
12947
+ "reason": string | null;
12948
+ "requestedBy": string | null;
12949
+ "createdAt": string;
12950
+ "settledAt": string | null;
12951
+ "summary": {
12952
+ "rechecked": number;
12953
+ "recorded": number;
12954
+ "filled": number;
12955
+ "agreed": number;
12956
+ "disagreements": Array<{
12957
+ "runId": string;
12958
+ "retestRunId": string;
12959
+ "run": {
12960
+ "pass": boolean | null;
12961
+ "status": "passed" | "failed" | "timeout" | "not-run" | "unavailable";
12962
+ "reason"?: string;
12963
+ };
12964
+ "recheck": {
12965
+ "pass": boolean | null;
12966
+ "status": "passed" | "failed" | "timeout" | "not-run" | "unavailable";
12967
+ "reason"?: string;
12968
+ };
12969
+ }>;
12970
+ };
12971
+ "runs": Array<{
12972
+ "runId": string;
12973
+ "retestRunId": string;
12974
+ "runStatus": string | null;
12975
+ "finishedAt": string | null;
12976
+ "verdict": {
12977
+ "pass": boolean | null;
12978
+ "status": "passed" | "failed" | "timeout" | "not-run" | "unavailable";
12979
+ "reason"?: string;
12980
+ } | null;
12981
+ }>;
12982
+ }>;
12983
+ };
12984
+ export type StudyRetestRequest = {
12985
+ "kind"?: "fill" | "audit";
12986
+ "runIds"?: Array<string>;
12987
+ "reason"?: string;
12988
+ "dryRun"?: boolean;
12989
+ };
12990
+ export type StudyRetestResponse = {
12991
+ "dryRun": boolean;
12992
+ "kind": "fill" | "audit";
12993
+ "retestId": string | null;
12994
+ "runs": Array<{
12995
+ "runId": string;
12996
+ "outcome"?: "dispatched" | "failed";
12997
+ "retestRunId"?: string | null;
12998
+ "error"?: string | null;
12999
+ }>;
13000
+ "skipped": Array<{
13001
+ "runId": string;
13002
+ "reason": "not_in_study" | "not_succeeded" | "no_verdict" | "no_saved_inputs" | "saved_files_missing";
13003
+ "detail": string;
13004
+ }>;
13005
+ };
13006
+ export type StudyRunCountingResponse = {
13007
+ "runCounting": {
13008
+ "policy": "hold_back_screened";
13009
+ "screen": "test_awareness";
13010
+ "levels": Array<"aware" | "influenced">;
13011
+ } | null;
13012
+ "policy": string;
13013
+ "offered": boolean;
13014
+ "refusal": {
13015
+ "code": "run_counting_not_enabled" | "run_counting_screen_not_enabled" | "run_counting_screen_undeclared" | "run_counting_pairs_unnamed" | "run_counting_platform_discloses_test" | "run_counting_benchmark_at_create";
13016
+ "message": string;
13017
+ } | null;
13018
+ "screen": string;
13019
+ "screenedRuns": number;
13020
+ "unscreenedRuns": number;
13021
+ "flagged": Record<string, unknown>;
13022
+ "heldBack": Array<{
13023
+ "runId": string;
13024
+ "group": string;
13025
+ "taskId": string;
13026
+ "level": string;
13027
+ }>;
13028
+ "groups": Array<{
13029
+ "key": string;
13030
+ "label": string;
13031
+ "runs": number;
13032
+ "heldBack": number;
13033
+ "counted": {
13034
+ "value": number | null;
13035
+ "interval": Array<unknown> | null;
13036
+ "runs": number;
13037
+ "tasks": number;
13038
+ } | null;
13039
+ "all": {
13040
+ "value": number | null;
13041
+ "interval": Array<unknown> | null;
13042
+ "runs": number;
13043
+ "tasks": number;
13044
+ } | null;
13045
+ }>;
13046
+ "imbalanced": boolean;
13047
+ "metric": {
13048
+ "key": string;
13049
+ "label": string;
13050
+ "family": string;
13051
+ };
13052
+ "changes": Array<{
13053
+ "change": number;
13054
+ "createdBy": string | null;
13055
+ "createdAt": string;
13056
+ "previous": {
13057
+ "policy": "hold_back_screened";
13058
+ "screen": "test_awareness";
13059
+ "levels": Array<"aware" | "influenced">;
13060
+ } | null;
13061
+ "runCounting": {
13062
+ "policy": "hold_back_screened";
13063
+ "screen": "test_awareness";
13064
+ "levels": Array<"aware" | "influenced">;
13065
+ } | null;
13066
+ "classifierVersion": string | null;
13067
+ }>;
13068
+ };
13069
+ export type StudySetRunCountingRequest = {
13070
+ "runCounting": {
13071
+ "policy": "hold_back_screened";
13072
+ "screen": "test_awareness";
13073
+ "levels": Array<"aware" | "influenced">;
13074
+ } | null;
13075
+ "dryRun"?: boolean;
13076
+ };
13077
+ export type StudySetRunCountingResponse = {
13078
+ "runCounting": {
13079
+ "policy": "hold_back_screened";
13080
+ "screen": "test_awareness";
13081
+ "levels": Array<"aware" | "influenced">;
13082
+ } | null;
13083
+ "policy": string;
13084
+ "offered": boolean;
13085
+ "refusal": {
13086
+ "code": "run_counting_not_enabled" | "run_counting_screen_not_enabled" | "run_counting_screen_undeclared" | "run_counting_pairs_unnamed" | "run_counting_platform_discloses_test" | "run_counting_benchmark_at_create";
13087
+ "message": string;
13088
+ } | null;
13089
+ "screen": string;
13090
+ "screenedRuns": number;
13091
+ "unscreenedRuns": number;
13092
+ "flagged": Record<string, unknown>;
13093
+ "heldBack": Array<{
13094
+ "runId": string;
13095
+ "group": string;
13096
+ "taskId": string;
13097
+ "level": string;
13098
+ }>;
13099
+ "groups": Array<{
13100
+ "key": string;
13101
+ "label": string;
13102
+ "runs": number;
13103
+ "heldBack": number;
13104
+ "counted": {
13105
+ "value": number | null;
13106
+ "interval": Array<unknown> | null;
13107
+ "runs": number;
13108
+ "tasks": number;
13109
+ } | null;
13110
+ "all": {
13111
+ "value": number | null;
13112
+ "interval": Array<unknown> | null;
13113
+ "runs": number;
13114
+ "tasks": number;
13115
+ } | null;
13116
+ }>;
13117
+ "imbalanced": boolean;
13118
+ "metric": {
13119
+ "key": string;
13120
+ "label": string;
13121
+ "family": string;
13122
+ };
13123
+ "changes": Array<{
13124
+ "change": number;
13125
+ "createdBy": string | null;
13126
+ "createdAt": string;
13127
+ "previous": {
13128
+ "policy": "hold_back_screened";
13129
+ "screen": "test_awareness";
13130
+ "levels": Array<"aware" | "influenced">;
13131
+ } | null;
13132
+ "runCounting": {
13133
+ "policy": "hold_back_screened";
13134
+ "screen": "test_awareness";
13135
+ "levels": Array<"aware" | "influenced">;
13136
+ } | null;
13137
+ "classifierVersion": string | null;
13138
+ }>;
13139
+ "dryRun": boolean;
13140
+ "changed": boolean;
13141
+ };
11891
13142
  export type StudyListRunAnswersResponse = {
11892
13143
  "coordinatorRunId": string;
11893
13144
  "answers": Array<{
@@ -11901,7 +13152,35 @@ export type StudyListRunAnswersResponse = {
11901
13152
  "finishedAt": string | null;
11902
13153
  }>;
11903
13154
  };
13155
+ export type StudyRunDeliveredResponse = {
13156
+ "takenAt": string | null;
13157
+ "items": Array<{
13158
+ "id": string;
13159
+ "kind": "file" | "image";
13160
+ "origin": string;
13161
+ "url"?: string;
13162
+ "path"?: string;
13163
+ "status": string;
13164
+ "reason"?: string;
13165
+ "contentType"?: string;
13166
+ "bytes"?: number;
13167
+ "sha256"?: string;
13168
+ "readable"?: boolean;
13169
+ "redactions"?: Array<string>;
13170
+ "truncated"?: boolean;
13171
+ "downloadable": boolean;
13172
+ "filename": string;
13173
+ }>;
13174
+ "workspaceSkipped"?: {
13175
+ "dependencyOrDot": number;
13176
+ "overCap": number;
13177
+ };
13178
+ };
11904
13179
  export type StudyRunInsightResponse = unknown;
13180
+ export type StudyRunScreensResponse = {
13181
+ "screens": Record<string, unknown>;
13182
+ };
13183
+ export type StudyRunSpansResponse = unknown;
11905
13184
  export type StudyRunTrajectoryResponse = unknown;
11906
13185
  export type StudyRotateShareTokenResponse = {
11907
13186
  "study": {
@@ -12038,6 +13317,8 @@ export type StudyRotateShareTokenResponse = {
12038
13317
  "startsPerMinute"?: number;
12039
13318
  } | null;
12040
13319
  "relaunchable"?: boolean;
13320
+ "archivedAt"?: string | null;
13321
+ "archivedBy"?: string | null;
12041
13322
  "arms"?: Array<{
12042
13323
  "key": string;
12043
13324
  "position": number;
@@ -12076,6 +13357,29 @@ export type StudyRotateShareTokenResponse = {
12076
13357
  "cellPlan"?: "crossed" | "per_setup";
12077
13358
  "skillDisclosure"?: "withheld" | "named";
12078
13359
  "modeOptions"?: Record<string, unknown>;
13360
+ "screens"?: Array<"test_awareness">;
13361
+ "runCounting"?: {
13362
+ "policy": "hold_back_screened";
13363
+ "screen": "test_awareness";
13364
+ "levels": Array<"aware" | "influenced">;
13365
+ } | null;
13366
+ "seriesKey"?: string | null;
13367
+ "ref"?: {
13368
+ "commit"?: string;
13369
+ "branch"?: string;
13370
+ "pr"?: number;
13371
+ "url"?: string;
13372
+ } | null;
13373
+ "gradingRevision"?: number;
13374
+ "gradingRevisedAt"?: string | null;
13375
+ "rehearsal"?: true;
13376
+ "rehearsalMode"?: string;
13377
+ "behaviors"?: Array<{
13378
+ "key": string;
13379
+ "name": string;
13380
+ "question": string;
13381
+ "version": number;
13382
+ }>;
12079
13383
  };
12080
13384
  "shareUrl": string | null;
12081
13385
  "shareLinkKept": boolean;
@@ -12302,6 +13606,8 @@ export type StudyUnpublishResponse = {
12302
13606
  "startsPerMinute"?: number;
12303
13607
  } | null;
12304
13608
  "relaunchable"?: boolean;
13609
+ "archivedAt"?: string | null;
13610
+ "archivedBy"?: string | null;
12305
13611
  "arms"?: Array<{
12306
13612
  "key": string;
12307
13613
  "position": number;
@@ -12340,8 +13646,93 @@ export type StudyUnpublishResponse = {
12340
13646
  "cellPlan"?: "crossed" | "per_setup";
12341
13647
  "skillDisclosure"?: "withheld" | "named";
12342
13648
  "modeOptions"?: Record<string, unknown>;
13649
+ "screens"?: Array<"test_awareness">;
13650
+ "runCounting"?: {
13651
+ "policy": "hold_back_screened";
13652
+ "screen": "test_awareness";
13653
+ "levels": Array<"aware" | "influenced">;
13654
+ } | null;
13655
+ "seriesKey"?: string | null;
13656
+ "ref"?: {
13657
+ "commit"?: string;
13658
+ "branch"?: string;
13659
+ "pr"?: number;
13660
+ "url"?: string;
13661
+ } | null;
13662
+ "gradingRevision"?: number;
13663
+ "gradingRevisedAt"?: string | null;
13664
+ "rehearsal"?: true;
13665
+ "rehearsalMode"?: string;
13666
+ "behaviors"?: Array<{
13667
+ "key": string;
13668
+ "name": string;
13669
+ "question": string;
13670
+ "version": number;
13671
+ }>;
13672
+ };
13673
+ };
13674
+ export type StudyViewsResponse = {
13675
+ "views": Array<{
13676
+ "id": string;
13677
+ "name": string;
13678
+ "query": unknown;
13679
+ "createdBy": string | null;
13680
+ "createdAt": string;
13681
+ }>;
13682
+ };
13683
+ export type StudySaveViewRequest = {
13684
+ "name": string;
13685
+ "query": {
13686
+ "metrics": Array<string>;
13687
+ "groupBy"?: Array<string>;
13688
+ "filters"?: Array<{
13689
+ "field": string;
13690
+ "in": Array<string>;
13691
+ }>;
13692
+ "limit"?: number;
13693
+ };
13694
+ };
13695
+ export type StudySaveViewResponse = {
13696
+ "view": {
13697
+ "id": string;
13698
+ "name": string;
13699
+ "query": unknown;
13700
+ "createdBy": string | null;
13701
+ "createdAt": string;
13702
+ };
13703
+ };
13704
+ export type StudyRunViewResponse = {
13705
+ "view": {
13706
+ "id": string;
13707
+ "name": string;
13708
+ "query": unknown;
13709
+ "createdBy": string | null;
13710
+ "createdAt": string;
13711
+ };
13712
+ "result": {
13713
+ "basis": "runs";
13714
+ "matchedRuns": number;
13715
+ "groupBy": Array<{
13716
+ "id": string;
13717
+ "label": string;
13718
+ }>;
13719
+ "metrics": Array<{
13720
+ "id": string;
13721
+ "label": string;
13722
+ "unit": "count" | "rate" | "points" | "usd" | "tokens" | "ms";
13723
+ "fold": string;
13724
+ }>;
13725
+ "rows": Array<{
13726
+ "key": Record<string, unknown>;
13727
+ "runs": number;
13728
+ "cells": Record<string, unknown>;
13729
+ }>;
13730
+ "truncated": boolean;
12343
13731
  };
12344
13732
  };
13733
+ export type StudyDeleteViewResponse = {
13734
+ "deleted": true;
13735
+ };
12345
13736
  export type StudySetVisibilityRequest = {
12346
13737
  "visibility": "private" | "org" | "link" | "public";
12347
13738
  };
@@ -12480,6 +13871,8 @@ export type StudySetVisibilityResponse = {
12480
13871
  "startsPerMinute"?: number;
12481
13872
  } | null;
12482
13873
  "relaunchable"?: boolean;
13874
+ "archivedAt"?: string | null;
13875
+ "archivedBy"?: string | null;
12483
13876
  "arms"?: Array<{
12484
13877
  "key": string;
12485
13878
  "position": number;
@@ -12518,6 +13911,29 @@ export type StudySetVisibilityResponse = {
12518
13911
  "cellPlan"?: "crossed" | "per_setup";
12519
13912
  "skillDisclosure"?: "withheld" | "named";
12520
13913
  "modeOptions"?: Record<string, unknown>;
13914
+ "screens"?: Array<"test_awareness">;
13915
+ "runCounting"?: {
13916
+ "policy": "hold_back_screened";
13917
+ "screen": "test_awareness";
13918
+ "levels": Array<"aware" | "influenced">;
13919
+ } | null;
13920
+ "seriesKey"?: string | null;
13921
+ "ref"?: {
13922
+ "commit"?: string;
13923
+ "branch"?: string;
13924
+ "pr"?: number;
13925
+ "url"?: string;
13926
+ } | null;
13927
+ "gradingRevision"?: number;
13928
+ "gradingRevisedAt"?: string | null;
13929
+ "rehearsal"?: true;
13930
+ "rehearsalMode"?: string;
13931
+ "behaviors"?: Array<{
13932
+ "key": string;
13933
+ "name": string;
13934
+ "question": string;
13935
+ "version": number;
13936
+ }>;
12521
13937
  };
12522
13938
  "shareUrl": string | null;
12523
13939
  "shareLinkKept": boolean;