@ora-ai/cli 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/README.md +93 -1
  2. package/dist/auth-commands.d.ts.map +1 -1
  3. package/dist/auth-commands.js +12 -1
  4. package/dist/auth-commands.js.map +1 -1
  5. package/dist/core/auth.d.ts +24 -13
  6. package/dist/core/auth.d.ts.map +1 -1
  7. package/dist/core/auth.js +26 -15
  8. package/dist/core/auth.js.map +1 -1
  9. package/dist/core/credential-sources.d.ts +45 -0
  10. package/dist/core/credential-sources.d.ts.map +1 -0
  11. package/dist/core/credential-sources.js +65 -0
  12. package/dist/core/credential-sources.js.map +1 -0
  13. package/dist/core/github-actions-sign-in.d.ts +33 -0
  14. package/dist/core/github-actions-sign-in.d.ts.map +1 -0
  15. package/dist/core/github-actions-sign-in.js +54 -0
  16. package/dist/core/github-actions-sign-in.js.map +1 -0
  17. package/dist/core/output.js +14 -1
  18. package/dist/core/output.js.map +1 -1
  19. package/dist/core/run-operation.d.ts +11 -4
  20. package/dist/core/run-operation.d.ts.map +1 -1
  21. package/dist/core/run-operation.js +14 -9
  22. package/dist/core/run-operation.js.map +1 -1
  23. package/dist/core/sign-in.d.ts +34 -0
  24. package/dist/core/sign-in.d.ts.map +1 -0
  25. package/dist/core/sign-in.js +21 -0
  26. package/dist/core/sign-in.js.map +1 -0
  27. package/dist/generated/operations.d.ts +1 -1
  28. package/dist/generated/operations.d.ts.map +1 -1
  29. package/dist/generated/operations.js +1236 -70
  30. package/dist/generated/operations.js.map +1 -1
  31. package/dist/generated/types.d.ts +1579 -112
  32. package/dist/generated/types.d.ts.map +1 -1
  33. package/dist/generated/types.js +1 -1
  34. package/dist/mcp/mcp-call.d.ts +25 -0
  35. package/dist/mcp/mcp-call.d.ts.map +1 -0
  36. package/dist/mcp/mcp-call.js +44 -0
  37. package/dist/mcp/mcp-call.js.map +1 -0
  38. package/dist/mcp/mcp-command.d.ts +26 -0
  39. package/dist/mcp/mcp-command.d.ts.map +1 -0
  40. package/dist/mcp/mcp-command.js +91 -0
  41. package/dist/mcp/mcp-command.js.map +1 -0
  42. package/dist/mcp/mcp-install.d.ts +50 -0
  43. package/dist/mcp/mcp-install.d.ts.map +1 -0
  44. package/dist/mcp/mcp-install.js +71 -0
  45. package/dist/mcp/mcp-install.js.map +1 -0
  46. package/dist/mcp/mcp-protocol.d.ts +69 -0
  47. package/dist/mcp/mcp-protocol.d.ts.map +1 -0
  48. package/dist/mcp/mcp-protocol.js +96 -0
  49. package/dist/mcp/mcp-protocol.js.map +1 -0
  50. package/dist/mcp/mcp-stdio.d.ts +17 -0
  51. package/dist/mcp/mcp-stdio.d.ts.map +1 -0
  52. package/dist/mcp/mcp-stdio.js +39 -0
  53. package/dist/mcp/mcp-stdio.js.map +1 -0
  54. package/dist/mcp/mcp-tools.d.ts +81 -0
  55. package/dist/mcp/mcp-tools.d.ts.map +1 -0
  56. package/dist/mcp/mcp-tools.js +134 -0
  57. package/dist/mcp/mcp-tools.js.map +1 -0
  58. package/dist/program.d.ts +4 -9
  59. package/dist/program.d.ts.map +1 -1
  60. package/dist/program.js +9 -3
  61. package/dist/program.js.map +1 -1
  62. package/dist/render/index.js +6 -0
  63. package/dist/render/index.js.map +1 -1
  64. package/dist/render/study/counting-views.d.ts +46 -0
  65. package/dist/render/study/counting-views.d.ts.map +1 -0
  66. package/dist/render/study/counting-views.js +49 -0
  67. package/dist/render/study/counting-views.js.map +1 -0
  68. package/dist/render/study/gate-views.d.ts +56 -0
  69. package/dist/render/study/gate-views.d.ts.map +1 -0
  70. package/dist/render/study/gate-views.js +64 -0
  71. package/dist/render/study/gate-views.js.map +1 -0
  72. package/dist/render/study/renderers.d.ts +8 -0
  73. package/dist/render/study/renderers.d.ts.map +1 -0
  74. package/dist/render/study/renderers.js +31 -0
  75. package/dist/render/study/renderers.js.map +1 -0
  76. package/dist/render/study/series-views.d.ts +65 -0
  77. package/dist/render/study/series-views.d.ts.map +1 -0
  78. package/dist/render/study/series-views.js +116 -0
  79. package/dist/render/study/series-views.js.map +1 -0
  80. package/dist/render/study/study-format.d.ts +24 -0
  81. package/dist/render/study/study-format.d.ts.map +1 -0
  82. package/dist/render/study/study-format.js +42 -0
  83. package/dist/render/study/study-format.js.map +1 -0
  84. package/dist/study-ci/ci-command.d.ts +31 -0
  85. package/dist/study-ci/ci-command.d.ts.map +1 -0
  86. package/dist/study-ci/ci-command.js +134 -0
  87. package/dist/study-ci/ci-command.js.map +1 -0
  88. package/dist/study-ci/ci-rehearse-command.d.ts +11 -0
  89. package/dist/study-ci/ci-rehearse-command.d.ts.map +1 -0
  90. package/dist/study-ci/ci-rehearse-command.js +83 -0
  91. package/dist/study-ci/ci-rehearse-command.js.map +1 -0
  92. package/dist/study-ci/ci-rehearse.d.ts +62 -0
  93. package/dist/study-ci/ci-rehearse.d.ts.map +1 -0
  94. package/dist/study-ci/ci-rehearse.js +49 -0
  95. package/dist/study-ci/ci-rehearse.js.map +1 -0
  96. package/dist/study-ci/ci-run.d.ts +87 -0
  97. package/dist/study-ci/ci-run.d.ts.map +1 -0
  98. package/dist/study-ci/ci-run.js +131 -0
  99. package/dist/study-ci/ci-run.js.map +1 -0
  100. package/dist/study-ci/gate-report.d.ts +64 -0
  101. package/dist/study-ci/gate-report.d.ts.map +1 -0
  102. package/dist/study-ci/gate-report.js +80 -0
  103. package/dist/study-ci/gate-report.js.map +1 -0
  104. package/dist/study-ci/params.d.ts +8 -0
  105. package/dist/study-ci/params.d.ts.map +1 -0
  106. package/dist/study-ci/params.js +72 -0
  107. package/dist/study-ci/params.js.map +1 -0
  108. package/package.json +1 -1
@@ -3362,6 +3362,23 @@ export type ExperimentRunsListResponse = {
3362
3362
  "output_tokens"?: number | null;
3363
3363
  "created_at": string;
3364
3364
  "finished_at"?: string | null;
3365
+ "phases"?: {
3366
+ "phases": Array<{
3367
+ "id": "waiting" | "boot" | "installations" | "startup" | "run" | "verification" | "teardown";
3368
+ "ms": number | null;
3369
+ "state": "measured" | "unmeasured" | "not_in_run";
3370
+ }>;
3371
+ "runTimeMs": number | null;
3372
+ "totalMs": number | null;
3373
+ "steps": Array<{
3374
+ "name": string;
3375
+ "kind": string;
3376
+ "outcome": string;
3377
+ "startedAt": string | null;
3378
+ "finishedAt": string | null;
3379
+ }>;
3380
+ "modelCalls": number | null;
3381
+ } | null;
3365
3382
  "insight"?: {
3366
3383
  "visibility_score": number | null;
3367
3384
  "intent_satisfied": boolean | null;
@@ -5592,6 +5609,23 @@ export type StudioRunsListResponse = {
5592
5609
  "output_tokens"?: number | null;
5593
5610
  "created_at": string;
5594
5611
  "finished_at"?: string | null;
5612
+ "phases"?: {
5613
+ "phases": Array<{
5614
+ "id": "waiting" | "boot" | "installations" | "startup" | "run" | "verification" | "teardown";
5615
+ "ms": number | null;
5616
+ "state": "measured" | "unmeasured" | "not_in_run";
5617
+ }>;
5618
+ "runTimeMs": number | null;
5619
+ "totalMs": number | null;
5620
+ "steps": Array<{
5621
+ "name": string;
5622
+ "kind": string;
5623
+ "outcome": string;
5624
+ "startedAt": string | null;
5625
+ "finishedAt": string | null;
5626
+ }>;
5627
+ "modelCalls": number | null;
5628
+ } | null;
5595
5629
  "insight"?: {
5596
5630
  "visibility_score": number | null;
5597
5631
  "intent_satisfied": boolean | null;
@@ -6828,6 +6862,23 @@ export type StudyRunsListResponse = {
6828
6862
  "output_tokens"?: number | null;
6829
6863
  "created_at": string;
6830
6864
  "finished_at"?: string | null;
6865
+ "phases"?: {
6866
+ "phases": Array<{
6867
+ "id": "waiting" | "boot" | "installations" | "startup" | "run" | "verification" | "teardown";
6868
+ "ms": number | null;
6869
+ "state": "measured" | "unmeasured" | "not_in_run";
6870
+ }>;
6871
+ "runTimeMs": number | null;
6872
+ "totalMs": number | null;
6873
+ "steps": Array<{
6874
+ "name": string;
6875
+ "kind": string;
6876
+ "outcome": string;
6877
+ "startedAt": string | null;
6878
+ "finishedAt": string | null;
6879
+ }>;
6880
+ "modelCalls": number | null;
6881
+ } | null;
6831
6882
  "insight"?: {
6832
6883
  "visibility_score": number | null;
6833
6884
  "intent_satisfied": boolean | null;
@@ -7123,6 +7174,8 @@ export type StudyListResponse = {
7123
7174
  "startsPerMinute"?: number;
7124
7175
  } | null;
7125
7176
  "relaunchable"?: boolean;
7177
+ "archivedAt"?: string | null;
7178
+ "archivedBy"?: string | null;
7126
7179
  "arms"?: Array<{
7127
7180
  "key": string;
7128
7181
  "position": number;
@@ -7161,6 +7214,29 @@ export type StudyListResponse = {
7161
7214
  "cellPlan"?: "crossed" | "per_setup";
7162
7215
  "skillDisclosure"?: "withheld" | "named";
7163
7216
  "modeOptions"?: Record<string, unknown>;
7217
+ "screens"?: Array<"test_awareness">;
7218
+ "runCounting"?: {
7219
+ "policy": "hold_back_screened";
7220
+ "screen": "test_awareness";
7221
+ "levels": Array<"aware" | "influenced">;
7222
+ } | null;
7223
+ "seriesKey"?: string | null;
7224
+ "ref"?: {
7225
+ "commit"?: string;
7226
+ "branch"?: string;
7227
+ "pr"?: number;
7228
+ "url"?: string;
7229
+ } | null;
7230
+ "gradingRevision"?: number;
7231
+ "gradingRevisedAt"?: string | null;
7232
+ "rehearsal"?: true;
7233
+ "rehearsalMode"?: string;
7234
+ "behaviors"?: Array<{
7235
+ "key": string;
7236
+ "name": string;
7237
+ "question": string;
7238
+ "version": number;
7239
+ }>;
7164
7240
  }>;
7165
7241
  };
7166
7242
  export type StudyCreateRequest = {
@@ -7306,6 +7382,28 @@ export type StudyCreateRequest = {
7306
7382
  "maxInFlight"?: number;
7307
7383
  "startsPerMinute"?: number;
7308
7384
  } | null;
7385
+ "screens"?: Array<"test_awareness">;
7386
+ "series"?: string;
7387
+ "ref"?: {
7388
+ "commit"?: string;
7389
+ "branch"?: string;
7390
+ "pr"?: number;
7391
+ "url"?: string;
7392
+ };
7393
+ "carry"?: {
7394
+ "from": "series";
7395
+ "maxAgeDays"?: number;
7396
+ };
7397
+ "runCounting"?: {
7398
+ "policy": "hold_back_screened";
7399
+ "screen": "test_awareness";
7400
+ "levels": Array<"aware" | "influenced">;
7401
+ } | null;
7402
+ "behaviors"?: Array<{
7403
+ "key": string;
7404
+ "name": string;
7405
+ "question": string;
7406
+ }>;
7309
7407
  "matrix": Array<{
7310
7408
  "harness": string;
7311
7409
  "model": string;
@@ -7456,6 +7554,8 @@ export type StudyCreateResponse = {
7456
7554
  "startsPerMinute"?: number;
7457
7555
  } | null;
7458
7556
  "relaunchable"?: boolean;
7557
+ "archivedAt"?: string | null;
7558
+ "archivedBy"?: string | null;
7459
7559
  "arms"?: Array<{
7460
7560
  "key": string;
7461
7561
  "position": number;
@@ -7494,6 +7594,29 @@ export type StudyCreateResponse = {
7494
7594
  "cellPlan"?: "crossed" | "per_setup";
7495
7595
  "skillDisclosure"?: "withheld" | "named";
7496
7596
  "modeOptions"?: Record<string, unknown>;
7597
+ "screens"?: Array<"test_awareness">;
7598
+ "runCounting"?: {
7599
+ "policy": "hold_back_screened";
7600
+ "screen": "test_awareness";
7601
+ "levels": Array<"aware" | "influenced">;
7602
+ } | null;
7603
+ "seriesKey"?: string | null;
7604
+ "ref"?: {
7605
+ "commit"?: string;
7606
+ "branch"?: string;
7607
+ "pr"?: number;
7608
+ "url"?: string;
7609
+ } | null;
7610
+ "gradingRevision"?: number;
7611
+ "gradingRevisedAt"?: string | null;
7612
+ "rehearsal"?: true;
7613
+ "rehearsalMode"?: string;
7614
+ "behaviors"?: Array<{
7615
+ "key": string;
7616
+ "name": string;
7617
+ "question": string;
7618
+ "version": number;
7619
+ }>;
7497
7620
  };
7498
7621
  "streamUrl": string;
7499
7622
  "warnings"?: Array<{
@@ -7514,6 +7637,10 @@ export type StudyComparisonModesListResponse = {
7514
7637
  "id": "crossed" | "per_setup";
7515
7638
  "enabled": boolean;
7516
7639
  }>;
7640
+ "customBehaviors": {
7641
+ "enabled": boolean;
7642
+ "max": number;
7643
+ };
7517
7644
  };
7518
7645
  export type StudyConvertWebmcpSuiteRequest = {
7519
7646
  "suite": unknown;
@@ -7670,6 +7797,28 @@ export type StudyPreflightRequest = {
7670
7797
  "maxInFlight"?: number;
7671
7798
  "startsPerMinute"?: number;
7672
7799
  } | null;
7800
+ "screens"?: Array<"test_awareness">;
7801
+ "series"?: string;
7802
+ "ref"?: {
7803
+ "commit"?: string;
7804
+ "branch"?: string;
7805
+ "pr"?: number;
7806
+ "url"?: string;
7807
+ };
7808
+ "carry"?: {
7809
+ "from": "series";
7810
+ "maxAgeDays"?: number;
7811
+ };
7812
+ "runCounting"?: {
7813
+ "policy": "hold_back_screened";
7814
+ "screen": "test_awareness";
7815
+ "levels": Array<"aware" | "influenced">;
7816
+ } | null;
7817
+ "behaviors"?: Array<{
7818
+ "key": string;
7819
+ "name": string;
7820
+ "question": string;
7821
+ }>;
7673
7822
  "matrix": Array<{
7674
7823
  "harness": string;
7675
7824
  "model": string;
@@ -7692,6 +7841,14 @@ export type StudyPreflightResponse = {
7692
7841
  "label": string;
7693
7842
  "plannedRunCount": number;
7694
7843
  "plannedJudgeComparisons": number;
7844
+ "screens"?: {
7845
+ "ids": Array<string>;
7846
+ "plannedReads": number;
7847
+ };
7848
+ "behaviors"?: {
7849
+ "keys": Array<string>;
7850
+ "plannedReads": number;
7851
+ };
7695
7852
  "runs": {
7696
7853
  "tasks": number;
7697
7854
  "variants": number;
@@ -7762,53 +7919,45 @@ export type StudyPreflightResponse = {
7762
7919
  "allowed": boolean;
7763
7920
  } | null;
7764
7921
  };
7765
- export type StudyGetResponse = {
7766
- "study": {
7767
- "id": string;
7768
- "executiveSummary"?: string | null;
7769
- "executiveSummaryDraft"?: string | null;
7770
- "executiveSummaryRequest"?: "live" | "draft" | null;
7771
- "executiveSummaryRequestedAt"?: string | null;
7772
- "executiveSummaryUnavailableAt"?: string | null;
7773
- "plannedRunCount": number;
7774
- "organizationId": string;
7775
- "createdBy": string | null;
7776
- "name": string;
7777
- "slug": string;
7778
- "description": string | null;
7779
- "skillRef": string | null;
7780
- "skillSlug": string | null;
7781
- "skillName": string | null;
7782
- "skillPromise": string;
7783
- "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
7784
- "subjectName": string | null;
7785
- "subjectLabel": string;
7786
- "armConfig": {
7787
- "baseline"?: {
7788
- "tools"?: "default" | "none" | {
7789
- "tools": Array<string>;
7790
- };
7791
- "addons"?: Record<string, unknown>;
7922
+ export type StudyRehearseRequest = {
7923
+ "name": string;
7924
+ "description"?: string;
7925
+ "subjectKind"?: "skill" | "tools" | "addon" | "custom" | "benchmark";
7926
+ "skillRef"?: string;
7927
+ "definitionVersionId"?: string;
7928
+ "subjectName"?: string;
7929
+ "armConfig"?: {
7930
+ "baseline"?: {
7931
+ "tools"?: "default" | "none" | {
7932
+ "tools": Array<string>;
7792
7933
  };
7793
- "treatment"?: {
7794
- "tools"?: "default" | "none" | {
7795
- "tools": Array<string>;
7796
- };
7797
- "addons"?: Record<string, unknown>;
7934
+ "addons"?: Record<string, unknown>;
7935
+ };
7936
+ "treatment"?: {
7937
+ "tools"?: "default" | "none" | {
7938
+ "tools": Array<string>;
7798
7939
  };
7940
+ "addons"?: Record<string, unknown>;
7799
7941
  };
7800
- "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
7801
- "subjectBackend": "model" | "harness" | null;
7802
- "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
7803
- "importedFrom"?: string;
7804
- "baselineLabel": string | null;
7805
- "treatmentLabel": string | null;
7806
- "promiseMetrics": Array<{
7807
- "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
7808
- "direction": "down" | "up";
7809
- "label": string;
7810
- }>;
7811
- "matrix": Array<{
7942
+ };
7943
+ "lenses"?: Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
7944
+ "judgeSupplementary"?: boolean;
7945
+ "proposeExpectations"?: boolean;
7946
+ "comparisonMode"?: "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
7947
+ "subjectBackend"?: "model" | "harness";
7948
+ "baselineLabel"?: string;
7949
+ "treatmentLabel"?: string;
7950
+ "arms"?: Array<{
7951
+ "key"?: string;
7952
+ "label": string;
7953
+ "role"?: "reference" | "peer";
7954
+ "config": {
7955
+ "tools"?: "default" | "none" | {
7956
+ "tools": Array<string>;
7957
+ };
7958
+ "addons"?: Record<string, unknown>;
7959
+ };
7960
+ "cells"?: Array<{
7812
7961
  "harness": string;
7813
7962
  "model": string;
7814
7963
  "provider"?: string;
@@ -7821,74 +7970,324 @@ export type StudyGetResponse = {
7821
7970
  "toolSearch"?: "on" | "off" | "directory";
7822
7971
  };
7823
7972
  }>;
7824
- "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
7825
- "judgeSupplementary"?: boolean;
7826
- "taskCount": number;
7827
- "variantsPerTask": number;
7828
- "repeats": number;
7829
- "subset": {
7830
- "count": number;
7831
- "seed": string;
7832
- "of": number;
7833
- } | null;
7834
- "status": string;
7835
- "error": string | null;
7836
- "preparation"?: {
7837
- "since": string;
7838
- "progressAt"?: string;
7839
- "launchedBy": {
7840
- "userId"?: string;
7841
- "role"?: string;
7842
- };
7843
- "waitingOn": Array<{
7844
- "imageKey": string;
7845
- "buildId": string;
7846
- "stage"?: "image" | "snapshot";
7847
- "retry"?: {
7848
- "failedAttempts": number;
7849
- "maxAttempts": number;
7850
- "notBefore": string;
7851
- "lastFailure": string;
7852
- };
7853
- }>;
7854
- } | null;
7855
- "visibility": "private" | "org" | "link" | "public";
7856
- "visibilityChangedAt"?: string | null;
7857
- "visibilityChangedBy"?: string | null;
7858
- "shareToken"?: string | null;
7859
- "lowSampleOverride": boolean;
7860
- "publishedAt": string | null;
7861
- "verdict": unknown;
7862
- "definitionVersionId": string | null;
7863
- "runnerKind": "ora-runs" | "batch-job";
7864
- "createdAt": string;
7865
- "updatedAt": string;
7866
- "liveUntil": string | null;
7867
- "launch"?: {
7868
- "attempt": number;
7869
- "state": "fanning_out" | "deferred" | "settled";
7870
- "startedAt": string;
7871
- "progressAt": string;
7872
- "planned"?: number;
7873
- "created"?: number;
7874
- "missing"?: {
7875
- "retryable": number;
7876
- "refused": number;
7877
- };
7878
- "notes"?: Array<string>;
7879
- "settledAt"?: string;
7880
- "pacing"?: {
7881
- "waiting": number;
7882
- "nextUnit"?: number;
7883
- "nextWaveAt"?: string;
7884
- "emptyWaves"?: number;
7885
- "stopped"?: string;
7973
+ }>;
7974
+ "contrasts"?: "vs_reference" | "all_pairs";
7975
+ "modeOptions"?: Record<string, unknown>;
7976
+ "taskCount"?: number;
7977
+ "variantsPerTask"?: number;
7978
+ "repeats"?: number;
7979
+ "tasks"?: Array<{
7980
+ "prompt": string;
7981
+ "rationale"?: string;
7982
+ "category"?: string;
7983
+ "requiresWeb"?: boolean;
7984
+ "rubric"?: Array<string>;
7985
+ "variants"?: Array<string>;
7986
+ "expectations"?: Array<{
7987
+ "label"?: string;
7988
+ "arms"?: Array<"baseline" | "skill" | string>;
7989
+ "kind": "mcp_tool_called";
7990
+ "server"?: string;
7991
+ "tool": string;
7992
+ "arguments"?: Record<string, unknown>;
7993
+ "minCalls"?: number;
7994
+ } | {
7995
+ "label"?: string;
7996
+ "arms"?: Array<"baseline" | "skill" | string>;
7997
+ "kind": "tool_not_called";
7998
+ "tool"?: string;
7999
+ "mcp"?: {
8000
+ "server"?: string;
8001
+ "tool": string;
7886
8002
  };
7887
- "recovery"?: {
7888
- "state": "retrying" | "needs_attention";
7889
- "attempts": number;
7890
- "lastAttemptAt"?: string;
7891
- "nextAttemptAt"?: string;
8003
+ "pattern"?: string;
8004
+ } | {
8005
+ "label"?: string;
8006
+ "arms"?: Array<"baseline" | "skill" | string>;
8007
+ "kind": "command_ran";
8008
+ "pattern": string;
8009
+ } | {
8010
+ "label"?: string;
8011
+ "arms"?: Array<"baseline" | "skill" | string>;
8012
+ "kind": "skill_loaded";
8013
+ "slug": string;
8014
+ } | {
8015
+ "label"?: string;
8016
+ "arms"?: Array<"baseline" | "skill" | string>;
8017
+ "kind": "file_operation";
8018
+ "op"?: "read" | "write" | "edit";
8019
+ "path": unknown;
8020
+ } | {
8021
+ "label"?: string;
8022
+ "arms"?: Array<"baseline" | "skill" | string>;
8023
+ "kind": "web_fetched";
8024
+ "urlHost"?: string;
8025
+ "urlPattern"?: string;
8026
+ } | {
8027
+ "label"?: string;
8028
+ "arms"?: Array<"baseline" | "skill" | string>;
8029
+ "kind": "response_contains";
8030
+ "text": string;
8031
+ "normalize"?: "prose";
8032
+ } | {
8033
+ "label"?: string;
8034
+ "arms"?: Array<"baseline" | "skill" | string>;
8035
+ "kind": "response_matches";
8036
+ "pattern": string;
8037
+ "ignoreCase"?: boolean;
8038
+ } | {
8039
+ "label"?: string;
8040
+ "arms"?: Array<"baseline" | "skill" | string>;
8041
+ "kind": "token_budget";
8042
+ "maxOutputTokens"?: number;
8043
+ "maxTotalTokens"?: number;
8044
+ } | {
8045
+ "label"?: string;
8046
+ "arms"?: Array<"baseline" | "skill" | string>;
8047
+ "kind": "sequence";
8048
+ "of": Array<unknown>;
8049
+ } | {
8050
+ "label"?: string;
8051
+ "arms"?: Array<"baseline" | "skill" | string>;
8052
+ "kind": "any_of";
8053
+ "of": Array<unknown>;
8054
+ }>;
8055
+ "private"?: boolean;
8056
+ }>;
8057
+ "subset"?: {
8058
+ "count": number;
8059
+ "seed"?: string;
8060
+ };
8061
+ "pacing"?: {
8062
+ "maxInFlight"?: number;
8063
+ "startsPerMinute"?: number;
8064
+ } | null;
8065
+ "screens"?: Array<"test_awareness">;
8066
+ "series"?: string;
8067
+ "ref"?: {
8068
+ "commit"?: string;
8069
+ "branch"?: string;
8070
+ "pr"?: number;
8071
+ "url"?: string;
8072
+ };
8073
+ "carry"?: {
8074
+ "from": "series";
8075
+ "maxAgeDays"?: number;
8076
+ };
8077
+ "runCounting"?: {
8078
+ "policy": "hold_back_screened";
8079
+ "screen": "test_awareness";
8080
+ "levels": Array<"aware" | "influenced">;
8081
+ } | null;
8082
+ "behaviors"?: Array<{
8083
+ "key": string;
8084
+ "name": string;
8085
+ "question": string;
8086
+ }>;
8087
+ "matrix": Array<{
8088
+ "harness": string;
8089
+ "model": string;
8090
+ "provider"?: string;
8091
+ "params"?: {
8092
+ "temperature"?: number;
8093
+ "maxOutputTokens"?: number;
8094
+ "reasoningEffort"?: "low" | "medium" | "high";
8095
+ "thinkingBudgetTokens"?: number;
8096
+ "codeMode"?: "auto" | "on" | "off";
8097
+ "toolSearch"?: "on" | "off" | "directory";
8098
+ };
8099
+ }>;
8100
+ "promise"?: string;
8101
+ };
8102
+ export type StudyRehearseResponse = {
8103
+ "rehearsal": {
8104
+ "studyId": string;
8105
+ "name": string;
8106
+ "status": string;
8107
+ "tasks": number;
8108
+ "mode": string;
8109
+ };
8110
+ };
8111
+ export type StudySeriesResponse = {
8112
+ "series": Array<{
8113
+ "key": string;
8114
+ "studies": number;
8115
+ "newestAt": string;
8116
+ }>;
8117
+ "capped": boolean;
8118
+ };
8119
+ export type StudySeriesTrendResponse = {
8120
+ "series": string;
8121
+ "metric": {
8122
+ "key": string;
8123
+ "label": string;
8124
+ "family": string;
8125
+ };
8126
+ "countingPolicy": string;
8127
+ "studies": Array<{
8128
+ "studyId": string;
8129
+ "name": string;
8130
+ "createdAt": string;
8131
+ "ref": {
8132
+ "commit"?: string;
8133
+ "branch"?: string;
8134
+ "pr"?: number;
8135
+ "url"?: string;
8136
+ } | null;
8137
+ "state": "settled" | "pending";
8138
+ "owed": Array<string>;
8139
+ "segment": number | null;
8140
+ "heldBackRuns"?: number;
8141
+ "groups": Array<{
8142
+ "key": string;
8143
+ "label": string;
8144
+ "value": number | null;
8145
+ "interval": Array<unknown> | null;
8146
+ "tasks": number;
8147
+ "runs": number;
8148
+ "change": {
8149
+ "effect": number;
8150
+ "unit": string;
8151
+ "interval": Array<unknown>;
8152
+ "tasks": number;
8153
+ } | null;
8154
+ }>;
8155
+ }>;
8156
+ "segments": Array<{
8157
+ "index": number;
8158
+ "comparabilityKey": string;
8159
+ "studyIds": Array<string>;
8160
+ "differsFromPrevious": Array<"corpus" | "lenses" | "graders" | "classifiers" | "countingPolicy" | "agents">;
8161
+ }>;
8162
+ "truncated": boolean;
8163
+ };
8164
+ export type StudyGetResponse = {
8165
+ "study": {
8166
+ "id": string;
8167
+ "executiveSummary"?: string | null;
8168
+ "executiveSummaryDraft"?: string | null;
8169
+ "executiveSummaryRequest"?: "live" | "draft" | null;
8170
+ "executiveSummaryRequestedAt"?: string | null;
8171
+ "executiveSummaryUnavailableAt"?: string | null;
8172
+ "plannedRunCount": number;
8173
+ "organizationId": string;
8174
+ "createdBy": string | null;
8175
+ "name": string;
8176
+ "slug": string;
8177
+ "description": string | null;
8178
+ "skillRef": string | null;
8179
+ "skillSlug": string | null;
8180
+ "skillName": string | null;
8181
+ "skillPromise": string;
8182
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
8183
+ "subjectName": string | null;
8184
+ "subjectLabel": string;
8185
+ "armConfig": {
8186
+ "baseline"?: {
8187
+ "tools"?: "default" | "none" | {
8188
+ "tools": Array<string>;
8189
+ };
8190
+ "addons"?: Record<string, unknown>;
8191
+ };
8192
+ "treatment"?: {
8193
+ "tools"?: "default" | "none" | {
8194
+ "tools": Array<string>;
8195
+ };
8196
+ "addons"?: Record<string, unknown>;
8197
+ };
8198
+ };
8199
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
8200
+ "subjectBackend": "model" | "harness" | null;
8201
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
8202
+ "importedFrom"?: string;
8203
+ "baselineLabel": string | null;
8204
+ "treatmentLabel": string | null;
8205
+ "promiseMetrics": Array<{
8206
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
8207
+ "direction": "down" | "up";
8208
+ "label": string;
8209
+ }>;
8210
+ "matrix": Array<{
8211
+ "harness": string;
8212
+ "model": string;
8213
+ "provider"?: string;
8214
+ "params"?: {
8215
+ "temperature"?: number;
8216
+ "maxOutputTokens"?: number;
8217
+ "reasoningEffort"?: "low" | "medium" | "high";
8218
+ "thinkingBudgetTokens"?: number;
8219
+ "codeMode"?: "auto" | "on" | "off";
8220
+ "toolSearch"?: "on" | "off" | "directory";
8221
+ };
8222
+ }>;
8223
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
8224
+ "judgeSupplementary"?: boolean;
8225
+ "taskCount": number;
8226
+ "variantsPerTask": number;
8227
+ "repeats": number;
8228
+ "subset": {
8229
+ "count": number;
8230
+ "seed": string;
8231
+ "of": number;
8232
+ } | null;
8233
+ "status": string;
8234
+ "error": string | null;
8235
+ "preparation"?: {
8236
+ "since": string;
8237
+ "progressAt"?: string;
8238
+ "launchedBy": {
8239
+ "userId"?: string;
8240
+ "role"?: string;
8241
+ };
8242
+ "waitingOn": Array<{
8243
+ "imageKey": string;
8244
+ "buildId": string;
8245
+ "stage"?: "image" | "snapshot";
8246
+ "retry"?: {
8247
+ "failedAttempts": number;
8248
+ "maxAttempts": number;
8249
+ "notBefore": string;
8250
+ "lastFailure": string;
8251
+ };
8252
+ }>;
8253
+ } | null;
8254
+ "visibility": "private" | "org" | "link" | "public";
8255
+ "visibilityChangedAt"?: string | null;
8256
+ "visibilityChangedBy"?: string | null;
8257
+ "shareToken"?: string | null;
8258
+ "lowSampleOverride": boolean;
8259
+ "publishedAt": string | null;
8260
+ "verdict": unknown;
8261
+ "definitionVersionId": string | null;
8262
+ "runnerKind": "ora-runs" | "batch-job";
8263
+ "createdAt": string;
8264
+ "updatedAt": string;
8265
+ "liveUntil": string | null;
8266
+ "launch"?: {
8267
+ "attempt": number;
8268
+ "state": "fanning_out" | "deferred" | "settled";
8269
+ "startedAt": string;
8270
+ "progressAt": string;
8271
+ "planned"?: number;
8272
+ "created"?: number;
8273
+ "missing"?: {
8274
+ "retryable": number;
8275
+ "refused": number;
8276
+ };
8277
+ "notes"?: Array<string>;
8278
+ "settledAt"?: string;
8279
+ "pacing"?: {
8280
+ "waiting": number;
8281
+ "nextUnit"?: number;
8282
+ "nextWaveAt"?: string;
8283
+ "emptyWaves"?: number;
8284
+ "stopped"?: string;
8285
+ };
8286
+ "recovery"?: {
8287
+ "state": "retrying" | "needs_attention";
8288
+ "attempts": number;
8289
+ "lastAttemptAt"?: string;
8290
+ "nextAttemptAt"?: string;
7892
8291
  "reason"?: string;
7893
8292
  };
7894
8293
  } | null;
@@ -7897,6 +8296,8 @@ export type StudyGetResponse = {
7897
8296
  "startsPerMinute"?: number;
7898
8297
  } | null;
7899
8298
  "relaunchable"?: boolean;
8299
+ "archivedAt"?: string | null;
8300
+ "archivedBy"?: string | null;
7900
8301
  "arms"?: Array<{
7901
8302
  "key": string;
7902
8303
  "position": number;
@@ -7935,6 +8336,29 @@ export type StudyGetResponse = {
7935
8336
  "cellPlan"?: "crossed" | "per_setup";
7936
8337
  "skillDisclosure"?: "withheld" | "named";
7937
8338
  "modeOptions"?: Record<string, unknown>;
8339
+ "screens"?: Array<"test_awareness">;
8340
+ "runCounting"?: {
8341
+ "policy": "hold_back_screened";
8342
+ "screen": "test_awareness";
8343
+ "levels": Array<"aware" | "influenced">;
8344
+ } | null;
8345
+ "seriesKey"?: string | null;
8346
+ "ref"?: {
8347
+ "commit"?: string;
8348
+ "branch"?: string;
8349
+ "pr"?: number;
8350
+ "url"?: string;
8351
+ } | null;
8352
+ "gradingRevision"?: number;
8353
+ "gradingRevisedAt"?: string | null;
8354
+ "rehearsal"?: true;
8355
+ "rehearsalMode"?: string;
8356
+ "behaviors"?: Array<{
8357
+ "key": string;
8358
+ "name": string;
8359
+ "question": string;
8360
+ "version": number;
8361
+ }>;
7938
8362
  };
7939
8363
  "tasks": Array<{
7940
8364
  "id": string;
@@ -8083,6 +8507,12 @@ export type StudyGetResponse = {
8083
8507
  }>;
8084
8508
  "modelCalls": number | null;
8085
8509
  } | null;
8510
+ "screens"?: Record<string, unknown>;
8511
+ "carried"?: {
8512
+ "fromRunId": string;
8513
+ "fromStudyId": string | null;
8514
+ "at": string;
8515
+ } | null;
8086
8516
  }>;
8087
8517
  "aggregate": {
8088
8518
  "summary": {
@@ -8346,6 +8776,18 @@ export type StudyGetResponse = {
8346
8776
  "byArm": Record<string, unknown>;
8347
8777
  "phasesByArm": Record<string, unknown>;
8348
8778
  };
8779
+ "runCounting"?: {
8780
+ "policy": string;
8781
+ "screen": string;
8782
+ "levels": Array<string>;
8783
+ "heldBack": number;
8784
+ "groups": Array<{
8785
+ "group": string;
8786
+ "heldBack": number;
8787
+ "runs": number;
8788
+ }>;
8789
+ "imbalanced": boolean;
8790
+ };
8349
8791
  } | {
8350
8792
  "shape": "multi-arm";
8351
8793
  "summary": {
@@ -9221,6 +9663,18 @@ export type StudyGetResponse = {
9221
9663
  "byArm": Record<string, unknown>;
9222
9664
  "phasesByArm": Record<string, unknown>;
9223
9665
  };
9666
+ "runCounting"?: {
9667
+ "policy": string;
9668
+ "screen": string;
9669
+ "levels": Array<string>;
9670
+ "heldBack": number;
9671
+ "groups": Array<{
9672
+ "group": string;
9673
+ "heldBack": number;
9674
+ "runs": number;
9675
+ }>;
9676
+ "imbalanced": boolean;
9677
+ };
9224
9678
  };
9225
9679
  "warnings": Array<{
9226
9680
  "type": "eval_capability_gap";
@@ -9231,7 +9685,12 @@ export type StudyGetResponse = {
9231
9685
  "detail": string;
9232
9686
  "taskIds"?: Array<string>;
9233
9687
  } | {
9234
- "type": "design_aggregate_budget";
9688
+ "type": "design_lint";
9689
+ "lint": string;
9690
+ "detail": string;
9691
+ "taskIds": Array<string>;
9692
+ } | {
9693
+ "type": "design_aggregate_budget";
9235
9694
  "detail": string;
9236
9695
  }>;
9237
9696
  "taskHealth": {
@@ -9396,6 +9855,8 @@ export type StudyUpdateDetailsResponse = {
9396
9855
  "startsPerMinute"?: number;
9397
9856
  } | null;
9398
9857
  "relaunchable"?: boolean;
9858
+ "archivedAt"?: string | null;
9859
+ "archivedBy"?: string | null;
9399
9860
  "arms"?: Array<{
9400
9861
  "key": string;
9401
9862
  "position": number;
@@ -9434,6 +9895,29 @@ export type StudyUpdateDetailsResponse = {
9434
9895
  "cellPlan"?: "crossed" | "per_setup";
9435
9896
  "skillDisclosure"?: "withheld" | "named";
9436
9897
  "modeOptions"?: Record<string, unknown>;
9898
+ "screens"?: Array<"test_awareness">;
9899
+ "runCounting"?: {
9900
+ "policy": "hold_back_screened";
9901
+ "screen": "test_awareness";
9902
+ "levels": Array<"aware" | "influenced">;
9903
+ } | null;
9904
+ "seriesKey"?: string | null;
9905
+ "ref"?: {
9906
+ "commit"?: string;
9907
+ "branch"?: string;
9908
+ "pr"?: number;
9909
+ "url"?: string;
9910
+ } | null;
9911
+ "gradingRevision"?: number;
9912
+ "gradingRevisedAt"?: string | null;
9913
+ "rehearsal"?: true;
9914
+ "rehearsalMode"?: string;
9915
+ "behaviors"?: Array<{
9916
+ "key": string;
9917
+ "name": string;
9918
+ "question": string;
9919
+ "version": number;
9920
+ }>;
9437
9921
  };
9438
9922
  };
9439
9923
  export type StudyDeleteResponse = unknown;
@@ -9543,10 +10027,150 @@ export type StudyAddAgentsResponse = {
9543
10027
  "reason": string | null;
9544
10028
  };
9545
10029
  };
10030
+ export type StudyBehaviorsResponse = {
10031
+ "behaviors": Array<{
10032
+ "key": string;
10033
+ "name": string;
10034
+ "question": string;
10035
+ "version": number;
10036
+ "screenable": number;
10037
+ "judged": number;
10038
+ "measurement": {
10039
+ "key": string;
10040
+ "version": number;
10041
+ "verdicts": {
10042
+ "present": number;
10043
+ "absent": number;
10044
+ "unknown": number;
10045
+ };
10046
+ "present": {
10047
+ "reviewed": number;
10048
+ "confirmed": number;
10049
+ "precision": {
10050
+ "rate": number;
10051
+ "lower": number;
10052
+ "upper": number;
10053
+ "successes": number;
10054
+ "n": number;
10055
+ } | null;
10056
+ };
10057
+ "absent": {
10058
+ "reviewed": number;
10059
+ "confirmed": number;
10060
+ "precision": {
10061
+ "rate": number;
10062
+ "lower": number;
10063
+ "upper": number;
10064
+ "successes": number;
10065
+ "n": number;
10066
+ } | null;
10067
+ };
10068
+ "contested": number;
10069
+ "agreement": {
10070
+ "runs": number;
10071
+ "agreed": number;
10072
+ "kappa": number | null;
10073
+ } | null;
10074
+ };
10075
+ }>;
10076
+ "reviews": number;
10077
+ "reviewsTruncated": boolean;
10078
+ };
10079
+ export type StudyCarryPlanResponse = {
10080
+ "policy": {
10081
+ "from": "series";
10082
+ "maxAgeDays": number;
10083
+ } | null;
10084
+ "sourceStudyId": string | null;
10085
+ "planned": number;
10086
+ "carried": number;
10087
+ "fresh": Record<string, unknown>;
10088
+ "differences": Array<string>;
10089
+ "carriedRuns": number;
10090
+ };
9546
10091
  export type StudyRegenerateSummaryResponse = {
9547
10092
  "status": "queued";
9548
10093
  "target": "live" | "draft";
9549
10094
  };
10095
+ export type StudyGateRequest = {
10096
+ "metric"?: "benchmark" | "goal" | "evals" | "quality" | "rubric" | "completion";
10097
+ "group"?: string;
10098
+ "floor"?: {
10099
+ "rate": number;
10100
+ } | {
10101
+ "points": number;
10102
+ };
10103
+ "maxDrop"?: {
10104
+ "pp": number;
10105
+ } | {
10106
+ "points": number;
10107
+ };
10108
+ "baseline"?: {
10109
+ "studyId": string;
10110
+ } | {
10111
+ "series": {
10112
+ "branch"?: string;
10113
+ };
10114
+ };
10115
+ "minTasks"?: number;
10116
+ "infraCeiling"?: number;
10117
+ "counting"?: "policy" | "all";
10118
+ };
10119
+ export type StudyGateResponse = {
10120
+ "state": "settled" | "pending";
10121
+ "owed": Array<string>;
10122
+ "countingPolicy": string;
10123
+ "metric": {
10124
+ "key": string;
10125
+ "label": string;
10126
+ "family": "rate" | "score";
10127
+ };
10128
+ "group": string;
10129
+ "groups": Array<{
10130
+ "key": string;
10131
+ "label": string;
10132
+ "value": number | null;
10133
+ "runs": number;
10134
+ "tasks": number;
10135
+ "interval": Array<unknown> | null;
10136
+ "carriedRuns"?: number;
10137
+ "heldBackRuns"?: number;
10138
+ }>;
10139
+ "baseline": {
10140
+ "studyId": string;
10141
+ "name": string;
10142
+ "comparable": boolean;
10143
+ "differs": Array<string>;
10144
+ "value": number | null;
10145
+ "interval": Array<unknown> | null;
10146
+ "difference": {
10147
+ "effect": number;
10148
+ "unit": "pp" | "pts";
10149
+ "interval": Array<unknown>;
10150
+ "tasks": number;
10151
+ } | null;
10152
+ } | null;
10153
+ "comparabilityKey": string;
10154
+ "series": {
10155
+ "key": string;
10156
+ "ref": {
10157
+ "commit"?: string;
10158
+ "branch"?: string;
10159
+ "pr"?: number;
10160
+ "url"?: string;
10161
+ } | null;
10162
+ } | null;
10163
+ "gate": {
10164
+ "decision": "pass" | "fail" | "undecided";
10165
+ "refusedBy": string | null;
10166
+ "reason": string;
10167
+ "rules": Array<{
10168
+ "rule": string;
10169
+ "decision": "pass" | "fail" | "undecided";
10170
+ "reason": string;
10171
+ }>;
10172
+ };
10173
+ };
9550
10174
  export type StudyGenerateRequest = {
9551
10175
  "proposeExpectations"?: boolean;
9552
10176
  };
@@ -9685,6 +10309,8 @@ export type StudyGenerateResponse = {
9685
10309
  "startsPerMinute"?: number;
9686
10310
  } | null;
9687
10311
  "relaunchable"?: boolean;
10312
+ "archivedAt"?: string | null;
10313
+ "archivedBy"?: string | null;
9688
10314
  "arms"?: Array<{
9689
10315
  "key": string;
9690
10316
  "position": number;
@@ -9723,6 +10349,29 @@ export type StudyGenerateResponse = {
9723
10349
  "cellPlan"?: "crossed" | "per_setup";
9724
10350
  "skillDisclosure"?: "withheld" | "named";
9725
10351
  "modeOptions"?: Record<string, unknown>;
10352
+ "screens"?: Array<"test_awareness">;
10353
+ "runCounting"?: {
10354
+ "policy": "hold_back_screened";
10355
+ "screen": "test_awareness";
10356
+ "levels": Array<"aware" | "influenced">;
10357
+ } | null;
10358
+ "seriesKey"?: string | null;
10359
+ "ref"?: {
10360
+ "commit"?: string;
10361
+ "branch"?: string;
10362
+ "pr"?: number;
10363
+ "url"?: string;
10364
+ } | null;
10365
+ "gradingRevision"?: number;
10366
+ "gradingRevisedAt"?: string | null;
10367
+ "rehearsal"?: true;
10368
+ "rehearsalMode"?: string;
10369
+ "behaviors"?: Array<{
10370
+ "key": string;
10371
+ "name": string;
10372
+ "question": string;
10373
+ "version": number;
10374
+ }>;
9726
10375
  };
9727
10376
  "streamUrl": string;
9728
10377
  "warnings"?: Array<{
@@ -9733,6 +10382,68 @@ export type StudyGenerateResponse = {
9733
10382
  "detail": string;
9734
10383
  }>;
9735
10384
  };
10385
+ export type StudyGradingRevisionsResponse = {
10386
+ "revision": number;
10387
+ "revisions": Array<{
10388
+ "revision": number;
10389
+ "createdBy": string | null;
10390
+ "createdAt": string;
10391
+ "reason": string;
10392
+ "changes": Array<{
10393
+ "taskId": string;
10394
+ "position": number;
10395
+ "before": {
10396
+ "rubric": Array<string>;
10397
+ "expectations": Array<unknown>;
10398
+ };
10399
+ "after": {
10400
+ "rubric": Array<string>;
10401
+ "expectations": Array<unknown>;
10402
+ };
10403
+ "lenses": Array<"judge" | "evals">;
10404
+ }>;
10405
+ }>;
10406
+ };
10407
+ export type StudyReviseGradingRequest = {
10408
+ "reason": string;
10409
+ "tasks": Array<{
10410
+ "taskId": string;
10411
+ "rubric"?: Array<string>;
10412
+ "expectations"?: Array<unknown>;
10413
+ }>;
10414
+ "dryRun"?: boolean;
10415
+ };
10416
+ export type StudyReviseGradingResponse = {
10417
+ "dryRun": boolean;
10418
+ "plan": {
10419
+ "revision": number;
10420
+ "lenses": Array<"judge" | "evals">;
10421
+ "tasks": number;
10422
+ "evalRegrades": number;
10423
+ "judgeReads": number;
10424
+ };
10425
+ "changes": Array<{
10426
+ "taskId": string;
10427
+ "position": number;
10428
+ "before": {
10429
+ "rubric": Array<string>;
10430
+ "expectations": Array<unknown>;
10431
+ };
10432
+ "after": {
10433
+ "rubric": Array<string>;
10434
+ "expectations": Array<unknown>;
10435
+ };
10436
+ "lenses": Array<"judge" | "evals">;
10437
+ }>;
10438
+ "flips"?: Array<{
10439
+ "runId": string;
10440
+ "taskId": string;
10441
+ "arm": string;
10442
+ "before": number | null;
10443
+ "after": number | null;
10444
+ }>;
10445
+ "flipCount"?: number;
10446
+ };
9736
10447
  export type StudyLaunchResponse = {
9737
10448
  "study": {
9738
10449
  "id": string;
@@ -9868,6 +10579,8 @@ export type StudyLaunchResponse = {
9868
10579
  "startsPerMinute"?: number;
9869
10580
  } | null;
9870
10581
  "relaunchable"?: boolean;
10582
+ "archivedAt"?: string | null;
10583
+ "archivedBy"?: string | null;
9871
10584
  "arms"?: Array<{
9872
10585
  "key": string;
9873
10586
  "position": number;
@@ -9906,6 +10619,29 @@ export type StudyLaunchResponse = {
9906
10619
  "cellPlan"?: "crossed" | "per_setup";
9907
10620
  "skillDisclosure"?: "withheld" | "named";
9908
10621
  "modeOptions"?: Record<string, unknown>;
10622
+ "screens"?: Array<"test_awareness">;
10623
+ "runCounting"?: {
10624
+ "policy": "hold_back_screened";
10625
+ "screen": "test_awareness";
10626
+ "levels": Array<"aware" | "influenced">;
10627
+ } | null;
10628
+ "seriesKey"?: string | null;
10629
+ "ref"?: {
10630
+ "commit"?: string;
10631
+ "branch"?: string;
10632
+ "pr"?: number;
10633
+ "url"?: string;
10634
+ } | null;
10635
+ "gradingRevision"?: number;
10636
+ "gradingRevisedAt"?: string | null;
10637
+ "rehearsal"?: true;
10638
+ "rehearsalMode"?: string;
10639
+ "behaviors"?: Array<{
10640
+ "key": string;
10641
+ "name": string;
10642
+ "question": string;
10643
+ "version": number;
10644
+ }>;
9909
10645
  };
9910
10646
  "streamUrl": string;
9911
10647
  "warnings"?: Array<{
@@ -10054,6 +10790,8 @@ export type StudyUpdateLensesResponse = {
10054
10790
  "startsPerMinute"?: number;
10055
10791
  } | null;
10056
10792
  "relaunchable"?: boolean;
10793
+ "archivedAt"?: string | null;
10794
+ "archivedBy"?: string | null;
10057
10795
  "arms"?: Array<{
10058
10796
  "key": string;
10059
10797
  "position": number;
@@ -10092,6 +10830,29 @@ export type StudyUpdateLensesResponse = {
10092
10830
  "cellPlan"?: "crossed" | "per_setup";
10093
10831
  "skillDisclosure"?: "withheld" | "named";
10094
10832
  "modeOptions"?: Record<string, unknown>;
10833
+ "screens"?: Array<"test_awareness">;
10834
+ "runCounting"?: {
10835
+ "policy": "hold_back_screened";
10836
+ "screen": "test_awareness";
10837
+ "levels": Array<"aware" | "influenced">;
10838
+ } | null;
10839
+ "seriesKey"?: string | null;
10840
+ "ref"?: {
10841
+ "commit"?: string;
10842
+ "branch"?: string;
10843
+ "pr"?: number;
10844
+ "url"?: string;
10845
+ } | null;
10846
+ "gradingRevision"?: number;
10847
+ "gradingRevisedAt"?: string | null;
10848
+ "rehearsal"?: true;
10849
+ "rehearsalMode"?: string;
10850
+ "behaviors"?: Array<{
10851
+ "key": string;
10852
+ "name": string;
10853
+ "question": string;
10854
+ "version": number;
10855
+ }>;
10095
10856
  };
10096
10857
  "warnings": Array<{
10097
10858
  "type": "judge_removed_rubric_unused" | "evals_skill_loaded_not_pointer" | "no_gradable_input";
@@ -10195,6 +10956,14 @@ export type StudyPreviewResponse = {
10195
10956
  "label": string;
10196
10957
  "skills": Array<string>;
10197
10958
  }>;
10959
+ "gradingRevision"?: number;
10960
+ "gradingRevisedAt"?: string | null;
10961
+ "behaviors"?: Array<{
10962
+ "key": string;
10963
+ "name": string;
10964
+ "question": string;
10965
+ "version": number;
10966
+ }>;
10198
10967
  };
10199
10968
  "tasks": Array<{
10200
10969
  "id": string;
@@ -10255,6 +11024,8 @@ export type StudyPreviewResponse = {
10255
11024
  "paramsKey"?: string;
10256
11025
  "textWithheld"?: boolean;
10257
11026
  "runTimeMs"?: number | null;
11027
+ "screens"?: Record<string, unknown>;
11028
+ "carriedRun"?: true;
10258
11029
  }>;
10259
11030
  "aggregate": {
10260
11031
  "summary": {
@@ -10518,6 +11289,18 @@ export type StudyPreviewResponse = {
10518
11289
  "byArm": Record<string, unknown>;
10519
11290
  "phasesByArm": Record<string, unknown>;
10520
11291
  };
11292
+ "runCounting"?: {
11293
+ "policy": string;
11294
+ "screen": string;
11295
+ "levels": Array<string>;
11296
+ "heldBack": number;
11297
+ "groups": Array<{
11298
+ "group": string;
11299
+ "heldBack": number;
11300
+ "runs": number;
11301
+ }>;
11302
+ "imbalanced": boolean;
11303
+ };
10521
11304
  } | {
10522
11305
  "shape": "multi-arm";
10523
11306
  "summary": {
@@ -11393,6 +12176,18 @@ export type StudyPreviewResponse = {
11393
12176
  "byArm": Record<string, unknown>;
11394
12177
  "phasesByArm": Record<string, unknown>;
11395
12178
  };
12179
+ "runCounting"?: {
12180
+ "policy": string;
12181
+ "screen": string;
12182
+ "levels": Array<string>;
12183
+ "heldBack": number;
12184
+ "groups": Array<{
12185
+ "group": string;
12186
+ "heldBack": number;
12187
+ "runs": number;
12188
+ }>;
12189
+ "imbalanced": boolean;
12190
+ };
11396
12191
  };
11397
12192
  "textWithheldReason"?: "sign_in_required" | "terms_required" | "licence";
11398
12193
  };
@@ -11537,6 +12332,8 @@ export type StudyPublishResponse = {
11537
12332
  "startsPerMinute"?: number;
11538
12333
  } | null;
11539
12334
  "relaunchable"?: boolean;
12335
+ "archivedAt"?: string | null;
12336
+ "archivedBy"?: string | null;
11540
12337
  "arms"?: Array<{
11541
12338
  "key": string;
11542
12339
  "position": number;
@@ -11575,8 +12372,72 @@ export type StudyPublishResponse = {
11575
12372
  "cellPlan"?: "crossed" | "per_setup";
11576
12373
  "skillDisclosure"?: "withheld" | "named";
11577
12374
  "modeOptions"?: Record<string, unknown>;
12375
+ "screens"?: Array<"test_awareness">;
12376
+ "runCounting"?: {
12377
+ "policy": "hold_back_screened";
12378
+ "screen": "test_awareness";
12379
+ "levels": Array<"aware" | "influenced">;
12380
+ } | null;
12381
+ "seriesKey"?: string | null;
12382
+ "ref"?: {
12383
+ "commit"?: string;
12384
+ "branch"?: string;
12385
+ "pr"?: number;
12386
+ "url"?: string;
12387
+ } | null;
12388
+ "gradingRevision"?: number;
12389
+ "gradingRevisedAt"?: string | null;
12390
+ "rehearsal"?: true;
12391
+ "rehearsalMode"?: string;
12392
+ "behaviors"?: Array<{
12393
+ "key": string;
12394
+ "name": string;
12395
+ "question": string;
12396
+ "version": number;
12397
+ }>;
11578
12398
  };
11579
12399
  };
12400
+ export type StudyQueryRequest = {
12401
+ "metrics": Array<string>;
12402
+ "groupBy"?: Array<string>;
12403
+ "filters"?: Array<{
12404
+ "field": string;
12405
+ "in": Array<string>;
12406
+ }>;
12407
+ "limit"?: number;
12408
+ };
12409
+ export type StudyQueryResponse = {
12410
+ "basis": "runs";
12411
+ "matchedRuns": number;
12412
+ "groupBy": Array<{
12413
+ "id": string;
12414
+ "label": string;
12415
+ }>;
12416
+ "metrics": Array<{
12417
+ "id": string;
12418
+ "label": string;
12419
+ "unit": "count" | "rate" | "points" | "usd" | "tokens" | "ms";
12420
+ "fold": string;
12421
+ }>;
12422
+ "rows": Array<{
12423
+ "key": Record<string, unknown>;
12424
+ "runs": number;
12425
+ "cells": Record<string, unknown>;
12426
+ }>;
12427
+ "truncated": boolean;
12428
+ };
12429
+ export type StudyQueryVocabularyResponse = {
12430
+ "dimensions": Array<{
12431
+ "id": string;
12432
+ "label": string;
12433
+ }>;
12434
+ "metrics": Array<{
12435
+ "id": string;
12436
+ "label": string;
12437
+ "unit": "count" | "rate" | "points" | "usd" | "tokens" | "ms";
12438
+ "fold": string;
12439
+ }>;
12440
+ };
11580
12441
  export type StudyReconcileRunsResponse = {
11581
12442
  "studyId": string;
11582
12443
  "laggingRuns": number;
@@ -11721,6 +12582,8 @@ export type StudyRegradeResponse = {
11721
12582
  "startsPerMinute"?: number;
11722
12583
  } | null;
11723
12584
  "relaunchable"?: boolean;
12585
+ "archivedAt"?: string | null;
12586
+ "archivedBy"?: string | null;
11724
12587
  "arms"?: Array<{
11725
12588
  "key": string;
11726
12589
  "position": number;
@@ -11759,6 +12622,29 @@ export type StudyRegradeResponse = {
11759
12622
  "cellPlan"?: "crossed" | "per_setup";
11760
12623
  "skillDisclosure"?: "withheld" | "named";
11761
12624
  "modeOptions"?: Record<string, unknown>;
12625
+ "screens"?: Array<"test_awareness">;
12626
+ "runCounting"?: {
12627
+ "policy": "hold_back_screened";
12628
+ "screen": "test_awareness";
12629
+ "levels": Array<"aware" | "influenced">;
12630
+ } | null;
12631
+ "seriesKey"?: string | null;
12632
+ "ref"?: {
12633
+ "commit"?: string;
12634
+ "branch"?: string;
12635
+ "pr"?: number;
12636
+ "url"?: string;
12637
+ } | null;
12638
+ "gradingRevision"?: number;
12639
+ "gradingRevisedAt"?: string | null;
12640
+ "rehearsal"?: true;
12641
+ "rehearsalMode"?: string;
12642
+ "behaviors"?: Array<{
12643
+ "key": string;
12644
+ "name": string;
12645
+ "question": string;
12646
+ "version": number;
12647
+ }>;
11762
12648
  };
11763
12649
  "streamUrl": string;
11764
12650
  "warnings"?: Array<{
@@ -11769,6 +12655,23 @@ export type StudyRegradeResponse = {
11769
12655
  "detail": string;
11770
12656
  }>;
11771
12657
  };
12658
+ export type StudyRehearsalReportResponse = {
12659
+ "studyId": string;
12660
+ "name": string;
12661
+ "status": string;
12662
+ "mode": string;
12663
+ "state": "pending" | "ready" | "not_ready";
12664
+ "groups": Array<{
12665
+ "key": string;
12666
+ "label": string;
12667
+ "state": "pending" | "ready" | "not_ready";
12668
+ "checks": Array<{
12669
+ "check": string;
12670
+ "answer": "pass" | "fail" | "pending" | "unknown";
12671
+ "detail": string;
12672
+ }>;
12673
+ }>;
12674
+ };
11772
12675
  export type StudyRerunAttemptsResponse = {
11773
12676
  "studyId": string;
11774
12677
  "attempts": Array<{
@@ -11837,6 +12740,405 @@ export type StudyRerunInfraFailuresResponse = {
11837
12740
  "reason": string;
11838
12741
  }>;
11839
12742
  };
12743
+ export type StudyRestoreResponse = {
12744
+ "study": {
12745
+ "id": string;
12746
+ "executiveSummary"?: string | null;
12747
+ "executiveSummaryDraft"?: string | null;
12748
+ "executiveSummaryRequest"?: "live" | "draft" | null;
12749
+ "executiveSummaryRequestedAt"?: string | null;
12750
+ "executiveSummaryUnavailableAt"?: string | null;
12751
+ "plannedRunCount": number;
12752
+ "organizationId": string;
12753
+ "createdBy": string | null;
12754
+ "name": string;
12755
+ "slug": string;
12756
+ "description": string | null;
12757
+ "skillRef": string | null;
12758
+ "skillSlug": string | null;
12759
+ "skillName": string | null;
12760
+ "skillPromise": string;
12761
+ "subjectKind": "skill" | "tools" | "addon" | "custom" | "benchmark";
12762
+ "subjectName": string | null;
12763
+ "subjectLabel": string;
12764
+ "armConfig": {
12765
+ "baseline"?: {
12766
+ "tools"?: "default" | "none" | {
12767
+ "tools": Array<string>;
12768
+ };
12769
+ "addons"?: Record<string, unknown>;
12770
+ };
12771
+ "treatment"?: {
12772
+ "tools"?: "default" | "none" | {
12773
+ "tools": Array<string>;
12774
+ };
12775
+ "addons"?: Record<string, unknown>;
12776
+ };
12777
+ };
12778
+ "comparisonMode": "ab_vs_none" | "ab_custom" | "single" | "multi_arm";
12779
+ "subjectBackend": "model" | "harness" | null;
12780
+ "environmentExecutionPolicy"?: "legacy-v1" | "inside-v1";
12781
+ "importedFrom"?: string;
12782
+ "baselineLabel": string | null;
12783
+ "treatmentLabel": string | null;
12784
+ "promiseMetrics": Array<{
12785
+ "metric": "total_tokens" | "output_tokens" | "cost_usd" | "duration_ms" | "num_turns" | "success_rate" | "completion_rate" | "goal_rate";
12786
+ "direction": "down" | "up";
12787
+ "label": string;
12788
+ }>;
12789
+ "matrix": Array<{
12790
+ "harness": string;
12791
+ "model": string;
12792
+ "provider"?: string;
12793
+ "params"?: {
12794
+ "temperature"?: number;
12795
+ "maxOutputTokens"?: number;
12796
+ "reasoningEffort"?: "low" | "medium" | "high";
12797
+ "thinkingBudgetTokens"?: number;
12798
+ "codeMode"?: "auto" | "on" | "off";
12799
+ "toolSearch"?: "on" | "off" | "directory";
12800
+ };
12801
+ }>;
12802
+ "lenses": Array<"judge" | "evals" | "reference" | "framework" | "execution" | "verifier">;
12803
+ "judgeSupplementary"?: boolean;
12804
+ "taskCount": number;
12805
+ "variantsPerTask": number;
12806
+ "repeats": number;
12807
+ "subset": {
12808
+ "count": number;
12809
+ "seed": string;
12810
+ "of": number;
12811
+ } | null;
12812
+ "status": string;
12813
+ "error": string | null;
12814
+ "preparation"?: {
12815
+ "since": string;
12816
+ "progressAt"?: string;
12817
+ "launchedBy": {
12818
+ "userId"?: string;
12819
+ "role"?: string;
12820
+ };
12821
+ "waitingOn": Array<{
12822
+ "imageKey": string;
12823
+ "buildId": string;
12824
+ "stage"?: "image" | "snapshot";
12825
+ "retry"?: {
12826
+ "failedAttempts": number;
12827
+ "maxAttempts": number;
12828
+ "notBefore": string;
12829
+ "lastFailure": string;
12830
+ };
12831
+ }>;
12832
+ } | null;
12833
+ "visibility": "private" | "org" | "link" | "public";
12834
+ "visibilityChangedAt"?: string | null;
12835
+ "visibilityChangedBy"?: string | null;
12836
+ "shareToken"?: string | null;
12837
+ "lowSampleOverride": boolean;
12838
+ "publishedAt": string | null;
12839
+ "verdict": unknown;
12840
+ "definitionVersionId": string | null;
12841
+ "runnerKind": "ora-runs" | "batch-job";
12842
+ "createdAt": string;
12843
+ "updatedAt": string;
12844
+ "liveUntil": string | null;
12845
+ "launch"?: {
12846
+ "attempt": number;
12847
+ "state": "fanning_out" | "deferred" | "settled";
12848
+ "startedAt": string;
12849
+ "progressAt": string;
12850
+ "planned"?: number;
12851
+ "created"?: number;
12852
+ "missing"?: {
12853
+ "retryable": number;
12854
+ "refused": number;
12855
+ };
12856
+ "notes"?: Array<string>;
12857
+ "settledAt"?: string;
12858
+ "pacing"?: {
12859
+ "waiting": number;
12860
+ "nextUnit"?: number;
12861
+ "nextWaveAt"?: string;
12862
+ "emptyWaves"?: number;
12863
+ "stopped"?: string;
12864
+ };
12865
+ "recovery"?: {
12866
+ "state": "retrying" | "needs_attention";
12867
+ "attempts": number;
12868
+ "lastAttemptAt"?: string;
12869
+ "nextAttemptAt"?: string;
12870
+ "reason"?: string;
12871
+ };
12872
+ } | null;
12873
+ "pacing"?: {
12874
+ "maxInFlight"?: number;
12875
+ "startsPerMinute"?: number;
12876
+ } | null;
12877
+ "relaunchable"?: boolean;
12878
+ "archivedAt"?: string | null;
12879
+ "archivedBy"?: string | null;
12880
+ "arms"?: Array<{
12881
+ "key": string;
12882
+ "position": number;
12883
+ "label": string;
12884
+ "role": "reference" | "peer";
12885
+ "payload": {
12886
+ "tools"?: "default" | "none" | {
12887
+ "tools": Array<string>;
12888
+ };
12889
+ "addons"?: Record<string, unknown>;
12890
+ };
12891
+ "cells"?: Array<{
12892
+ "harness": string;
12893
+ "model": string;
12894
+ "provider"?: string;
12895
+ "params"?: {
12896
+ "temperature"?: number;
12897
+ "maxOutputTokens"?: number;
12898
+ "reasoningEffort"?: "low" | "medium" | "high";
12899
+ "thinkingBudgetTokens"?: number;
12900
+ "codeMode"?: "auto" | "on" | "off";
12901
+ "toolSearch"?: "on" | "off" | "directory";
12902
+ };
12903
+ }>;
12904
+ "skills"?: Array<{
12905
+ "ref": string;
12906
+ "name": string;
12907
+ "changedSinceCreate"?: boolean;
12908
+ }>;
12909
+ "addedAgents"?: Array<{
12910
+ "position": number;
12911
+ "addedAt": string;
12912
+ }>;
12913
+ }>;
12914
+ "contrastPlan"?: "vs_reference" | "all_pairs";
12915
+ "cellPlan"?: "crossed" | "per_setup";
12916
+ "skillDisclosure"?: "withheld" | "named";
12917
+ "modeOptions"?: Record<string, unknown>;
12918
+ "screens"?: Array<"test_awareness">;
12919
+ "runCounting"?: {
12920
+ "policy": "hold_back_screened";
12921
+ "screen": "test_awareness";
12922
+ "levels": Array<"aware" | "influenced">;
12923
+ } | null;
12924
+ "seriesKey"?: string | null;
12925
+ "ref"?: {
12926
+ "commit"?: string;
12927
+ "branch"?: string;
12928
+ "pr"?: number;
12929
+ "url"?: string;
12930
+ } | null;
12931
+ "gradingRevision"?: number;
12932
+ "gradingRevisedAt"?: string | null;
12933
+ "rehearsal"?: true;
12934
+ "rehearsalMode"?: string;
12935
+ "behaviors"?: Array<{
12936
+ "key": string;
12937
+ "name": string;
12938
+ "question": string;
12939
+ "version": number;
12940
+ }>;
12941
+ };
12942
+ };
12943
+ export type StudyRetestsResponse = {
12944
+ "retests": Array<{
12945
+ "id": string;
12946
+ "kind": "fill" | "audit";
12947
+ "reason": string | null;
12948
+ "requestedBy": string | null;
12949
+ "createdAt": string;
12950
+ "settledAt": string | null;
12951
+ "summary": {
12952
+ "rechecked": number;
12953
+ "recorded": number;
12954
+ "filled": number;
12955
+ "agreed": number;
12956
+ "disagreements": Array<{
12957
+ "runId": string;
12958
+ "retestRunId": string;
12959
+ "run": {
12960
+ "pass": boolean | null;
12961
+ "status": "passed" | "failed" | "timeout" | "not-run" | "unavailable";
12962
+ "reason"?: string;
12963
+ };
12964
+ "recheck": {
12965
+ "pass": boolean | null;
12966
+ "status": "passed" | "failed" | "timeout" | "not-run" | "unavailable";
12967
+ "reason"?: string;
12968
+ };
12969
+ }>;
12970
+ };
12971
+ "runs": Array<{
12972
+ "runId": string;
12973
+ "retestRunId": string;
12974
+ "runStatus": string | null;
12975
+ "finishedAt": string | null;
12976
+ "verdict": {
12977
+ "pass": boolean | null;
12978
+ "status": "passed" | "failed" | "timeout" | "not-run" | "unavailable";
12979
+ "reason"?: string;
12980
+ } | null;
12981
+ }>;
12982
+ }>;
12983
+ };
12984
+ export type StudyRetestRequest = {
12985
+ "kind"?: "fill" | "audit";
12986
+ "runIds"?: Array<string>;
12987
+ "reason"?: string;
12988
+ "dryRun"?: boolean;
12989
+ };
12990
+ export type StudyRetestResponse = {
12991
+ "dryRun": boolean;
12992
+ "kind": "fill" | "audit";
12993
+ "retestId": string | null;
12994
+ "runs": Array<{
12995
+ "runId": string;
12996
+ "outcome"?: "dispatched" | "failed";
12997
+ "retestRunId"?: string | null;
12998
+ "error"?: string | null;
12999
+ }>;
13000
+ "skipped": Array<{
13001
+ "runId": string;
13002
+ "reason": "not_in_study" | "not_succeeded" | "no_verdict" | "no_saved_inputs" | "saved_files_missing";
13003
+ "detail": string;
13004
+ }>;
13005
+ };
13006
+ export type StudyRunCountingResponse = {
13007
+ "runCounting": {
13008
+ "policy": "hold_back_screened";
13009
+ "screen": "test_awareness";
13010
+ "levels": Array<"aware" | "influenced">;
13011
+ } | null;
13012
+ "policy": string;
13013
+ "offered": boolean;
13014
+ "refusal": {
13015
+ "code": "run_counting_not_enabled" | "run_counting_screen_not_enabled" | "run_counting_screen_undeclared" | "run_counting_pairs_unnamed" | "run_counting_platform_discloses_test" | "run_counting_benchmark_at_create";
13016
+ "message": string;
13017
+ } | null;
13018
+ "screen": string;
13019
+ "screenedRuns": number;
13020
+ "unscreenedRuns": number;
13021
+ "flagged": Record<string, unknown>;
13022
+ "heldBack": Array<{
13023
+ "runId": string;
13024
+ "group": string;
13025
+ "taskId": string;
13026
+ "level": string;
13027
+ }>;
13028
+ "groups": Array<{
13029
+ "key": string;
13030
+ "label": string;
13031
+ "runs": number;
13032
+ "heldBack": number;
13033
+ "counted": {
13034
+ "value": number | null;
13035
+ "interval": Array<unknown> | null;
13036
+ "runs": number;
13037
+ "tasks": number;
13038
+ } | null;
13039
+ "all": {
13040
+ "value": number | null;
13041
+ "interval": Array<unknown> | null;
13042
+ "runs": number;
13043
+ "tasks": number;
13044
+ } | null;
13045
+ }>;
13046
+ "imbalanced": boolean;
13047
+ "metric": {
13048
+ "key": string;
13049
+ "label": string;
13050
+ "family": string;
13051
+ };
13052
+ "changes": Array<{
13053
+ "change": number;
13054
+ "createdBy": string | null;
13055
+ "createdAt": string;
13056
+ "previous": {
13057
+ "policy": "hold_back_screened";
13058
+ "screen": "test_awareness";
13059
+ "levels": Array<"aware" | "influenced">;
13060
+ } | null;
13061
+ "runCounting": {
13062
+ "policy": "hold_back_screened";
13063
+ "screen": "test_awareness";
13064
+ "levels": Array<"aware" | "influenced">;
13065
+ } | null;
13066
+ "classifierVersion": string | null;
13067
+ }>;
13068
+ };
13069
+ export type StudySetRunCountingRequest = {
13070
+ "runCounting": {
13071
+ "policy": "hold_back_screened";
13072
+ "screen": "test_awareness";
13073
+ "levels": Array<"aware" | "influenced">;
13074
+ } | null;
13075
+ "dryRun"?: boolean;
13076
+ };
13077
+ export type StudySetRunCountingResponse = {
13078
+ "runCounting": {
13079
+ "policy": "hold_back_screened";
13080
+ "screen": "test_awareness";
13081
+ "levels": Array<"aware" | "influenced">;
13082
+ } | null;
13083
+ "policy": string;
13084
+ "offered": boolean;
13085
+ "refusal": {
13086
+ "code": "run_counting_not_enabled" | "run_counting_screen_not_enabled" | "run_counting_screen_undeclared" | "run_counting_pairs_unnamed" | "run_counting_platform_discloses_test" | "run_counting_benchmark_at_create";
13087
+ "message": string;
13088
+ } | null;
13089
+ "screen": string;
13090
+ "screenedRuns": number;
13091
+ "unscreenedRuns": number;
13092
+ "flagged": Record<string, unknown>;
13093
+ "heldBack": Array<{
13094
+ "runId": string;
13095
+ "group": string;
13096
+ "taskId": string;
13097
+ "level": string;
13098
+ }>;
13099
+ "groups": Array<{
13100
+ "key": string;
13101
+ "label": string;
13102
+ "runs": number;
13103
+ "heldBack": number;
13104
+ "counted": {
13105
+ "value": number | null;
13106
+ "interval": Array<unknown> | null;
13107
+ "runs": number;
13108
+ "tasks": number;
13109
+ } | null;
13110
+ "all": {
13111
+ "value": number | null;
13112
+ "interval": Array<unknown> | null;
13113
+ "runs": number;
13114
+ "tasks": number;
13115
+ } | null;
13116
+ }>;
13117
+ "imbalanced": boolean;
13118
+ "metric": {
13119
+ "key": string;
13120
+ "label": string;
13121
+ "family": string;
13122
+ };
13123
+ "changes": Array<{
13124
+ "change": number;
13125
+ "createdBy": string | null;
13126
+ "createdAt": string;
13127
+ "previous": {
13128
+ "policy": "hold_back_screened";
13129
+ "screen": "test_awareness";
13130
+ "levels": Array<"aware" | "influenced">;
13131
+ } | null;
13132
+ "runCounting": {
13133
+ "policy": "hold_back_screened";
13134
+ "screen": "test_awareness";
13135
+ "levels": Array<"aware" | "influenced">;
13136
+ } | null;
13137
+ "classifierVersion": string | null;
13138
+ }>;
13139
+ "dryRun": boolean;
13140
+ "changed": boolean;
13141
+ };
11840
13142
  export type StudyListRunAnswersResponse = {
11841
13143
  "coordinatorRunId": string;
11842
13144
  "answers": Array<{
@@ -11850,7 +13152,35 @@ export type StudyListRunAnswersResponse = {
11850
13152
  "finishedAt": string | null;
11851
13153
  }>;
11852
13154
  };
13155
+ export type StudyRunDeliveredResponse = {
13156
+ "takenAt": string | null;
13157
+ "items": Array<{
13158
+ "id": string;
13159
+ "kind": "file" | "image";
13160
+ "origin": string;
13161
+ "url"?: string;
13162
+ "path"?: string;
13163
+ "status": string;
13164
+ "reason"?: string;
13165
+ "contentType"?: string;
13166
+ "bytes"?: number;
13167
+ "sha256"?: string;
13168
+ "readable"?: boolean;
13169
+ "redactions"?: Array<string>;
13170
+ "truncated"?: boolean;
13171
+ "downloadable": boolean;
13172
+ "filename": string;
13173
+ }>;
13174
+ "workspaceSkipped"?: {
13175
+ "dependencyOrDot": number;
13176
+ "overCap": number;
13177
+ };
13178
+ };
11853
13179
  export type StudyRunInsightResponse = unknown;
13180
+ export type StudyRunScreensResponse = {
13181
+ "screens": Record<string, unknown>;
13182
+ };
13183
+ export type StudyRunSpansResponse = unknown;
11854
13184
  export type StudyRunTrajectoryResponse = unknown;
11855
13185
  export type StudyRotateShareTokenResponse = {
11856
13186
  "study": {
@@ -11987,6 +13317,8 @@ export type StudyRotateShareTokenResponse = {
11987
13317
  "startsPerMinute"?: number;
11988
13318
  } | null;
11989
13319
  "relaunchable"?: boolean;
13320
+ "archivedAt"?: string | null;
13321
+ "archivedBy"?: string | null;
11990
13322
  "arms"?: Array<{
11991
13323
  "key": string;
11992
13324
  "position": number;
@@ -12025,6 +13357,29 @@ export type StudyRotateShareTokenResponse = {
12025
13357
  "cellPlan"?: "crossed" | "per_setup";
12026
13358
  "skillDisclosure"?: "withheld" | "named";
12027
13359
  "modeOptions"?: Record<string, unknown>;
13360
+ "screens"?: Array<"test_awareness">;
13361
+ "runCounting"?: {
13362
+ "policy": "hold_back_screened";
13363
+ "screen": "test_awareness";
13364
+ "levels": Array<"aware" | "influenced">;
13365
+ } | null;
13366
+ "seriesKey"?: string | null;
13367
+ "ref"?: {
13368
+ "commit"?: string;
13369
+ "branch"?: string;
13370
+ "pr"?: number;
13371
+ "url"?: string;
13372
+ } | null;
13373
+ "gradingRevision"?: number;
13374
+ "gradingRevisedAt"?: string | null;
13375
+ "rehearsal"?: true;
13376
+ "rehearsalMode"?: string;
13377
+ "behaviors"?: Array<{
13378
+ "key": string;
13379
+ "name": string;
13380
+ "question": string;
13381
+ "version": number;
13382
+ }>;
12028
13383
  };
12029
13384
  "shareUrl": string | null;
12030
13385
  "shareLinkKept": boolean;
@@ -12251,6 +13606,8 @@ export type StudyUnpublishResponse = {
12251
13606
  "startsPerMinute"?: number;
12252
13607
  } | null;
12253
13608
  "relaunchable"?: boolean;
13609
+ "archivedAt"?: string | null;
13610
+ "archivedBy"?: string | null;
12254
13611
  "arms"?: Array<{
12255
13612
  "key": string;
12256
13613
  "position": number;
@@ -12289,8 +13646,93 @@ export type StudyUnpublishResponse = {
12289
13646
  "cellPlan"?: "crossed" | "per_setup";
12290
13647
  "skillDisclosure"?: "withheld" | "named";
12291
13648
  "modeOptions"?: Record<string, unknown>;
13649
+ "screens"?: Array<"test_awareness">;
13650
+ "runCounting"?: {
13651
+ "policy": "hold_back_screened";
13652
+ "screen": "test_awareness";
13653
+ "levels": Array<"aware" | "influenced">;
13654
+ } | null;
13655
+ "seriesKey"?: string | null;
13656
+ "ref"?: {
13657
+ "commit"?: string;
13658
+ "branch"?: string;
13659
+ "pr"?: number;
13660
+ "url"?: string;
13661
+ } | null;
13662
+ "gradingRevision"?: number;
13663
+ "gradingRevisedAt"?: string | null;
13664
+ "rehearsal"?: true;
13665
+ "rehearsalMode"?: string;
13666
+ "behaviors"?: Array<{
13667
+ "key": string;
13668
+ "name": string;
13669
+ "question": string;
13670
+ "version": number;
13671
+ }>;
13672
+ };
13673
+ };
13674
+ export type StudyViewsResponse = {
13675
+ "views": Array<{
13676
+ "id": string;
13677
+ "name": string;
13678
+ "query": unknown;
13679
+ "createdBy": string | null;
13680
+ "createdAt": string;
13681
+ }>;
13682
+ };
13683
+ export type StudySaveViewRequest = {
13684
+ "name": string;
13685
+ "query": {
13686
+ "metrics": Array<string>;
13687
+ "groupBy"?: Array<string>;
13688
+ "filters"?: Array<{
13689
+ "field": string;
13690
+ "in": Array<string>;
13691
+ }>;
13692
+ "limit"?: number;
12292
13693
  };
12293
13694
  };
13695
+ export type StudySaveViewResponse = {
13696
+ "view": {
13697
+ "id": string;
13698
+ "name": string;
13699
+ "query": unknown;
13700
+ "createdBy": string | null;
13701
+ "createdAt": string;
13702
+ };
13703
+ };
13704
+ export type StudyRunViewResponse = {
13705
+ "view": {
13706
+ "id": string;
13707
+ "name": string;
13708
+ "query": unknown;
13709
+ "createdBy": string | null;
13710
+ "createdAt": string;
13711
+ };
13712
+ "result": {
13713
+ "basis": "runs";
13714
+ "matchedRuns": number;
13715
+ "groupBy": Array<{
13716
+ "id": string;
13717
+ "label": string;
13718
+ }>;
13719
+ "metrics": Array<{
13720
+ "id": string;
13721
+ "label": string;
13722
+ "unit": "count" | "rate" | "points" | "usd" | "tokens" | "ms";
13723
+ "fold": string;
13724
+ }>;
13725
+ "rows": Array<{
13726
+ "key": Record<string, unknown>;
13727
+ "runs": number;
13728
+ "cells": Record<string, unknown>;
13729
+ }>;
13730
+ "truncated": boolean;
13731
+ };
13732
+ };
13733
+ export type StudyDeleteViewResponse = {
13734
+ "deleted": true;
13735
+ };
12294
13736
  export type StudySetVisibilityRequest = {
12295
13737
  "visibility": "private" | "org" | "link" | "public";
12296
13738
  };
@@ -12429,6 +13871,8 @@ export type StudySetVisibilityResponse = {
12429
13871
  "startsPerMinute"?: number;
12430
13872
  } | null;
12431
13873
  "relaunchable"?: boolean;
13874
+ "archivedAt"?: string | null;
13875
+ "archivedBy"?: string | null;
12432
13876
  "arms"?: Array<{
12433
13877
  "key": string;
12434
13878
  "position": number;
@@ -12467,6 +13911,29 @@ export type StudySetVisibilityResponse = {
12467
13911
  "cellPlan"?: "crossed" | "per_setup";
12468
13912
  "skillDisclosure"?: "withheld" | "named";
12469
13913
  "modeOptions"?: Record<string, unknown>;
13914
+ "screens"?: Array<"test_awareness">;
13915
+ "runCounting"?: {
13916
+ "policy": "hold_back_screened";
13917
+ "screen": "test_awareness";
13918
+ "levels": Array<"aware" | "influenced">;
13919
+ } | null;
13920
+ "seriesKey"?: string | null;
13921
+ "ref"?: {
13922
+ "commit"?: string;
13923
+ "branch"?: string;
13924
+ "pr"?: number;
13925
+ "url"?: string;
13926
+ } | null;
13927
+ "gradingRevision"?: number;
13928
+ "gradingRevisedAt"?: string | null;
13929
+ "rehearsal"?: true;
13930
+ "rehearsalMode"?: string;
13931
+ "behaviors"?: Array<{
13932
+ "key": string;
13933
+ "name": string;
13934
+ "question": string;
13935
+ "version": number;
13936
+ }>;
12470
13937
  };
12471
13938
  "shareUrl": string | null;
12472
13939
  "shareLinkKept": boolean;