retab 0.1.9 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/lib/retab/base_client.rb +1 -5
  3. data/lib/retab/classifications.rb +1 -1
  4. data/lib/retab/edit_templates.rb +1 -1
  5. data/lib/retab/edits.rb +1 -1
  6. data/lib/retab/experiment_run_metrics/experiment_metrics_stale_error.rb +3 -3
  7. data/lib/retab/experiment_run_metrics/experiment_summary_metrics_response.rb +3 -3
  8. data/lib/retab/experiment_run_metrics/metrics_stale_error_last_run.rb +3 -3
  9. data/lib/retab/experiment_run_results/error_workflow_experiment_result.rb +1 -22
  10. data/lib/retab/experiment_runs/create_experiment_run_request.rb +5 -2
  11. data/lib/retab/experiment_runs/error_workflow_experiment_run.rb +1 -1
  12. data/lib/retab/experiment_runs/experiment_run.rb +3 -3
  13. data/lib/retab/experiment_runs.rb +4 -1
  14. data/lib/retab/extractions/extraction.rb +0 -3
  15. data/lib/retab/extractions/extraction_request.rb +0 -3
  16. data/lib/retab/extractions.rb +53 -4
  17. data/lib/retab/files/create_file_blueprint_request.rb +3 -3
  18. data/lib/retab/files/file_blueprint.rb +5 -5
  19. data/lib/retab/files.rb +3 -3
  20. data/lib/retab/mime.rb +77 -6
  21. data/lib/retab/parses/parse.rb +0 -3
  22. data/lib/retab/parses/parse_request.rb +0 -3
  23. data/lib/retab/parses.rb +1 -4
  24. data/lib/retab/partitions.rb +1 -1
  25. data/lib/retab/schemas/generate_schema_request.rb +0 -3
  26. data/lib/retab/schemas.rb +1 -4
  27. data/lib/retab/splits.rb +1 -1
  28. data/lib/retab/types/{workflow_test_run_results_order.rb → eval_run_freshness_reasons.rb} +1 -1
  29. data/lib/retab/types/eval_run_freshness_status.rb +9 -0
  30. data/lib/retab/types/eval_run_trigger_type.rb +18 -0
  31. data/lib/retab/types/{latest_block_test_run_summary_outcome.rb → latest_block_eval_run_summary_outcome.rb} +1 -1
  32. data/lib/retab/types/{latest_block_test_run_summary_status.rb → latest_block_eval_run_summary_status.rb} +1 -1
  33. data/lib/retab/types/trigger_info_type.rb +1 -10
  34. data/lib/retab/types/{workflow_test_runs_order.rb → workflow_eval_result_verdict.rb} +1 -1
  35. data/lib/retab/types/workflow_eval_run_results_order.rb +9 -0
  36. data/lib/retab/types/{workflow_test_result_verdict.rb → workflow_eval_runs_order.rb} +1 -1
  37. data/lib/retab/types/workflow_eval_schema_drift.rb +9 -0
  38. data/lib/retab/types/{workflow_tests_order.rb → workflow_evals_order.rb} +1 -1
  39. data/lib/retab/types/workflow_experiment_freshness_state.rb +9 -0
  40. data/lib/retab/types/workflow_experiment_run_plan_mode.rb +15 -0
  41. data/lib/retab/types/workflow_experiments_exclude_status.rb +1 -1
  42. data/lib/retab/types/workflow_experiments_status.rb +1 -1
  43. data/lib/retab/version.rb +1 -1
  44. data/lib/retab/workflow_artifacts/extraction_workflow_artifact.rb +0 -3
  45. data/lib/retab/workflow_artifacts/parse_workflow_artifact.rb +0 -3
  46. data/lib/retab/workflow_artifacts/workflow_artifact.rb +5 -2
  47. data/lib/retab/workflow_block_executions/stored_block_execution.rb +3 -3
  48. data/lib/retab/workflow_edges.rb +6 -0
  49. data/lib/retab/{workflow_test_run_results/cancelled_workflow_test_run.rb → workflow_eval_run_results/cancelled_workflow_eval_run.rb} +1 -1
  50. data/lib/retab/{workflow_test_run_results/completed_workflow_test_run.rb → workflow_eval_run_results/completed_workflow_eval_run.rb} +1 -1
  51. data/lib/retab/workflow_eval_run_results/error_workflow_eval_run.rb +28 -0
  52. data/lib/retab/workflow_eval_run_results/pending_workflow_eval_run.rb +7 -0
  53. data/lib/retab/{workflow_test_run_results/error_workflow_test_run.rb → workflow_eval_run_results/queued_workflow_eval_run.rb} +1 -1
  54. data/lib/retab/{workflow_test_run_results/workflow_test_run_timing.rb → workflow_eval_run_results/result_file_ref.rb} +1 -1
  55. data/lib/retab/workflow_eval_run_results/running_workflow_eval_run.rb +7 -0
  56. data/lib/retab/{workflow_test_run_results/workflow_test_result.rb → workflow_eval_run_results/workflow_eval_result.rb} +17 -17
  57. data/lib/retab/workflow_eval_run_results/workflow_eval_run_timing.rb +7 -0
  58. data/lib/retab/{workflow_test_run_results.rb → workflow_eval_run_results.rb} +10 -10
  59. data/lib/retab/{workflow_test_runs/block_test_batch_execution_counts.rb → workflow_eval_runs/block_eval_batch_execution_counts.rb} +3 -3
  60. data/lib/retab/{workflow_test_runs/block_test_lifecycle_counts.rb → workflow_eval_runs/block_eval_lifecycle_counts.rb} +1 -1
  61. data/lib/retab/{workflow_test_runs/block_test_outcome_counts.rb → workflow_eval_runs/block_eval_outcome_counts.rb} +1 -1
  62. data/lib/retab/{workflow_test_runs/create_workflow_test_run_request.rb → workflow_eval_runs/create_workflow_eval_run_request.rb} +4 -4
  63. data/lib/retab/{workflow_tests/workflow_test_block_target.rb → workflow_eval_runs/eval_run_block_target.rb} +1 -1
  64. data/lib/retab/workflow_eval_runs/eval_run_freshness.rb +34 -0
  65. data/lib/retab/{workflow_test_runs/workflow_test_run_workflow_scope.rb → workflow_eval_runs/eval_run_trigger.rb} +1 -1
  66. data/lib/retab/{workflow_test_runs/workflow_test_run.rb → workflow_eval_runs/workflow_eval_run.rb} +18 -18
  67. data/lib/retab/{workflow_test_runs/workflow_test_run_block_scope.rb → workflow_eval_runs/workflow_eval_run_block_scope.rb} +1 -1
  68. data/lib/retab/{workflow_test_runs/workflow_test_run_single_scope.rb → workflow_eval_runs/workflow_eval_run_single_scope.rb} +4 -4
  69. data/lib/retab/workflow_eval_runs/workflow_eval_run_workflow_scope.rb +20 -0
  70. data/lib/retab/{workflow_test_runs.rb → workflow_eval_runs.rb} +24 -24
  71. data/lib/retab/{workflow_tests/create_workflow_test_request.rb → workflow_evals/create_workflow_eval_request.rb} +4 -4
  72. data/lib/retab/workflow_evals/eval_file_handle_input.rb +7 -0
  73. data/lib/retab/workflow_evals/eval_json_handle_input.rb +7 -0
  74. data/lib/retab/workflow_evals/eval_public_file_ref.rb +7 -0
  75. data/lib/retab/{workflow_tests/latest_block_test_run_summary.rb → workflow_evals/latest_block_eval_run_summary.rb} +4 -4
  76. data/lib/retab/{workflow_tests/manual_workflow_test_source.rb → workflow_evals/manual_workflow_eval_source.rb} +1 -1
  77. data/lib/retab/{workflow_tests/run_step_workflow_test_source.rb → workflow_evals/run_step_workflow_eval_source.rb} +1 -1
  78. data/lib/retab/{workflow_tests/update_workflow_test_request.rb → workflow_evals/update_workflow_eval_request.rb} +3 -3
  79. data/lib/retab/{workflow_tests/workflow_test.rb → workflow_evals/workflow_eval.rb} +7 -7
  80. data/lib/retab/workflow_evals/workflow_eval_block_target.rb +7 -0
  81. data/lib/retab/{workflow_tests.rb → workflow_evals.rb} +31 -31
  82. data/lib/retab/workflow_experiments/workflow_experiment.rb +12 -0
  83. data/lib/retab/workflow_runs.rb +1 -1
  84. data/lib/retab/workflows.rb +2 -2
  85. data/lib/retab.rb +3 -3
  86. data/rbi/retab/{block_test_batch_execution_counts.rbi → block_eval_batch_execution_counts.rbi} +5 -5
  87. data/rbi/retab/{block_test_lifecycle_counts.rbi → block_eval_lifecycle_counts.rbi} +1 -1
  88. data/rbi/retab/{block_test_outcome_counts.rbi → block_eval_outcome_counts.rbi} +1 -1
  89. data/rbi/retab/{cancelled_workflow_test_run.rbi → cancelled_workflow_eval_run.rbi} +1 -1
  90. data/rbi/retab/classification_request.rbi +2 -2
  91. data/rbi/retab/{completed_workflow_test_run.rbi → completed_workflow_eval_run.rbi} +1 -1
  92. data/rbi/retab/create_experiment_run_request.rbi +6 -0
  93. data/rbi/retab/create_file_blueprint_request.rbi +4 -4
  94. data/rbi/retab/{create_workflow_test_request.rbi → create_workflow_eval_request.rbi} +5 -5
  95. data/rbi/retab/{create_workflow_test_run_request.rbi → create_workflow_eval_run_request.rbi} +3 -3
  96. data/rbi/retab/create_workflow_run_request.rbi +2 -2
  97. data/rbi/retab/edit_request.rbi +2 -2
  98. data/rbi/retab/{error_workflow_test_run.rbi → error_workflow_eval_run.rbi} +1 -1
  99. data/rbi/retab/eval_file_handle_input.rbi +30 -0
  100. data/rbi/retab/eval_json_handle_input.rbi +30 -0
  101. data/rbi/retab/eval_public_file_ref.rbi +36 -0
  102. data/rbi/retab/{workflow_test_block_target.rbi → eval_run_block_target.rbi} +1 -1
  103. data/rbi/retab/eval_run_freshness.rbi +48 -0
  104. data/rbi/retab/{workflow_test_run_workflow_scope.rbi → eval_run_trigger.rbi} +1 -1
  105. data/rbi/retab/experiment_metrics_stale_error.rbi +2 -2
  106. data/rbi/retab/experiment_run.rbi +2 -2
  107. data/rbi/retab/experiment_runs.rbi +2 -1
  108. data/rbi/retab/experiment_summary_metrics_response.rbi +2 -2
  109. data/rbi/retab/extraction.rbi +0 -6
  110. data/rbi/retab/extraction_request.rbi +2 -8
  111. data/rbi/retab/extraction_workflow_artifact.rbi +0 -6
  112. data/rbi/retab/extractions.rbi +19 -2
  113. data/rbi/retab/file_blueprint.rbi +6 -6
  114. data/rbi/retab/file_handle_input.rbi +2 -2
  115. data/rbi/retab/files.rbi +2 -2
  116. data/rbi/retab/generate_schema_request.rbi +0 -6
  117. data/rbi/retab/{latest_block_test_run_summary.rbi → latest_block_eval_run_summary.rbi} +3 -3
  118. data/rbi/retab/{manual_workflow_test_source.rbi → manual_workflow_eval_source.rbi} +3 -3
  119. data/rbi/retab/metrics_stale_error_last_run.rbi +2 -2
  120. data/rbi/retab/parse.rbi +0 -6
  121. data/rbi/retab/parse_request.rbi +2 -8
  122. data/rbi/retab/parse_workflow_artifact.rbi +0 -6
  123. data/rbi/retab/parses.rbi +1 -2
  124. data/rbi/retab/partition_request.rbi +2 -2
  125. data/rbi/retab/{pending_workflow_test_run.rbi → pending_workflow_eval_run.rbi} +1 -1
  126. data/rbi/retab/{queued_workflow_test_run.rbi → queued_workflow_eval_run.rbi} +1 -1
  127. data/rbi/retab/result_file_ref.rbi +36 -0
  128. data/rbi/retab/{run_step_workflow_test_source.rbi → run_step_workflow_eval_source.rbi} +1 -1
  129. data/rbi/retab/{running_workflow_test_run.rbi → running_workflow_eval_run.rbi} +1 -1
  130. data/rbi/retab/schemas.rbi +1 -2
  131. data/rbi/retab/split_request.rbi +2 -2
  132. data/rbi/retab/stored_block_execution.rbi +2 -2
  133. data/rbi/retab/{update_workflow_test_request.rbi → update_workflow_eval_request.rbi} +3 -3
  134. data/rbi/retab/workflow_artifact.rbi +6 -0
  135. data/rbi/retab/workflow_edges.rbi +2 -1
  136. data/rbi/retab/{workflow_test.rbi → workflow_eval.rbi} +11 -11
  137. data/rbi/retab/workflow_eval_block_target.rbi +30 -0
  138. data/rbi/retab/{workflow_test_result.rbi → workflow_eval_result.rbi} +11 -11
  139. data/rbi/retab/workflow_eval_run.rbi +84 -0
  140. data/rbi/retab/{workflow_test_run_block_scope.rbi → workflow_eval_run_block_scope.rbi} +1 -1
  141. data/rbi/retab/{workflow_test_run_results.rbi → workflow_eval_run_results.rbi} +3 -3
  142. data/rbi/retab/{workflow_test_run_single_scope.rbi → workflow_eval_run_single_scope.rbi} +3 -3
  143. data/rbi/retab/{workflow_test_run_timing.rbi → workflow_eval_run_timing.rbi} +1 -1
  144. data/rbi/retab/workflow_eval_run_workflow_scope.rbi +24 -0
  145. data/rbi/retab/{workflow_test_runs.rbi → workflow_eval_runs.rbi} +8 -8
  146. data/rbi/retab/{workflow_tests.rbi → workflow_evals.rbi} +16 -16
  147. data/rbi/retab/workflow_experiment.rbi +24 -0
  148. data/rbi/retab/workflows.rbi +2 -2
  149. metadata +111 -92
  150. data/lib/retab/types/workflow_test_schema_drift.rb +0 -9
  151. data/lib/retab/workflow_test_run_results/pending_workflow_test_run.rb +0 -7
  152. data/lib/retab/workflow_test_run_results/queued_workflow_test_run.rb +0 -7
  153. data/lib/retab/workflow_test_run_results/running_workflow_test_run.rb +0 -7
  154. data/rbi/retab/workflow_test_run.rbi +0 -84
  155. /data/lib/retab/{workflow_test_run_results → workflow_eval_run_results}/assertion_failure.rb +0 -0
  156. /data/lib/retab/{workflow_test_run_results → workflow_eval_run_results}/assertion_result.rb +0 -0
  157. /data/lib/retab/{workflow_experiments → workflow_eval_run_results}/file_handle_input.rb +0 -0
  158. /data/lib/retab/{workflow_experiments → workflow_eval_run_results}/json_handle_input.rb +0 -0
  159. /data/lib/retab/{workflow_test_run_results → workflow_eval_run_results}/verdict_summary.rb +0 -0
  160. /data/lib/retab/{workflow_tests → workflow_evals}/all_items_match_condition.rb +0 -0
  161. /data/lib/retab/{workflow_tests → workflow_evals}/any_item_matches_condition.rb +0 -0
  162. /data/lib/retab/{workflow_tests → workflow_evals}/array_contains_condition.rb +0 -0
  163. /data/lib/retab/{workflow_experiments → workflow_evals}/artifact_drift.rb +0 -0
  164. /data/lib/retab/{workflow_experiments → workflow_evals}/artifact_freshness.rb +0 -0
  165. /data/lib/retab/{workflow_tests → workflow_evals}/assertion_schema_dep.rb +0 -0
  166. /data/lib/retab/{workflow_tests → workflow_evals}/assertion_spec.rb +0 -0
  167. /data/lib/retab/{workflow_tests → workflow_evals}/between_condition.rb +0 -0
  168. /data/lib/retab/{workflow_tests → workflow_evals}/contain_condition.rb +0 -0
  169. /data/lib/retab/{workflow_tests → workflow_evals}/ends_with_condition.rb +0 -0
  170. /data/lib/retab/{workflow_tests → workflow_evals}/equal_condition.rb +0 -0
  171. /data/lib/retab/{workflow_tests → workflow_evals}/exist_condition.rb +0 -0
  172. /data/lib/retab/{workflow_tests → workflow_evals}/json_schema_valid_condition.rb +0 -0
  173. /data/lib/retab/{workflow_tests → workflow_evals}/length_compare_condition.rb +0 -0
  174. /data/lib/retab/{workflow_tests → workflow_evals}/llm_judged_as_condition.rb +0 -0
  175. /data/lib/retab/{workflow_tests → workflow_evals}/llm_not_judged_as_condition.rb +0 -0
  176. /data/lib/retab/{workflow_tests → workflow_evals}/matche_regex_condition.rb +0 -0
  177. /data/lib/retab/{workflow_tests → workflow_evals}/not_contains_condition.rb +0 -0
  178. /data/lib/retab/{workflow_tests → workflow_evals}/not_equals_condition.rb +0 -0
  179. /data/lib/retab/{workflow_tests → workflow_evals}/not_exists_condition.rb +0 -0
  180. /data/lib/retab/{workflow_tests → workflow_evals}/number_compare_condition.rb +0 -0
  181. /data/lib/retab/{workflow_tests → workflow_evals}/object_contains_condition.rb +0 -0
  182. /data/lib/retab/{workflow_tests → workflow_evals}/output_target.rb +0 -0
  183. /data/lib/retab/{workflow_tests → workflow_evals}/similarity_gte_condition.rb +0 -0
  184. /data/lib/retab/{workflow_tests → workflow_evals}/split_iou_condition.rb +0 -0
  185. /data/lib/retab/{workflow_tests → workflow_evals}/start_with_condition.rb +0 -0
@@ -5,28 +5,28 @@
5
5
  require "json"
6
6
 
7
7
  module Retab
8
- class WorkflowTests
8
+ class WorkflowEvals
9
9
  def initialize(client)
10
10
  @client = client
11
11
  end
12
12
 
13
13
  def results
14
- @results ||= Retab::WorkflowTestRunResults.new(@client)
14
+ @results ||= Retab::WorkflowEvalRunResults.new(@client)
15
15
  end
16
16
 
17
17
  def runs
18
- @runs ||= Retab::WorkflowTestRuns.new(@client)
18
+ @runs ||= Retab::WorkflowEvalRuns.new(@client)
19
19
  end
20
20
 
21
- # List Tests
21
+ # List Workflow Evals
22
22
  # @param workflow_id [String]
23
23
  # @param target_block_id [String, nil]
24
24
  # @param before [String, nil]
25
25
  # @param after [String, nil]
26
26
  # @param limit [Integer, nil]
27
- # @param order [Retab::Types::WorkflowTestsOrder, nil]
27
+ # @param order [Retab::Types::WorkflowEvalsOrder, nil]
28
28
  # @param request_options [Hash] (see Retab::Types::RequestOptions)
29
- # @return [Retab::PaginatedList<Retab::WorkflowTest>]
29
+ # @return [Retab::PaginatedList<Retab::WorkflowEval>]
30
30
  def list(
31
31
  workflow_id:,
32
32
  target_block_id: nil,
@@ -46,7 +46,7 @@ module Retab
46
46
  }.compact
47
47
  response = @client.request(
48
48
  method: :get,
49
- path: "/v1/workflows/tests",
49
+ path: "/v1/workflows/evals",
50
50
  auth: true,
51
51
  params: params,
52
52
  request_options: request_options
@@ -64,7 +64,7 @@ module Retab
64
64
  }
65
65
  Retab::PaginatedList.from_response(
66
66
  response,
67
- model: Retab::WorkflowTest,
67
+ model: Retab::WorkflowEval,
68
68
  filters: {
69
69
  workflow_id: workflow_id,
70
70
  target_block_id: target_block_id,
@@ -76,14 +76,14 @@ module Retab
76
76
  )
77
77
  end
78
78
 
79
- # Create Test
79
+ # Create Workflow Eval
80
80
  # @param workflow_id [String]
81
- # @param target [Retab::WorkflowTestBlockTarget]
82
- # @param source [Retab::ManualWorkflowTestSource, Retab::RunStepWorkflowTestSource]
81
+ # @param target [Retab::WorkflowEvalBlockTarget]
82
+ # @param source [Retab::ManualWorkflowEvalSource, Retab::RunStepWorkflowEvalSource]
83
83
  # @param name [String, nil]
84
84
  # @param assertion [Retab::AssertionSpec]
85
85
  # @param request_options [Hash] (see Retab::Types::RequestOptions)
86
- # @return [Retab::WorkflowTest]
86
+ # @return [Retab::WorkflowEval]
87
87
  def create(
88
88
  workflow_id:,
89
89
  target:,
@@ -101,12 +101,12 @@ module Retab
101
101
  }.compact
102
102
  response = @client.request(
103
103
  method: :post,
104
- path: "/v1/workflows/tests",
104
+ path: "/v1/workflows/evals",
105
105
  auth: true,
106
106
  body: body,
107
107
  request_options: request_options
108
108
  )
109
- result = Retab::WorkflowTest.new(response.body)
109
+ result = Retab::WorkflowEval.new(response.body)
110
110
  result.last_response = Retab::Types::ApiResponse.new(
111
111
  http_status: response.code.to_i,
112
112
  http_headers: response.each_header.to_h,
@@ -115,21 +115,21 @@ module Retab
115
115
  result
116
116
  end
117
117
 
118
- # Get Test
119
- # @param test_id [String]
118
+ # Get Workflow Eval
119
+ # @param eval_id [String]
120
120
  # @param request_options [Hash] (see Retab::Types::RequestOptions)
121
- # @return [Retab::WorkflowTest]
121
+ # @return [Retab::WorkflowEval]
122
122
  def get(
123
- test_id:,
123
+ eval_id:,
124
124
  request_options: {}
125
125
  )
126
126
  response = @client.request(
127
127
  method: :get,
128
- path: "/v1/workflows/tests/#{Retab::Util.encode_path(test_id)}",
128
+ path: "/v1/workflows/evals/#{Retab::Util.encode_path(eval_id)}",
129
129
  auth: true,
130
130
  request_options: request_options
131
131
  )
132
- result = Retab::WorkflowTest.new(response.body)
132
+ result = Retab::WorkflowEval.new(response.body)
133
133
  result.last_response = Retab::Types::ApiResponse.new(
134
134
  http_status: response.code.to_i,
135
135
  http_headers: response.each_header.to_h,
@@ -138,15 +138,15 @@ module Retab
138
138
  result
139
139
  end
140
140
 
141
- # Update Test
142
- # @param test_id [String]
141
+ # Update Workflow Eval
142
+ # @param eval_id [String]
143
143
  # @param name [String, nil]
144
144
  # @param assertion [Retab::AssertionSpec, nil]
145
- # @param source [Retab::ManualWorkflowTestSource, Retab::RunStepWorkflowTestSource, nil]
145
+ # @param source [Retab::ManualWorkflowEvalSource, Retab::RunStepWorkflowEvalSource, nil]
146
146
  # @param request_options [Hash] (see Retab::Types::RequestOptions)
147
- # @return [Retab::WorkflowTest]
147
+ # @return [Retab::WorkflowEval]
148
148
  def update(
149
- test_id:,
149
+ eval_id:,
150
150
  name: nil,
151
151
  assertion: nil,
152
152
  source: nil,
@@ -159,12 +159,12 @@ module Retab
159
159
  }.compact
160
160
  response = @client.request(
161
161
  method: :patch,
162
- path: "/v1/workflows/tests/#{Retab::Util.encode_path(test_id)}",
162
+ path: "/v1/workflows/evals/#{Retab::Util.encode_path(eval_id)}",
163
163
  auth: true,
164
164
  body: body,
165
165
  request_options: request_options
166
166
  )
167
- result = Retab::WorkflowTest.new(response.body)
167
+ result = Retab::WorkflowEval.new(response.body)
168
168
  result.last_response = Retab::Types::ApiResponse.new(
169
169
  http_status: response.code.to_i,
170
170
  http_headers: response.each_header.to_h,
@@ -173,17 +173,17 @@ module Retab
173
173
  result
174
174
  end
175
175
 
176
- # Delete Test
177
- # @param test_id [String]
176
+ # Delete Workflow Eval
177
+ # @param eval_id [String]
178
178
  # @param request_options [Hash] (see Retab::Types::RequestOptions)
179
179
  # @return [void]
180
180
  def delete(
181
- test_id:,
181
+ eval_id:,
182
182
  request_options: {}
183
183
  )
184
184
  @client.request(
185
185
  method: :delete,
186
- path: "/v1/workflows/tests/#{Retab::Util.encode_path(test_id)}",
186
+ path: "/v1/workflows/evals/#{Retab::Util.encode_path(eval_id)}",
187
187
  auth: true,
188
188
  request_options: request_options
189
189
  )
@@ -20,6 +20,10 @@ module Retab
20
20
  score: :score,
21
21
  is_stale: :is_stale,
22
22
  freshness: :freshness,
23
+ freshness_state: :freshness_state,
24
+ freshness_reasons: :freshness_reasons,
25
+ run_plan_mode: :run_plan_mode,
26
+ rerunnable_document_count: :rerunnable_document_count,
23
27
  schema_drift: :schema_drift,
24
28
  schema_drift_detail: :schema_drift_detail,
25
29
  drift: :drift
@@ -40,6 +44,10 @@ module Retab
40
44
  :score,
41
45
  :is_stale,
42
46
  :freshness,
47
+ :freshness_state,
48
+ :freshness_reasons,
49
+ :run_plan_mode,
50
+ :rerunnable_document_count,
43
51
  :schema_drift,
44
52
  :schema_drift_detail,
45
53
  :drift
@@ -62,6 +70,10 @@ module Retab
62
70
  @score = hash[:score]
63
71
  @is_stale = hash[:is_stale].nil? ? false : hash[:is_stale]
64
72
  @freshness = hash[:freshness] ? Retab::ArtifactFreshness.new(hash[:freshness]) : nil
73
+ @freshness_state = hash[:freshness_state].nil? ? "unknown" : hash[:freshness_state]
74
+ @freshness_reasons = (hash[:freshness_reasons] || [])
75
+ @run_plan_mode = hash[:run_plan_mode].nil? ? "unknown" : hash[:run_plan_mode]
76
+ @rerunnable_document_count = hash[:rerunnable_document_count]
65
77
  @schema_drift = hash[:schema_drift].nil? ? "unknown" : hash[:schema_drift]
66
78
  @schema_drift_detail = hash[:schema_drift_detail]
67
79
  @drift = hash[:drift] ? Retab::ArtifactDrift.new(hash[:drift]) : nil
@@ -122,7 +122,7 @@ module Retab
122
122
  version: nil,
123
123
  request_options: {}
124
124
  )
125
- documents = Retab::MimeData.coerce_document_map(documents) unless documents.nil?
125
+ documents = Retab::MimeData.coerce_document_map(documents, client: @client) unless documents.nil?
126
126
  body = {
127
127
  "workflow_id" => workflow_id,
128
128
  "documents" => documents,
@@ -42,8 +42,8 @@ module Retab
42
42
  @steps ||= Retab::WorkflowSteps.new(@client)
43
43
  end
44
44
 
45
- def tests
46
- @tests ||= Retab::WorkflowTests.new(@client)
45
+ def evals
46
+ @evals ||= Retab::WorkflowEvals.new(@client)
47
47
  end
48
48
 
49
49
  # List Workflows
data/lib/retab.rb CHANGED
@@ -29,15 +29,15 @@ loader.collapse("#{__dir__}/retab/workflow_artifacts")
29
29
  loader.collapse("#{__dir__}/retab/workflow_block_executions")
30
30
  loader.collapse("#{__dir__}/retab/workflow_blocks")
31
31
  loader.collapse("#{__dir__}/retab/workflow_edges")
32
+ loader.collapse("#{__dir__}/retab/workflow_eval_run_results")
33
+ loader.collapse("#{__dir__}/retab/workflow_eval_runs")
34
+ loader.collapse("#{__dir__}/retab/workflow_evals")
32
35
  loader.collapse("#{__dir__}/retab/workflow_experiments")
33
36
  loader.collapse("#{__dir__}/retab/workflow_review_versions")
34
37
  loader.collapse("#{__dir__}/retab/workflow_reviews")
35
38
  loader.collapse("#{__dir__}/retab/workflow_runs")
36
39
  loader.collapse("#{__dir__}/retab/workflow_spec")
37
40
  loader.collapse("#{__dir__}/retab/workflow_steps")
38
- loader.collapse("#{__dir__}/retab/workflow_test_run_results")
39
- loader.collapse("#{__dir__}/retab/workflow_test_runs")
40
- loader.collapse("#{__dir__}/retab/workflow_tests")
41
41
  loader.collapse("#{__dir__}/retab/workflows")
42
42
  loader.ignore("#{__dir__}/retab/errors.rb")
43
43
  loader.ignore("#{__dir__}/retab/inflections.rb")
@@ -5,20 +5,20 @@
5
5
  # typed: strong
6
6
 
7
7
  module Retab
8
- class BlockTestBatchExecutionCounts
8
+ class BlockEvalBatchExecutionCounts
9
9
  sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
10
  def initialize(json); end
11
11
 
12
- sig { returns(T.nilable(Retab::BlockTestLifecycleCounts)) }
12
+ sig { returns(T.nilable(Retab::BlockEvalLifecycleCounts)) }
13
13
  def lifecycle_counts; end
14
14
 
15
- sig { params(value: T.nilable(Retab::BlockTestLifecycleCounts)).returns(T.nilable(Retab::BlockTestLifecycleCounts)) }
15
+ sig { params(value: T.nilable(Retab::BlockEvalLifecycleCounts)).returns(T.nilable(Retab::BlockEvalLifecycleCounts)) }
16
16
  def lifecycle_counts=(value); end
17
17
 
18
- sig { returns(T.nilable(Retab::BlockTestOutcomeCounts)) }
18
+ sig { returns(T.nilable(Retab::BlockEvalOutcomeCounts)) }
19
19
  def outcome; end
20
20
 
21
- sig { params(value: T.nilable(Retab::BlockTestOutcomeCounts)).returns(T.nilable(Retab::BlockTestOutcomeCounts)) }
21
+ sig { params(value: T.nilable(Retab::BlockEvalOutcomeCounts)).returns(T.nilable(Retab::BlockEvalOutcomeCounts)) }
22
22
  def outcome=(value); end
23
23
 
24
24
  sig { returns(T::Hash[Symbol, T.untyped]) }
@@ -5,7 +5,7 @@
5
5
  # typed: strong
6
6
 
7
7
  module Retab
8
- class BlockTestLifecycleCounts
8
+ class BlockEvalLifecycleCounts
9
9
  sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
10
  def initialize(json); end
11
11
 
@@ -5,7 +5,7 @@
5
5
  # typed: strong
6
6
 
7
7
  module Retab
8
- class BlockTestOutcomeCounts
8
+ class BlockEvalOutcomeCounts
9
9
  sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
10
  def initialize(json); end
11
11
 
@@ -5,7 +5,7 @@
5
5
  # typed: strong
6
6
 
7
7
  module Retab
8
- class CancelledWorkflowTestRun
8
+ class CancelledWorkflowEvalRun
9
9
  sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
10
  def initialize(json); end
11
11
 
@@ -9,10 +9,10 @@ module Retab
9
9
  sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
10
  def initialize(json); end
11
11
 
12
- sig { returns(T.any(Retab::MimeData, Retab::FileRef)) }
12
+ sig { returns(Retab::MimeData) }
13
13
  def document; end
14
14
 
15
- sig { params(value: T.any(Retab::MimeData, Retab::FileRef)).returns(T.any(Retab::MimeData, Retab::FileRef)) }
15
+ sig { params(value: Retab::MimeData).returns(Retab::MimeData) }
16
16
  def document=(value); end
17
17
 
18
18
  sig { returns(T::Array[Retab::Category]) }
@@ -5,7 +5,7 @@
5
5
  # typed: strong
6
6
 
7
7
  module Retab
8
- class CompletedWorkflowTestRun
8
+ class CompletedWorkflowEvalRun
9
9
  sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
10
  def initialize(json); end
11
11
 
@@ -21,6 +21,12 @@ module Retab
21
21
  sig { params(value: T.nilable(String)).returns(T.nilable(String)) }
22
22
  def workflow_id=(value); end
23
23
 
24
+ sig { returns(T.nilable(String)) }
25
+ def plan_token; end
26
+
27
+ sig { params(value: T.nilable(String)).returns(T.nilable(String)) }
28
+ def plan_token=(value); end
29
+
24
30
  sig { returns(T::Hash[Symbol, T.untyped]) }
25
31
  def to_h; end
26
32
 
@@ -16,16 +16,16 @@ module Retab
16
16
  def file_id=(value); end
17
17
 
18
18
  sig { returns(T.nilable(String)) }
19
- def mode; end
19
+ def intent; end
20
20
 
21
21
  sig { params(value: T.nilable(String)).returns(T.nilable(String)) }
22
- def mode=(value); end
22
+ def intent=(value); end
23
23
 
24
24
  sig { returns(T.nilable(String)) }
25
- def intent; end
25
+ def mode; end
26
26
 
27
27
  sig { params(value: T.nilable(String)).returns(T.nilable(String)) }
28
- def intent=(value); end
28
+ def mode=(value); end
29
29
 
30
30
  sig { returns(T.nilable(T::Boolean)) }
31
31
  def background; end
@@ -5,7 +5,7 @@
5
5
  # typed: strong
6
6
 
7
7
  module Retab
8
- class CreateWorkflowTestRequest
8
+ class CreateWorkflowEvalRequest
9
9
  sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
10
  def initialize(json); end
11
11
 
@@ -15,16 +15,16 @@ module Retab
15
15
  sig { params(value: String).returns(String) }
16
16
  def workflow_id=(value); end
17
17
 
18
- sig { returns(Retab::WorkflowTestBlockTarget) }
18
+ sig { returns(Retab::WorkflowEvalBlockTarget) }
19
19
  def target; end
20
20
 
21
- sig { params(value: Retab::WorkflowTestBlockTarget).returns(Retab::WorkflowTestBlockTarget) }
21
+ sig { params(value: Retab::WorkflowEvalBlockTarget).returns(Retab::WorkflowEvalBlockTarget) }
22
22
  def target=(value); end
23
23
 
24
- sig { returns(T.any(Retab::ManualWorkflowTestSource, Retab::RunStepWorkflowTestSource)) }
24
+ sig { returns(T.any(Retab::ManualWorkflowEvalSource, Retab::RunStepWorkflowEvalSource)) }
25
25
  def source; end
26
26
 
27
- sig { params(value: T.any(Retab::ManualWorkflowTestSource, Retab::RunStepWorkflowTestSource)).returns(T.any(Retab::ManualWorkflowTestSource, Retab::RunStepWorkflowTestSource)) }
27
+ sig { params(value: T.any(Retab::ManualWorkflowEvalSource, Retab::RunStepWorkflowEvalSource)).returns(T.any(Retab::ManualWorkflowEvalSource, Retab::RunStepWorkflowEvalSource)) }
28
28
  def source=(value); end
29
29
 
30
30
  sig { returns(T.nilable(String)) }
@@ -5,7 +5,7 @@
5
5
  # typed: strong
6
6
 
7
7
  module Retab
8
- class CreateWorkflowTestRunRequest
8
+ class CreateWorkflowEvalRunRequest
9
9
  sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
10
  def initialize(json); end
11
11
 
@@ -15,10 +15,10 @@ module Retab
15
15
  sig { params(value: String).returns(String) }
16
16
  def workflow_id=(value); end
17
17
 
18
- sig { returns(T.nilable(T.any(Retab::WorkflowTestRunSingleScope, Retab::WorkflowTestRunWorkflowScope, Retab::WorkflowTestRunBlockScope))) }
18
+ sig { returns(T.nilable(T.any(Retab::WorkflowEvalRunSingleScope, Retab::WorkflowEvalRunWorkflowScope, Retab::WorkflowEvalRunBlockScope))) }
19
19
  def scope; end
20
20
 
21
- sig { params(value: T.nilable(T.any(Retab::WorkflowTestRunSingleScope, Retab::WorkflowTestRunWorkflowScope, Retab::WorkflowTestRunBlockScope))).returns(T.nilable(T.any(Retab::WorkflowTestRunSingleScope, Retab::WorkflowTestRunWorkflowScope, Retab::WorkflowTestRunBlockScope))) }
21
+ sig { params(value: T.nilable(T.any(Retab::WorkflowEvalRunSingleScope, Retab::WorkflowEvalRunWorkflowScope, Retab::WorkflowEvalRunBlockScope))).returns(T.nilable(T.any(Retab::WorkflowEvalRunSingleScope, Retab::WorkflowEvalRunWorkflowScope, Retab::WorkflowEvalRunBlockScope))) }
22
22
  def scope=(value); end
23
23
 
24
24
  sig { returns(T::Hash[Symbol, T.untyped]) }
@@ -15,10 +15,10 @@ module Retab
15
15
  sig { params(value: String).returns(String) }
16
16
  def workflow_id=(value); end
17
17
 
18
- sig { returns(T.nilable(T::Hash[String, T.any(Retab::FileRef, Retab::MimeData)])) }
18
+ sig { returns(T.nilable(T::Hash[String, Retab::MimeData])) }
19
19
  def documents; end
20
20
 
21
- sig { params(value: T.nilable(T::Hash[String, T.any(Retab::FileRef, Retab::MimeData)])).returns(T.nilable(T::Hash[String, T.any(Retab::FileRef, Retab::MimeData)])) }
21
+ sig { params(value: T.nilable(T::Hash[String, Retab::MimeData])).returns(T.nilable(T::Hash[String, Retab::MimeData])) }
22
22
  def documents=(value); end
23
23
 
24
24
  sig { returns(T.nilable(T::Hash[String, T.untyped])) }
@@ -15,10 +15,10 @@ module Retab
15
15
  sig { params(value: String).returns(String) }
16
16
  def instructions=(value); end
17
17
 
18
- sig { returns(T.nilable(T.any(Retab::MimeData, Retab::FileRef))) }
18
+ sig { returns(T.nilable(Retab::MimeData)) }
19
19
  def document; end
20
20
 
21
- sig { params(value: T.nilable(T.any(Retab::MimeData, Retab::FileRef))).returns(T.nilable(T.any(Retab::MimeData, Retab::FileRef))) }
21
+ sig { params(value: T.nilable(Retab::MimeData)).returns(T.nilable(Retab::MimeData)) }
22
22
  def document=(value); end
23
23
 
24
24
  sig { returns(T.nilable(String)) }
@@ -5,7 +5,7 @@
5
5
  # typed: strong
6
6
 
7
7
  module Retab
8
- class ErrorWorkflowTestRun
8
+ class ErrorWorkflowEvalRun
9
9
  sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
10
  def initialize(json); end
11
11
 
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This file is auto-generated by oagen. Do not edit.
4
+
5
+ # typed: strong
6
+
7
+ module Retab
8
+ class EvalFileHandleInput
9
+ sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
+ def initialize(json); end
11
+
12
+ sig { returns(Retab::EvalPublicFileRef) }
13
+ def document; end
14
+
15
+ sig { params(value: Retab::EvalPublicFileRef).returns(Retab::EvalPublicFileRef) }
16
+ def document=(value); end
17
+
18
+ sig { returns(T.nilable(String)) }
19
+ def type; end
20
+
21
+ sig { params(value: T.nilable(String)).returns(T.nilable(String)) }
22
+ def type=(value); end
23
+
24
+ sig { returns(T::Hash[Symbol, T.untyped]) }
25
+ def to_h; end
26
+
27
+ sig { params(args: T.untyped).returns(String) }
28
+ def to_json(*args); end
29
+ end
30
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This file is auto-generated by oagen. Do not edit.
4
+
5
+ # typed: strong
6
+
7
+ module Retab
8
+ class EvalJsonHandleInput
9
+ sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
+ def initialize(json); end
11
+
12
+ sig { returns(T.nilable(T.untyped)) }
13
+ def data; end
14
+
15
+ sig { params(value: T.nilable(T.untyped)).returns(T.nilable(T.untyped)) }
16
+ def data=(value); end
17
+
18
+ sig { returns(T.nilable(String)) }
19
+ def type; end
20
+
21
+ sig { params(value: T.nilable(String)).returns(T.nilable(String)) }
22
+ def type=(value); end
23
+
24
+ sig { returns(T::Hash[Symbol, T.untyped]) }
25
+ def to_h; end
26
+
27
+ sig { params(args: T.untyped).returns(String) }
28
+ def to_json(*args); end
29
+ end
30
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This file is auto-generated by oagen. Do not edit.
4
+
5
+ # typed: strong
6
+
7
+ module Retab
8
+ class EvalPublicFileRef
9
+ sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
+ def initialize(json); end
11
+
12
+ sig { returns(String) }
13
+ def id; end
14
+
15
+ sig { params(value: String).returns(String) }
16
+ def id=(value); end
17
+
18
+ sig { returns(String) }
19
+ def filename; end
20
+
21
+ sig { params(value: String).returns(String) }
22
+ def filename=(value); end
23
+
24
+ sig { returns(String) }
25
+ def mime_type; end
26
+
27
+ sig { params(value: String).returns(String) }
28
+ def mime_type=(value); end
29
+
30
+ sig { returns(T::Hash[Symbol, T.untyped]) }
31
+ def to_h; end
32
+
33
+ sig { params(args: T.untyped).returns(String) }
34
+ def to_json(*args); end
35
+ end
36
+ end
@@ -5,7 +5,7 @@
5
5
  # typed: strong
6
6
 
7
7
  module Retab
8
- class WorkflowTestBlockTarget
8
+ class EvalRunBlockTarget
9
9
  sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
10
  def initialize(json); end
11
11
 
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This file is auto-generated by oagen. Do not edit.
4
+
5
+ # typed: strong
6
+
7
+ module Retab
8
+ class EvalRunFreshness
9
+ sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
+ def initialize(json); end
11
+
12
+ sig { returns(T.nilable(String)) }
13
+ def status; end
14
+
15
+ sig { params(value: T.nilable(String)).returns(T.nilable(String)) }
16
+ def status=(value); end
17
+
18
+ sig { returns(T.nilable(T::Array[String])) }
19
+ def reasons; end
20
+
21
+ sig { params(value: T.nilable(T::Array[String])).returns(T.nilable(T::Array[String])) }
22
+ def reasons=(value); end
23
+
24
+ sig { returns(T.nilable(String)) }
25
+ def validity_fingerprint; end
26
+
27
+ sig { params(value: T.nilable(String)).returns(T.nilable(String)) }
28
+ def validity_fingerprint=(value); end
29
+
30
+ sig { returns(T.nilable(String)) }
31
+ def input_fingerprint; end
32
+
33
+ sig { params(value: T.nilable(String)).returns(T.nilable(String)) }
34
+ def input_fingerprint=(value); end
35
+
36
+ sig { returns(T.nilable(String)) }
37
+ def baseline_run_id; end
38
+
39
+ sig { params(value: T.nilable(String)).returns(T.nilable(String)) }
40
+ def baseline_run_id=(value); end
41
+
42
+ sig { returns(T::Hash[Symbol, T.untyped]) }
43
+ def to_h; end
44
+
45
+ sig { params(args: T.untyped).returns(String) }
46
+ def to_json(*args); end
47
+ end
48
+ end
@@ -5,7 +5,7 @@
5
5
  # typed: strong
6
6
 
7
7
  module Retab
8
- class WorkflowTestRunWorkflowScope
8
+ class EvalRunTrigger
9
9
  sig { params(json: T.any(String, T::Hash[Symbol, T.untyped])).void }
10
10
  def initialize(json); end
11
11
 
@@ -40,10 +40,10 @@ module Retab
40
40
  def last_run=(value); end
41
41
 
42
42
  sig { returns(T.nilable(String)) }
43
- def current_config_fingerprint; end
43
+ def current_block_execution_fingerprint; end
44
44
 
45
45
  sig { params(value: T.nilable(String)).returns(T.nilable(String)) }
46
- def current_config_fingerprint=(value); end
46
+ def current_block_execution_fingerprint=(value); end
47
47
 
48
48
  sig { returns(String) }
49
49
  def message; end
@@ -94,10 +94,10 @@ module Retab
94
94
  def metrics_validity_fingerprint_version=(value); end
95
95
 
96
96
  sig { returns(String) }
97
- def definition_fingerprint; end
97
+ def block_execution_fingerprint; end
98
98
 
99
99
  sig { params(value: String).returns(String) }
100
- def definition_fingerprint=(value); end
100
+ def block_execution_fingerprint=(value); end
101
101
 
102
102
  sig { returns(String) }
103
103
  def documents_fingerprint; end
@@ -33,10 +33,11 @@ module Retab
33
33
  params(
34
34
  experiment_id: String,
35
35
  workflow_id: T.nilable(String),
36
+ plan_token: T.nilable(String),
36
37
  request_options: T::Hash[Symbol, T.untyped]
37
38
  ).returns(Retab::ExperimentRun)
38
39
  end
39
- def create(experiment_id:, workflow_id:, request_options:); end
40
+ def create(experiment_id:, workflow_id:, plan_token:, request_options:); end
40
41
 
41
42
  sig do
42
43
  params(