bitfab 0.77.0 → 0.78.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 70548c4852d1ccf420ff549ab12baf2fc60cc05f58550c5553ee67cecefb6d86
4
- data.tar.gz: 8842182c4ae2c227b1532c6f398e0d66fd0d0e8b7464e19653ab8d5986a8f4db
3
+ metadata.gz: 8df415900dea564b2faa8d2ecc01c2538bf44ad2d88d16a669c54439cba933b6
4
+ data.tar.gz: 262a531fc1566e6e276ec4f1b7136f09327fe6a2a97089f1e7f9fb067710937b
5
5
  SHA512:
6
- metadata.gz: 3cab10abd13b21e2d29202714f81be75ec97227928dbca2b378dec538d220f29af0b3c0ae4c18f1ec6cf10cd23723b975c9f82bc5f9492142413747762b5c6c9
7
- data.tar.gz: 52e818a16d56887ea6cee3cbfad1911780924d27c9607f48fc54a470a04748e2990732bd231995f41483be7f480505d0c42bf410679a1556825bc74bdb6848f1
6
+ metadata.gz: 05ead01bb8ece0e612217c20bd59f49233699735896f401db844430f0664c5bff188169185092f34bf7a01e2ba3d717f11738fc5e26066287e698cf87039e409
7
+ data.tar.gz: 7ddc3c3294263875bdf80b9edb7ffc1eeadcda7e699f6b77c51135e606b5efcb950169584effdd303027aa1012de72ecd2699ac380c39ce1ad6784da56c23345
data/lib/bitfab/client.rb CHANGED
@@ -198,9 +198,15 @@ module Bitfab
198
198
  # directly to this experiment, graded as the union with the dataset's
199
199
  # runnable graders at completion; each must be an active/live grader in the
200
200
  # same org and trace function or the server rejects the replay
201
+ # @param only_drafts [Boolean] replay only the selected traces that
202
+ # carry at least one draft assertion: not confirmed, not
203
+ # rejected, and not archived. Narrows whatever limit, trace_ids or
204
+ # dataset_ids selected, the way only_with_assertions does. With
205
+ # dataset_ids the experiment is recorded as a subset of the dataset. A
206
+ # selection that narrows to nothing replays nothing.
201
207
  # @param category_ids [Array<String>, nil] optional category UUIDs. With
202
208
  # trace_ids, dataset_id or dataset_ids, only the selected traces carrying
203
- # an approved assertion in one of these categories replay. It changes
209
+ # a confirmed assertion in one of these categories replay. It changes
204
210
  # which traces replay, not which assertions are judged. Raises
205
211
  # ArgumentError when empty or passed without a trace ID or dataset
206
212
  # selection.
@@ -250,7 +256,7 @@ module Bitfab
250
256
  # @return [Hash] with :items, :experiment_id, :experiment_url, and the
251
257
  # deprecated aliases :test_run_id and :test_run_url carrying the same values
252
258
  def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
253
- name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, skip_experiment_group: false, agent_session_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
259
+ name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, skip_experiment_group: false, agent_session_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, only_drafts: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
254
260
  adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
255
261
  on_experiment_start: nil, experimental_selective_replay: nil)
256
262
  Replay.run(
@@ -267,6 +273,7 @@ module Bitfab
267
273
  max_concurrency:,
268
274
  attempts:,
269
275
  only_with_assertions:,
276
+ only_drafts:,
270
277
  category_ids:,
271
278
  skip_assertion_judging:,
272
279
  judge_assertions:,
@@ -329,15 +329,21 @@ module Bitfab
329
329
  # directly to this experiment, graded as the union with the dataset's
330
330
  # runnable graders at completion; each must be an active/live grader in the
331
331
  # same org and trace function or the server rejects the replay
332
+ # @param only_drafts [Boolean] replay only the selected traces that
333
+ # carry at least one draft assertion: not confirmed, not
334
+ # rejected, and not archived. Narrows whatever limit, trace_ids or
335
+ # dataset_ids selected, the way only_with_assertions does. With
336
+ # dataset_ids the experiment is recorded as a subset of the dataset. A
337
+ # selection that narrows to nothing replays nothing.
332
338
  # @param category_ids [Array<String>, nil] optional category UUIDs. With
333
339
  # trace_ids, dataset_id or dataset_ids, only the selected traces carrying
334
- # an approved assertion in one of these categories replay. It changes
340
+ # a confirmed assertion in one of these categories replay. It changes
335
341
  # which traces replay, not which assertions are judged. Raises
336
342
  # ArgumentError when empty or passed without a trace ID or dataset
337
343
  # selection.
338
344
  def start_replay(trace_function_key, limit, trace_ids: nil, code_change_type: nil, code_change_description: nil,
339
345
  code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, require_experiment_group: true, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
340
- grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil, agent_session_id: nil)
346
+ grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, only_drafts: false, category_ids: nil, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil, agent_session_id: nil)
341
347
  payload = {"traceFunctionKey" => trace_function_key}
342
348
  payload["requireExperimentGroup"] = true if require_experiment_group
343
349
  # limit is only meaningful without trace_ids (an explicit ID list
@@ -367,6 +373,7 @@ module Bitfab
367
373
  payload["graderIds"] = grader_ids unless grader_ids.nil?
368
374
  payload["attempts"] = attempts if attempts > 1
369
375
  payload["onlyWithAssertions"] = true if only_with_assertions
376
+ payload["onlyDrafts"] = true if only_drafts
370
377
  payload["categoryIds"] = category_ids unless category_ids.nil?
371
378
  payload["judgeAssertions"] = true unless skip_assertion_judging
372
379
  payload["dryRun"] = true if dry_run
data/lib/bitfab/replay.rb CHANGED
@@ -294,9 +294,15 @@ module Bitfab
294
294
  # directly to this experiment, graded as the union with the dataset's
295
295
  # runnable graders at completion; each must be an active/live grader in the
296
296
  # same org and trace function or the server rejects the replay
297
+ # @param only_drafts [Boolean] replay only the selected traces that
298
+ # carry at least one draft assertion: not confirmed, not
299
+ # rejected, and not archived. Narrows whatever limit, trace_ids or
300
+ # dataset_ids selected, the way only_with_assertions does. With
301
+ # dataset_ids the experiment is recorded as a subset of the dataset. A
302
+ # selection that narrows to nothing replays nothing.
297
303
  # @param category_ids [Array<String>, nil] optional category UUIDs. With
298
304
  # trace_ids, dataset_id or dataset_ids, only the selected traces carrying
299
- # an approved assertion in one of these categories replay. It changes
305
+ # a confirmed assertion in one of these categories replay. It changes
300
306
  # which traces replay, not which assertions are judged. Raises
301
307
  # ArgumentError when empty or passed without a trace ID or dataset
302
308
  # selection.
@@ -346,7 +352,7 @@ module Bitfab
346
352
  # and progress events then carry a nil :experiment_id too.
347
353
  def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
348
354
  concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, skip_experiment_group: false, agent_session_id: nil,
349
- dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
355
+ dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, only_drafts: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
350
356
  adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
351
357
  on_experiment_start: nil, experimental_selective_replay: nil)
352
358
  validate_code_change(code_change, code_change_description, code_change_files, code_change_base)
@@ -458,6 +464,7 @@ module Bitfab
458
464
  grader_ids:,
459
465
  attempts:,
460
466
  only_with_assertions:,
467
+ only_drafts:,
461
468
  category_ids:,
462
469
  skip_assertion_judging:,
463
470
  include_original_metadata: !adapt_inputs.nil?,
@@ -20,7 +20,7 @@ module Bitfab
20
20
  class ReplayRegistry
21
21
  OPTION_NAMES = %i[
22
22
  limit trace_ids name notes metadata concurrency max_concurrency experiment_group_id experiment_group_name baseline_experiment_id skip_experiment_group agent_session_id dataset_id dataset_ids grader_ids mock
23
- mock_override adapt_inputs db_branch attempts only_with_assertions category_ids skip_assertion_judging judge_assertions dry_run on_item_finish
23
+ mock_override adapt_inputs db_branch attempts only_with_assertions only_drafts category_ids skip_assertion_judging judge_assertions dry_run on_item_finish
24
24
  ].freeze
25
25
 
26
26
  def initialize
@@ -99,43 +99,65 @@ module Bitfab
99
99
 
100
100
  module_function
101
101
 
102
- # get_assertions returns every state so a reviewer can see drafts, but only an
103
- # approved assertion is checked on a replay, so narrowing on anything else
102
+ # get_assertions returns every status so a reviewer can see drafts, but only a
103
+ # confirmed assertion is checked on a replay, so narrowing on anything else
104
104
  # would pick traces the run cannot be judged against. The server applies the
105
105
  # same rule when it revalidates the selection.
106
- def approved_assertion?(assertions, category_ids = nil)
106
+ def confirmed_assertion?(assertions, category_ids = nil)
107
107
  assertions.any? do |assertion|
108
- assertion["approvalState"] == "approved" &&
108
+ assertion["status"] == "confirmed" &&
109
109
  (category_ids.nil? || category_ids.include?(assertion["category_id"]))
110
110
  end
111
111
  end
112
112
 
113
- def bound_trace_ids(client, trace_ids, limit, only_with_assertions, category_ids = nil)
113
+ def draft_assertion?(assertions)
114
+ assertions.any? { |assertion| assertion["status"] == "draft" }
115
+ end
116
+
117
+ def narrows_to_confirmed_assertions?(narrowing)
118
+ narrowing[:only_with_assertions] || !narrowing[:category_ids].nil?
119
+ end
120
+
121
+ def keeps_trace?(assertions, narrowing)
122
+ if narrows_to_confirmed_assertions?(narrowing) && !confirmed_assertion?(assertions, narrowing[:category_ids])
123
+ return false
124
+ end
125
+
126
+ !narrowing[:only_drafts] || draft_assertion?(assertions)
127
+ end
128
+
129
+ def no_traces_matched_message(narrowing)
130
+ if narrowing[:only_drafts]
131
+ return "No traces with a draft assertion matched this replay selection."
132
+ end
133
+ if narrowing[:category_ids]
134
+ return "No traces with a confirmed assertion in the given categories matched this replay selection."
135
+ end
136
+
137
+ "No traces with confirmed assertions matched this replay selection."
138
+ end
139
+
140
+ def bound_trace_ids(client, trace_ids, limit, narrowing = {})
114
141
  unless limit.is_a?(Integer) && limit.positive?
115
142
  raise ArgumentError, "Replay limit must be a positive integer."
116
143
  end
117
- narrows = only_with_assertions || !category_ids.nil?
144
+ narrows = narrows_to_confirmed_assertions?(narrowing) || narrowing[:only_drafts]
118
145
  return trace_ids.first(limit) if !narrows || limit >= trace_ids.length
119
146
 
120
147
  selected = []
121
148
  trace_ids.each do |trace_id|
122
149
  result = client.traces.get_assertions(trace_id)
123
- if result.fetch("inheritedFrom").nil? && approved_assertion?(result.fetch("assertions"), category_ids)
150
+ if result.fetch("inheritedFrom").nil? && keeps_trace?(result.fetch("assertions"), narrowing)
124
151
  selected << trace_id
125
152
  break if selected.length == limit
126
153
  end
127
154
  end
128
- if selected.empty?
129
- if category_ids
130
- raise ArgumentError,
131
- "No traces with an approved assertion in the given categories matched this replay selection."
132
- end
133
- raise ArgumentError, "No traces with approved assertions matched this replay selection."
134
- end
155
+ raise ArgumentError, no_traces_matched_message(narrowing) if selected.empty?
156
+
135
157
  selected
136
158
  end
137
159
 
138
- def pinned_dataset_members(client, dataset_ids, limit, only_with_assertions = false, category_ids = nil)
160
+ def pinned_dataset_members(client, dataset_ids, limit, narrowing = {})
139
161
  members = dataset_ids.flat_map { |dataset_id|
140
162
  client.datasets.list_traces(dataset_id).fetch("traceIds")
141
163
  }.uniq.sort
@@ -148,7 +170,7 @@ module Bitfab
148
170
  "every selected dataset in full."
149
171
  end
150
172
 
151
- bound_trace_ids(client, members, limit, only_with_assertions, category_ids)
173
+ bound_trace_ids(client, members, limit, narrowing)
152
174
  end
153
175
 
154
176
  def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
@@ -187,8 +209,12 @@ module Bitfab
187
209
  options.delete(:trace_ids)
188
210
  end
189
211
 
190
- only_with_assertions = args[:only_with_assertions] || options[:only_with_assertions]
191
212
  category_ids = args[:category_ids] || options[:category_ids]
213
+ narrowing = {
214
+ only_with_assertions: args[:only_with_assertions] || options[:only_with_assertions],
215
+ only_drafts: args[:only_drafts] || options[:only_drafts],
216
+ category_ids:
217
+ }
192
218
  bound = args[:limit] || options[:limit]
193
219
  options.delete(:limit)
194
220
  bounded_dataset_ids = args[:dataset_ids] || options[:dataset_ids]
@@ -200,20 +226,18 @@ module Bitfab
200
226
 
201
227
  if options[:trace_ids]
202
228
  if bound
203
- options[:trace_ids] = bound_trace_ids(registration.client, options[:trace_ids], bound, only_with_assertions,
204
- category_ids)
229
+ options[:trace_ids] = bound_trace_ids(registration.client, options[:trace_ids], bound, narrowing)
205
230
  end
206
231
  elsif bounded_dataset_ids
207
232
  if bound
208
- pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound, only_with_assertions,
209
- category_ids)
233
+ pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound, narrowing)
210
234
  options[:trace_ids] = pinned if pinned
211
235
  end
212
236
  else
213
237
  options[:limit] = bound || 10
214
238
  end
215
239
 
216
- %i[name notes metadata max_concurrency experiment_group_id experiment_group_name baseline_experiment_id skip_experiment_group agent_session_id dataset_ids grader_ids mock attempts only_with_assertions category_ids skip_assertion_judging dry_run].each do |key|
240
+ %i[name notes metadata max_concurrency experiment_group_id experiment_group_name baseline_experiment_id skip_experiment_group agent_session_id dataset_ids grader_ids mock attempts only_with_assertions only_drafts category_ids skip_assertion_judging dry_run].each do |key|
217
241
  options[key] = args[key] unless args[key].nil?
218
242
  end
219
243
 
@@ -335,7 +359,11 @@ module Bitfab
335
359
  "How many traces to replay. It bounds a trace ID or dataset selection instead of replacing it") do |value|
336
360
  args[:limit] = positive_integer("--limit", value)
337
361
  end
338
- options.on("--trace-ids IDS") { |value| args[:trace_ids] = comma_separated("--trace-ids", value) }
362
+ options.on("--trace-ids IDS",
363
+ "Traces to replay. With --dataset-ids it replays only these members of the datasets, " \
364
+ "recorded on the experiment as a subset of them") do |value|
365
+ args[:trace_ids] = comma_separated("--trace-ids", value)
366
+ end
339
367
  options.on("--seed PATH", "--cases PATH") { |value| args[:seed] = value }
340
368
  options.on("--run") { args[:run] = true }
341
369
  options.on("--from-trace IDS") { |value| args[:from_trace] = comma_separated("--from-trace", value) }
@@ -397,13 +425,16 @@ module Bitfab
397
425
  options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
398
426
  options.on("--attempts N", Integer) { |value| args[:attempts] = positive_integer("--attempts", value) }
399
427
  options.on("--only-with-assertions") { args[:only_with_assertions] = true }
428
+ options.on("--only-drafts",
429
+ "Replay only traces carrying a draft assertion (not yet confirmed or rejected); " \
430
+ "with --dataset-ids the run is recorded as a subset of the dataset") { args[:only_drafts] = true }
400
431
  options.on("--category-ids id1,id2",
401
- "With --trace-ids or --dataset-ids: replay only the traces carrying an approved assertion in one of " \
432
+ "With --trace-ids or --dataset-ids: replay only the traces carrying a confirmed assertion in one of " \
402
433
  "these categories; which assertions are judged stays the same") do |value|
403
434
  args[:category_ids] = comma_separated("--category-ids", value)
404
435
  end
405
436
  options.on("--skip-assertion-judging",
406
- "Don't judge approved assertions on each replay; judging is on by default and costs model calls") { args[:skip_assertion_judging] = true }
437
+ "Don't judge confirmed assertions on each replay; judging is on by default and costs model calls") { args[:skip_assertion_judging] = true }
407
438
  options.on("--judge-assertions", "Deprecated: judging is on by default") { args[:judge_assertions] = true }
408
439
  options.on("--dry-run") { args[:dry_run] = true }
409
440
  options.on("--fail-on-error",
@@ -418,10 +449,6 @@ module Bitfab
418
449
  options.on("--param NAME=VALUE") { |value| (args[:param] ||= []) << value }
419
450
  end
420
451
  parser.parse!(argv)
421
- if args[:trace_ids] && args[:dataset_ids]
422
- raise OptionParser::InvalidArgument,
423
- "--trace-ids and --dataset-ids select different replay sources and cannot be used together"
424
- end
425
452
  pipeline = argv.shift
426
453
  known = pipeline && (registry.nil? || registry.names.include?(pipeline))
427
454
  raise OptionParser::MissingArgument, parser.banner unless known
@@ -470,6 +497,15 @@ module Bitfab
470
497
  end
471
498
 
472
499
  def describe_selection(options)
500
+ selection = describe_selected_traces(options)
501
+ options[:only_drafts] ? "#{selection} with a draft assertion" : selection
502
+ end
503
+
504
+ def describe_selected_traces(options)
505
+ if options[:trace_ids] && options[:dataset_ids]
506
+ datasets = (options[:dataset_ids].length == 1) ? "the dataset" : "#{options[:dataset_ids].length} datasets"
507
+ return "#{options[:trace_ids].length} traces of #{datasets}"
508
+ end
473
509
  return "#{options[:trace_ids].length} traces" if options[:trace_ids]
474
510
  return "#{options[:limit]} traces" if options[:limit]
475
511
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Bitfab
4
- VERSION = "0.77.0"
4
+ VERSION = "0.78.0"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bitfab
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.77.0
4
+ version: 0.78.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Harvest Team