bitfab 0.77.0 → 0.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/bitfab/client.rb +9 -2
- data/lib/bitfab/http_client.rb +9 -2
- data/lib/bitfab/replay.rb +9 -2
- data/lib/bitfab/replay_registry.rb +66 -30
- data/lib/bitfab/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8df415900dea564b2faa8d2ecc01c2538bf44ad2d88d16a669c54439cba933b6
|
|
4
|
+
data.tar.gz: 262a531fc1566e6e276ec4f1b7136f09327fe6a2a97089f1e7f9fb067710937b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 05ead01bb8ece0e612217c20bd59f49233699735896f401db844430f0664c5bff188169185092f34bf7a01e2ba3d717f11738fc5e26066287e698cf87039e409
|
|
7
|
+
data.tar.gz: 7ddc3c3294263875bdf80b9edb7ffc1eeadcda7e699f6b77c51135e606b5efcb950169584effdd303027aa1012de72ecd2699ac380c39ce1ad6784da56c23345
|
data/lib/bitfab/client.rb
CHANGED
|
@@ -198,9 +198,15 @@ module Bitfab
|
|
|
198
198
|
# directly to this experiment, graded as the union with the dataset's
|
|
199
199
|
# runnable graders at completion; each must be an active/live grader in the
|
|
200
200
|
# same org and trace function or the server rejects the replay
|
|
201
|
+
# @param only_drafts [Boolean] replay only the selected traces that
|
|
202
|
+
# carry at least one draft assertion: not confirmed, not
|
|
203
|
+
# rejected, and not archived. Narrows whatever limit, trace_ids or
|
|
204
|
+
# dataset_ids selected, the way only_with_assertions does. With
|
|
205
|
+
# dataset_ids the experiment is recorded as a subset of the dataset. A
|
|
206
|
+
# selection that narrows to nothing replays nothing.
|
|
201
207
|
# @param category_ids [Array<String>, nil] optional category UUIDs. With
|
|
202
208
|
# trace_ids, dataset_id or dataset_ids, only the selected traces carrying
|
|
203
|
-
#
|
|
209
|
+
# a confirmed assertion in one of these categories replay. It changes
|
|
204
210
|
# which traces replay, not which assertions are judged. Raises
|
|
205
211
|
# ArgumentError when empty or passed without a trace ID or dataset
|
|
206
212
|
# selection.
|
|
@@ -250,7 +256,7 @@ module Bitfab
|
|
|
250
256
|
# @return [Hash] with :items, :experiment_id, :experiment_url, and the
|
|
251
257
|
# deprecated aliases :test_run_id and :test_run_url carrying the same values
|
|
252
258
|
def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
|
|
253
|
-
name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, skip_experiment_group: false, agent_session_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
|
|
259
|
+
name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, skip_experiment_group: false, agent_session_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, only_drafts: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
|
|
254
260
|
adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
|
|
255
261
|
on_experiment_start: nil, experimental_selective_replay: nil)
|
|
256
262
|
Replay.run(
|
|
@@ -267,6 +273,7 @@ module Bitfab
|
|
|
267
273
|
max_concurrency:,
|
|
268
274
|
attempts:,
|
|
269
275
|
only_with_assertions:,
|
|
276
|
+
only_drafts:,
|
|
270
277
|
category_ids:,
|
|
271
278
|
skip_assertion_judging:,
|
|
272
279
|
judge_assertions:,
|
data/lib/bitfab/http_client.rb
CHANGED
|
@@ -329,15 +329,21 @@ module Bitfab
|
|
|
329
329
|
# directly to this experiment, graded as the union with the dataset's
|
|
330
330
|
# runnable graders at completion; each must be an active/live grader in the
|
|
331
331
|
# same org and trace function or the server rejects the replay
|
|
332
|
+
# @param only_drafts [Boolean] replay only the selected traces that
|
|
333
|
+
# carry at least one draft assertion: not confirmed, not
|
|
334
|
+
# rejected, and not archived. Narrows whatever limit, trace_ids or
|
|
335
|
+
# dataset_ids selected, the way only_with_assertions does. With
|
|
336
|
+
# dataset_ids the experiment is recorded as a subset of the dataset. A
|
|
337
|
+
# selection that narrows to nothing replays nothing.
|
|
332
338
|
# @param category_ids [Array<String>, nil] optional category UUIDs. With
|
|
333
339
|
# trace_ids, dataset_id or dataset_ids, only the selected traces carrying
|
|
334
|
-
#
|
|
340
|
+
# a confirmed assertion in one of these categories replay. It changes
|
|
335
341
|
# which traces replay, not which assertions are judged. Raises
|
|
336
342
|
# ArgumentError when empty or passed without a trace ID or dataset
|
|
337
343
|
# selection.
|
|
338
344
|
def start_replay(trace_function_key, limit, trace_ids: nil, code_change_type: nil, code_change_description: nil,
|
|
339
345
|
code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, require_experiment_group: true, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
|
|
340
|
-
grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil, agent_session_id: nil)
|
|
346
|
+
grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, only_drafts: false, category_ids: nil, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil, agent_session_id: nil)
|
|
341
347
|
payload = {"traceFunctionKey" => trace_function_key}
|
|
342
348
|
payload["requireExperimentGroup"] = true if require_experiment_group
|
|
343
349
|
# limit is only meaningful without trace_ids (an explicit ID list
|
|
@@ -367,6 +373,7 @@ module Bitfab
|
|
|
367
373
|
payload["graderIds"] = grader_ids unless grader_ids.nil?
|
|
368
374
|
payload["attempts"] = attempts if attempts > 1
|
|
369
375
|
payload["onlyWithAssertions"] = true if only_with_assertions
|
|
376
|
+
payload["onlyDrafts"] = true if only_drafts
|
|
370
377
|
payload["categoryIds"] = category_ids unless category_ids.nil?
|
|
371
378
|
payload["judgeAssertions"] = true unless skip_assertion_judging
|
|
372
379
|
payload["dryRun"] = true if dry_run
|
data/lib/bitfab/replay.rb
CHANGED
|
@@ -294,9 +294,15 @@ module Bitfab
|
|
|
294
294
|
# directly to this experiment, graded as the union with the dataset's
|
|
295
295
|
# runnable graders at completion; each must be an active/live grader in the
|
|
296
296
|
# same org and trace function or the server rejects the replay
|
|
297
|
+
# @param only_drafts [Boolean] replay only the selected traces that
|
|
298
|
+
# carry at least one draft assertion: not confirmed, not
|
|
299
|
+
# rejected, and not archived. Narrows whatever limit, trace_ids or
|
|
300
|
+
# dataset_ids selected, the way only_with_assertions does. With
|
|
301
|
+
# dataset_ids the experiment is recorded as a subset of the dataset. A
|
|
302
|
+
# selection that narrows to nothing replays nothing.
|
|
297
303
|
# @param category_ids [Array<String>, nil] optional category UUIDs. With
|
|
298
304
|
# trace_ids, dataset_id or dataset_ids, only the selected traces carrying
|
|
299
|
-
#
|
|
305
|
+
# a confirmed assertion in one of these categories replay. It changes
|
|
300
306
|
# which traces replay, not which assertions are judged. Raises
|
|
301
307
|
# ArgumentError when empty or passed without a trace ID or dataset
|
|
302
308
|
# selection.
|
|
@@ -346,7 +352,7 @@ module Bitfab
|
|
|
346
352
|
# and progress events then carry a nil :experiment_id too.
|
|
347
353
|
def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
|
|
348
354
|
concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, skip_experiment_group: false, agent_session_id: nil,
|
|
349
|
-
dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
|
|
355
|
+
dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, only_drafts: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
|
|
350
356
|
adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
|
|
351
357
|
on_experiment_start: nil, experimental_selective_replay: nil)
|
|
352
358
|
validate_code_change(code_change, code_change_description, code_change_files, code_change_base)
|
|
@@ -458,6 +464,7 @@ module Bitfab
|
|
|
458
464
|
grader_ids:,
|
|
459
465
|
attempts:,
|
|
460
466
|
only_with_assertions:,
|
|
467
|
+
only_drafts:,
|
|
461
468
|
category_ids:,
|
|
462
469
|
skip_assertion_judging:,
|
|
463
470
|
include_original_metadata: !adapt_inputs.nil?,
|
|
@@ -20,7 +20,7 @@ module Bitfab
|
|
|
20
20
|
class ReplayRegistry
|
|
21
21
|
OPTION_NAMES = %i[
|
|
22
22
|
limit trace_ids name notes metadata concurrency max_concurrency experiment_group_id experiment_group_name baseline_experiment_id skip_experiment_group agent_session_id dataset_id dataset_ids grader_ids mock
|
|
23
|
-
mock_override adapt_inputs db_branch attempts only_with_assertions category_ids skip_assertion_judging judge_assertions dry_run on_item_finish
|
|
23
|
+
mock_override adapt_inputs db_branch attempts only_with_assertions only_drafts category_ids skip_assertion_judging judge_assertions dry_run on_item_finish
|
|
24
24
|
].freeze
|
|
25
25
|
|
|
26
26
|
def initialize
|
|
@@ -99,43 +99,65 @@ module Bitfab
|
|
|
99
99
|
|
|
100
100
|
module_function
|
|
101
101
|
|
|
102
|
-
# get_assertions returns every
|
|
103
|
-
#
|
|
102
|
+
# get_assertions returns every status so a reviewer can see drafts, but only a
|
|
103
|
+
# confirmed assertion is checked on a replay, so narrowing on anything else
|
|
104
104
|
# would pick traces the run cannot be judged against. The server applies the
|
|
105
105
|
# same rule when it revalidates the selection.
|
|
106
|
-
def
|
|
106
|
+
def confirmed_assertion?(assertions, category_ids = nil)
|
|
107
107
|
assertions.any? do |assertion|
|
|
108
|
-
assertion["
|
|
108
|
+
assertion["status"] == "confirmed" &&
|
|
109
109
|
(category_ids.nil? || category_ids.include?(assertion["category_id"]))
|
|
110
110
|
end
|
|
111
111
|
end
|
|
112
112
|
|
|
113
|
-
def
|
|
113
|
+
def draft_assertion?(assertions)
|
|
114
|
+
assertions.any? { |assertion| assertion["status"] == "draft" }
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def narrows_to_confirmed_assertions?(narrowing)
|
|
118
|
+
narrowing[:only_with_assertions] || !narrowing[:category_ids].nil?
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def keeps_trace?(assertions, narrowing)
|
|
122
|
+
if narrows_to_confirmed_assertions?(narrowing) && !confirmed_assertion?(assertions, narrowing[:category_ids])
|
|
123
|
+
return false
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
!narrowing[:only_drafts] || draft_assertion?(assertions)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def no_traces_matched_message(narrowing)
|
|
130
|
+
if narrowing[:only_drafts]
|
|
131
|
+
return "No traces with a draft assertion matched this replay selection."
|
|
132
|
+
end
|
|
133
|
+
if narrowing[:category_ids]
|
|
134
|
+
return "No traces with a confirmed assertion in the given categories matched this replay selection."
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
"No traces with confirmed assertions matched this replay selection."
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def bound_trace_ids(client, trace_ids, limit, narrowing = {})
|
|
114
141
|
unless limit.is_a?(Integer) && limit.positive?
|
|
115
142
|
raise ArgumentError, "Replay limit must be a positive integer."
|
|
116
143
|
end
|
|
117
|
-
narrows =
|
|
144
|
+
narrows = narrows_to_confirmed_assertions?(narrowing) || narrowing[:only_drafts]
|
|
118
145
|
return trace_ids.first(limit) if !narrows || limit >= trace_ids.length
|
|
119
146
|
|
|
120
147
|
selected = []
|
|
121
148
|
trace_ids.each do |trace_id|
|
|
122
149
|
result = client.traces.get_assertions(trace_id)
|
|
123
|
-
if result.fetch("inheritedFrom").nil? &&
|
|
150
|
+
if result.fetch("inheritedFrom").nil? && keeps_trace?(result.fetch("assertions"), narrowing)
|
|
124
151
|
selected << trace_id
|
|
125
152
|
break if selected.length == limit
|
|
126
153
|
end
|
|
127
154
|
end
|
|
128
|
-
if selected.empty?
|
|
129
|
-
|
|
130
|
-
raise ArgumentError,
|
|
131
|
-
"No traces with an approved assertion in the given categories matched this replay selection."
|
|
132
|
-
end
|
|
133
|
-
raise ArgumentError, "No traces with approved assertions matched this replay selection."
|
|
134
|
-
end
|
|
155
|
+
raise ArgumentError, no_traces_matched_message(narrowing) if selected.empty?
|
|
156
|
+
|
|
135
157
|
selected
|
|
136
158
|
end
|
|
137
159
|
|
|
138
|
-
def pinned_dataset_members(client, dataset_ids, limit,
|
|
160
|
+
def pinned_dataset_members(client, dataset_ids, limit, narrowing = {})
|
|
139
161
|
members = dataset_ids.flat_map { |dataset_id|
|
|
140
162
|
client.datasets.list_traces(dataset_id).fetch("traceIds")
|
|
141
163
|
}.uniq.sort
|
|
@@ -148,7 +170,7 @@ module Bitfab
|
|
|
148
170
|
"every selected dataset in full."
|
|
149
171
|
end
|
|
150
172
|
|
|
151
|
-
bound_trace_ids(client, members, limit,
|
|
173
|
+
bound_trace_ids(client, members, limit, narrowing)
|
|
152
174
|
end
|
|
153
175
|
|
|
154
176
|
def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
|
|
@@ -187,8 +209,12 @@ module Bitfab
|
|
|
187
209
|
options.delete(:trace_ids)
|
|
188
210
|
end
|
|
189
211
|
|
|
190
|
-
only_with_assertions = args[:only_with_assertions] || options[:only_with_assertions]
|
|
191
212
|
category_ids = args[:category_ids] || options[:category_ids]
|
|
213
|
+
narrowing = {
|
|
214
|
+
only_with_assertions: args[:only_with_assertions] || options[:only_with_assertions],
|
|
215
|
+
only_drafts: args[:only_drafts] || options[:only_drafts],
|
|
216
|
+
category_ids:
|
|
217
|
+
}
|
|
192
218
|
bound = args[:limit] || options[:limit]
|
|
193
219
|
options.delete(:limit)
|
|
194
220
|
bounded_dataset_ids = args[:dataset_ids] || options[:dataset_ids]
|
|
@@ -200,20 +226,18 @@ module Bitfab
|
|
|
200
226
|
|
|
201
227
|
if options[:trace_ids]
|
|
202
228
|
if bound
|
|
203
|
-
options[:trace_ids] = bound_trace_ids(registration.client, options[:trace_ids], bound,
|
|
204
|
-
category_ids)
|
|
229
|
+
options[:trace_ids] = bound_trace_ids(registration.client, options[:trace_ids], bound, narrowing)
|
|
205
230
|
end
|
|
206
231
|
elsif bounded_dataset_ids
|
|
207
232
|
if bound
|
|
208
|
-
pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound,
|
|
209
|
-
category_ids)
|
|
233
|
+
pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound, narrowing)
|
|
210
234
|
options[:trace_ids] = pinned if pinned
|
|
211
235
|
end
|
|
212
236
|
else
|
|
213
237
|
options[:limit] = bound || 10
|
|
214
238
|
end
|
|
215
239
|
|
|
216
|
-
%i[name notes metadata max_concurrency experiment_group_id experiment_group_name baseline_experiment_id skip_experiment_group agent_session_id dataset_ids grader_ids mock attempts only_with_assertions category_ids skip_assertion_judging dry_run].each do |key|
|
|
240
|
+
%i[name notes metadata max_concurrency experiment_group_id experiment_group_name baseline_experiment_id skip_experiment_group agent_session_id dataset_ids grader_ids mock attempts only_with_assertions only_drafts category_ids skip_assertion_judging dry_run].each do |key|
|
|
217
241
|
options[key] = args[key] unless args[key].nil?
|
|
218
242
|
end
|
|
219
243
|
|
|
@@ -335,7 +359,11 @@ module Bitfab
|
|
|
335
359
|
"How many traces to replay. It bounds a trace ID or dataset selection instead of replacing it") do |value|
|
|
336
360
|
args[:limit] = positive_integer("--limit", value)
|
|
337
361
|
end
|
|
338
|
-
options.on("--trace-ids IDS"
|
|
362
|
+
options.on("--trace-ids IDS",
|
|
363
|
+
"Traces to replay. With --dataset-ids it replays only these members of the datasets, " \
|
|
364
|
+
"recorded on the experiment as a subset of them") do |value|
|
|
365
|
+
args[:trace_ids] = comma_separated("--trace-ids", value)
|
|
366
|
+
end
|
|
339
367
|
options.on("--seed PATH", "--cases PATH") { |value| args[:seed] = value }
|
|
340
368
|
options.on("--run") { args[:run] = true }
|
|
341
369
|
options.on("--from-trace IDS") { |value| args[:from_trace] = comma_separated("--from-trace", value) }
|
|
@@ -397,13 +425,16 @@ module Bitfab
|
|
|
397
425
|
options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
|
|
398
426
|
options.on("--attempts N", Integer) { |value| args[:attempts] = positive_integer("--attempts", value) }
|
|
399
427
|
options.on("--only-with-assertions") { args[:only_with_assertions] = true }
|
|
428
|
+
options.on("--only-drafts",
|
|
429
|
+
"Replay only traces carrying a draft assertion (not yet confirmed or rejected); " \
|
|
430
|
+
"with --dataset-ids the run is recorded as a subset of the dataset") { args[:only_drafts] = true }
|
|
400
431
|
options.on("--category-ids id1,id2",
|
|
401
|
-
"With --trace-ids or --dataset-ids: replay only the traces carrying
|
|
432
|
+
"With --trace-ids or --dataset-ids: replay only the traces carrying a confirmed assertion in one of " \
|
|
402
433
|
"these categories; which assertions are judged stays the same") do |value|
|
|
403
434
|
args[:category_ids] = comma_separated("--category-ids", value)
|
|
404
435
|
end
|
|
405
436
|
options.on("--skip-assertion-judging",
|
|
406
|
-
"Don't judge
|
|
437
|
+
"Don't judge confirmed assertions on each replay; judging is on by default and costs model calls") { args[:skip_assertion_judging] = true }
|
|
407
438
|
options.on("--judge-assertions", "Deprecated: judging is on by default") { args[:judge_assertions] = true }
|
|
408
439
|
options.on("--dry-run") { args[:dry_run] = true }
|
|
409
440
|
options.on("--fail-on-error",
|
|
@@ -418,10 +449,6 @@ module Bitfab
|
|
|
418
449
|
options.on("--param NAME=VALUE") { |value| (args[:param] ||= []) << value }
|
|
419
450
|
end
|
|
420
451
|
parser.parse!(argv)
|
|
421
|
-
if args[:trace_ids] && args[:dataset_ids]
|
|
422
|
-
raise OptionParser::InvalidArgument,
|
|
423
|
-
"--trace-ids and --dataset-ids select different replay sources and cannot be used together"
|
|
424
|
-
end
|
|
425
452
|
pipeline = argv.shift
|
|
426
453
|
known = pipeline && (registry.nil? || registry.names.include?(pipeline))
|
|
427
454
|
raise OptionParser::MissingArgument, parser.banner unless known
|
|
@@ -470,6 +497,15 @@ module Bitfab
|
|
|
470
497
|
end
|
|
471
498
|
|
|
472
499
|
def describe_selection(options)
|
|
500
|
+
selection = describe_selected_traces(options)
|
|
501
|
+
options[:only_drafts] ? "#{selection} with a draft assertion" : selection
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
def describe_selected_traces(options)
|
|
505
|
+
if options[:trace_ids] && options[:dataset_ids]
|
|
506
|
+
datasets = (options[:dataset_ids].length == 1) ? "the dataset" : "#{options[:dataset_ids].length} datasets"
|
|
507
|
+
return "#{options[:trace_ids].length} traces of #{datasets}"
|
|
508
|
+
end
|
|
473
509
|
return "#{options[:trace_ids].length} traces" if options[:trace_ids]
|
|
474
510
|
return "#{options[:limit]} traces" if options[:limit]
|
|
475
511
|
|
data/lib/bitfab/version.rb
CHANGED