bitfab 0.76.1 → 0.76.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/bitfab/client.rb +7 -1
- data/lib/bitfab/experiments.rb +6 -3
- data/lib/bitfab/http_client.rb +4 -1
- data/lib/bitfab/replay.rb +7 -1
- data/lib/bitfab/replay_registry.rb +5 -2
- data/lib/bitfab/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c5942fdb14d1fbffd77e851fdef0ca8352b3815b236ce984733732ddefae688f
|
|
4
|
+
data.tar.gz: b94626cada8efc2b017c471f0b9ebd977c6f42d0c70fe83945547ee0d826f7eb
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 133fb765afdd0a9441e4e894019e96992ec2dff1dbbb84ca6e8b07e7750612f27c6f9dc0851ba0eb31502d989b3d5778e4839cc55d14ab3b11042aa9c70c837d
|
|
7
|
+
data.tar.gz: 3dd647921a6fa3799f4a3574476776e4b7a9743028d1adca7ae55530050950b7267fd5f569679ef67593bb0a6a6732c25adc7ee901a303fd7dfdc3b0f618079f
|
data/lib/bitfab/client.rb
CHANGED
|
@@ -170,6 +170,11 @@ module Bitfab
|
|
|
170
170
|
# experiment group, applied when this replay creates the group or the group
|
|
171
171
|
# has no name yet. A group that already has a name keeps it. Requires
|
|
172
172
|
# experiment_group_id; passing it alone raises ArgumentError
|
|
173
|
+
# @param baseline_experiment_id [String, nil] optional UUID of an earlier
|
|
174
|
+
# experiment this run is compared against. It takes precedence over the
|
|
175
|
+
# first finished experiment in the group and over the original traces.
|
|
176
|
+
# It must belong to the caller's organization or the server rejects the
|
|
177
|
+
# replay
|
|
173
178
|
# @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
|
|
174
179
|
# replay runs against, for benchmarking one method against several corpora
|
|
175
180
|
# in a single run; mutually exclusive with dataset_id. Passing trace_ids
|
|
@@ -234,7 +239,7 @@ module Bitfab
|
|
|
234
239
|
# @return [Hash] with :items, :experiment_id, :experiment_url, and the
|
|
235
240
|
# deprecated aliases :test_run_id and :test_run_url carrying the same values
|
|
236
241
|
def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
|
|
237
|
-
name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
|
|
242
|
+
name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
|
|
238
243
|
adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
|
|
239
244
|
on_experiment_start: nil, experimental_selective_replay: nil)
|
|
240
245
|
Replay.run(
|
|
@@ -261,6 +266,7 @@ module Bitfab
|
|
|
261
266
|
code_change_base:,
|
|
262
267
|
experiment_group_id:,
|
|
263
268
|
experiment_group_name:,
|
|
269
|
+
baseline_experiment_id:,
|
|
264
270
|
dataset_id:,
|
|
265
271
|
dataset_ids:,
|
|
266
272
|
grader_ids:,
|
data/lib/bitfab/experiments.rb
CHANGED
|
@@ -30,6 +30,8 @@ module Bitfab
|
|
|
30
30
|
@http_client.get(query.empty? ? PATH : "#{PATH}?#{query}")
|
|
31
31
|
end
|
|
32
32
|
|
|
33
|
+
# Reads one experiment as a Hash. "baselineExperimentId" and "baselineExperimentName" name the earlier
|
|
34
|
+
# experiment set as this run's baseline, and are nil when none was set.
|
|
33
35
|
def get(id)
|
|
34
36
|
@http_client.get("#{PATH}/#{encode(id)}")["experiment"]
|
|
35
37
|
end
|
|
@@ -47,9 +49,10 @@ module Bitfab
|
|
|
47
49
|
|
|
48
50
|
# Reads one run's written summary. "status" is "pending" while the run has not finished or its verdicts are
|
|
49
51
|
# still arriving; the summary is written once every verdict is in. "unavailable" means the run ended failed or
|
|
50
|
-
# interrupted and no summary will be written. "ready" carries the summary, compared against the
|
|
51
|
-
# run
|
|
52
|
-
#
|
|
52
|
+
# interrupted and no summary will be written. "ready" carries the summary, compared against the experiment set
|
|
53
|
+
# as this run's baseline (baseline kind "chosen_experiment"), else the first finished run in the same
|
|
54
|
+
# experiment group for the same traced function ("group_experiment"), else the run's original traces
|
|
55
|
+
# ("original_traces").
|
|
53
56
|
def get_summary(id)
|
|
54
57
|
@http_client.get("#{PATH}/#{encode(id)}/summary")
|
|
55
58
|
end
|
data/lib/bitfab/http_client.rb
CHANGED
|
@@ -265,6 +265,8 @@ module Bitfab
|
|
|
265
265
|
# @param experiment_group_name [String, nil] optional display name for the
|
|
266
266
|
# experiment group, applied when this replay creates the group or the group
|
|
267
267
|
# has no name yet; only sent together with experiment_group_id
|
|
268
|
+
# @param baseline_experiment_id [String, nil] optional UUID of an earlier
|
|
269
|
+
# experiment this run is compared against, sent as baselineExperimentId
|
|
268
270
|
# @param name [String, nil] optional display name for the resulting
|
|
269
271
|
# experiment
|
|
270
272
|
# @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
|
|
@@ -283,7 +285,7 @@ module Bitfab
|
|
|
283
285
|
# ArgumentError when empty or passed without a trace ID or dataset
|
|
284
286
|
# selection.
|
|
285
287
|
def start_replay(trace_function_key, limit, trace_ids: nil, code_change_type: nil, code_change_description: nil,
|
|
286
|
-
code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
|
|
288
|
+
code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
|
|
287
289
|
grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil)
|
|
288
290
|
payload = {
|
|
289
291
|
"traceFunctionKey" => trace_function_key
|
|
@@ -300,6 +302,7 @@ module Bitfab
|
|
|
300
302
|
payload["codeChangeFiles"] = normalize_code_change_files(code_change_files) unless code_change_files.nil?
|
|
301
303
|
payload["experimentGroupId"] = experiment_group_id unless experiment_group_id.nil?
|
|
302
304
|
payload["experimentGroupName"] = experiment_group_name unless experiment_group_name.nil?
|
|
305
|
+
payload["baselineExperimentId"] = baseline_experiment_id unless baseline_experiment_id.nil?
|
|
303
306
|
payload["includeDbBranchLease"] = true if include_db_branch_lease
|
|
304
307
|
payload["lazyDbBranchLease"] = true if include_db_branch_lease
|
|
305
308
|
unless dataset_ids.nil?
|
data/lib/bitfab/replay.rb
CHANGED
|
@@ -256,6 +256,11 @@ module Bitfab
|
|
|
256
256
|
# experiment group, applied when this replay creates the group or the group
|
|
257
257
|
# has no name yet. A group that already has a name keeps it. Requires
|
|
258
258
|
# experiment_group_id; passing it alone raises ArgumentError
|
|
259
|
+
# @param baseline_experiment_id [String, nil] optional UUID of an earlier
|
|
260
|
+
# experiment this run is compared against. It takes precedence over the
|
|
261
|
+
# first finished experiment in the group and over the original traces.
|
|
262
|
+
# It must belong to the caller's organization or the server rejects the
|
|
263
|
+
# replay
|
|
259
264
|
# @param dataset_id [String, nil] optional UUID of the dataset this replay
|
|
260
265
|
# runs against, stored on the resulting experiment for durable attribution;
|
|
261
266
|
# mutually exclusive with dataset_ids. Alone it replays the dataset's full
|
|
@@ -321,7 +326,7 @@ module Bitfab
|
|
|
321
326
|
# The experiment fields are nil for a dry run, which creates no experiment,
|
|
322
327
|
# and progress events then carry a nil :experiment_id too.
|
|
323
328
|
def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
|
|
324
|
-
concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil,
|
|
329
|
+
concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil,
|
|
325
330
|
dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
|
|
326
331
|
adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
|
|
327
332
|
on_experiment_start: nil, experimental_selective_replay: nil)
|
|
@@ -420,6 +425,7 @@ module Bitfab
|
|
|
420
425
|
code_change_files:,
|
|
421
426
|
experiment_group_id:,
|
|
422
427
|
experiment_group_name:,
|
|
428
|
+
baseline_experiment_id:,
|
|
423
429
|
include_db_branch_lease:,
|
|
424
430
|
dataset_ids: resolved_dataset_ids,
|
|
425
431
|
grader_ids:,
|
|
@@ -19,7 +19,7 @@ module Bitfab
|
|
|
19
19
|
# Project-owned registry of production replay roots.
|
|
20
20
|
class ReplayRegistry
|
|
21
21
|
OPTION_NAMES = %i[
|
|
22
|
-
limit trace_ids name notes metadata concurrency max_concurrency experiment_group_id experiment_group_name dataset_id dataset_ids grader_ids mock
|
|
22
|
+
limit trace_ids name notes metadata concurrency max_concurrency experiment_group_id experiment_group_name baseline_experiment_id dataset_id dataset_ids grader_ids mock
|
|
23
23
|
mock_override adapt_inputs db_branch attempts only_with_assertions category_ids skip_assertion_judging judge_assertions dry_run on_item_finish
|
|
24
24
|
].freeze
|
|
25
25
|
|
|
@@ -206,7 +206,7 @@ module Bitfab
|
|
|
206
206
|
options[:limit] = bound || 10
|
|
207
207
|
end
|
|
208
208
|
|
|
209
|
-
%i[name notes metadata max_concurrency experiment_group_id experiment_group_name dataset_ids grader_ids mock attempts only_with_assertions category_ids skip_assertion_judging dry_run].each do |key|
|
|
209
|
+
%i[name notes metadata max_concurrency experiment_group_id experiment_group_name baseline_experiment_id dataset_ids grader_ids mock attempts only_with_assertions category_ids skip_assertion_judging dry_run].each do |key|
|
|
210
210
|
options[key] = args[key] unless args[key].nil?
|
|
211
211
|
end
|
|
212
212
|
|
|
@@ -375,6 +375,9 @@ module Bitfab
|
|
|
375
375
|
options.on("--experiment-group-name NAME",
|
|
376
376
|
"Name the experiment group when this run creates it or it has no name yet. " \
|
|
377
377
|
"A group that already has a name keeps it. Requires --experiment-group-id.") { |value| args[:experiment_group_name] = value }
|
|
378
|
+
options.on("--baseline-experiment-id UUID",
|
|
379
|
+
"Compare this run against an earlier experiment instead of the group's first finished run " \
|
|
380
|
+
"or the original traces.") { |value| args[:baseline_experiment_id] = value }
|
|
378
381
|
options.on("--dataset-ids IDS", "--dataset-id IDS") { |value| args[:dataset_ids] = comma_separated("--dataset-ids", value) }
|
|
379
382
|
options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
|
|
380
383
|
options.on("--attempts N", Integer) { |value| args[:attempts] = positive_integer("--attempts", value) }
|
data/lib/bitfab/version.rb
CHANGED