bitfab 0.76.1 → 0.76.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f272aee3a4e633dffcc3b05633f62a5e2f51c629fd06cc39e1999eadd870dc5f
4
- data.tar.gz: 96f0ca223998a9a8b26c13fc3cd161e2eee815e322190f19ffeabe6b8f1e930b
3
+ metadata.gz: c5942fdb14d1fbffd77e851fdef0ca8352b3815b236ce984733732ddefae688f
4
+ data.tar.gz: b94626cada8efc2b017c471f0b9ebd977c6f42d0c70fe83945547ee0d826f7eb
5
5
  SHA512:
6
- metadata.gz: cfd82ea9b07e7395db282e94636470b7c530076298cd06bab9e2d6f8510fec42279084ce86c1aa39dd88e0c7abf04d9bf7b5b774659e92a40397f45c451b49fe
7
- data.tar.gz: d550786d6876ec1a4d782f841b915f4a871e0cc517e75ebc92f4f5001f40dd84670375b367b1ad0818f073046b1a1be0c1c1414af203458549002c8973403338
6
+ metadata.gz: 133fb765afdd0a9441e4e894019e96992ec2dff1dbbb84ca6e8b07e7750612f27c6f9dc0851ba0eb31502d989b3d5778e4839cc55d14ab3b11042aa9c70c837d
7
+ data.tar.gz: 3dd647921a6fa3799f4a3574476776e4b7a9743028d1adca7ae55530050950b7267fd5f569679ef67593bb0a6a6732c25adc7ee901a303fd7dfdc3b0f618079f
data/lib/bitfab/client.rb CHANGED
@@ -170,6 +170,11 @@ module Bitfab
170
170
  # experiment group, applied when this replay creates the group or the group
171
171
  # has no name yet. A group that already has a name keeps it. Requires
172
172
  # experiment_group_id; passing it alone raises ArgumentError
173
+ # @param baseline_experiment_id [String, nil] optional UUID of an earlier
174
+ # experiment this run is compared against. It takes precedence over the
175
+ # first finished experiment in the group and over the original traces.
176
+ # It must belong to the caller's organization or the server rejects the
177
+ # replay
173
178
  # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
174
179
  # replay runs against, for benchmarking one method against several corpora
175
180
  # in a single run; mutually exclusive with dataset_id. Passing trace_ids
@@ -234,7 +239,7 @@ module Bitfab
234
239
  # @return [Hash] with :items, :experiment_id, :experiment_url, and the
235
240
  # deprecated aliases :test_run_id and :test_run_url carrying the same values
236
241
  def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
237
- name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
242
+ name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
238
243
  adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
239
244
  on_experiment_start: nil, experimental_selective_replay: nil)
240
245
  Replay.run(
@@ -261,6 +266,7 @@ module Bitfab
261
266
  code_change_base:,
262
267
  experiment_group_id:,
263
268
  experiment_group_name:,
269
+ baseline_experiment_id:,
264
270
  dataset_id:,
265
271
  dataset_ids:,
266
272
  grader_ids:,
@@ -30,6 +30,8 @@ module Bitfab
30
30
  @http_client.get(query.empty? ? PATH : "#{PATH}?#{query}")
31
31
  end
32
32
 
33
+ # Reads one experiment as a Hash. "baselineExperimentId" and "baselineExperimentName" name the earlier
34
+ # experiment set as this run's baseline, and are nil when none was set.
33
35
  def get(id)
34
36
  @http_client.get("#{PATH}/#{encode(id)}")["experiment"]
35
37
  end
@@ -47,9 +49,10 @@ module Bitfab
47
49
 
48
50
  # Reads one run's written summary. "status" is "pending" while the run has not finished or its verdicts are
49
51
  # still arriving; the summary is written once every verdict is in. "unavailable" means the run ended failed or
50
- # interrupted and no summary will be written. "ready" carries the summary, compared against the first finished
51
- # run in the same experiment group for the same traced function, or against the run's original traces when
52
- # there is none.
52
+ # interrupted and no summary will be written. "ready" carries the summary, compared against the experiment set
53
+ # as this run's baseline (baseline kind "chosen_experiment"), else the first finished run in the same
54
+ # experiment group for the same traced function ("group_experiment"), else the run's original traces
55
+ # ("original_traces").
53
56
  def get_summary(id)
54
57
  @http_client.get("#{PATH}/#{encode(id)}/summary")
55
58
  end
@@ -265,6 +265,8 @@ module Bitfab
265
265
  # @param experiment_group_name [String, nil] optional display name for the
266
266
  # experiment group, applied when this replay creates the group or the group
267
267
  # has no name yet; only sent together with experiment_group_id
268
+ # @param baseline_experiment_id [String, nil] optional UUID of an earlier
269
+ # experiment this run is compared against, sent as baselineExperimentId
268
270
  # @param name [String, nil] optional display name for the resulting
269
271
  # experiment
270
272
  # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
@@ -283,7 +285,7 @@ module Bitfab
283
285
  # ArgumentError when empty or passed without a trace ID or dataset
284
286
  # selection.
285
287
  def start_replay(trace_function_key, limit, trace_ids: nil, code_change_type: nil, code_change_description: nil,
286
- code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
288
+ code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
287
289
  grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil)
288
290
  payload = {
289
291
  "traceFunctionKey" => trace_function_key
@@ -300,6 +302,7 @@ module Bitfab
300
302
  payload["codeChangeFiles"] = normalize_code_change_files(code_change_files) unless code_change_files.nil?
301
303
  payload["experimentGroupId"] = experiment_group_id unless experiment_group_id.nil?
302
304
  payload["experimentGroupName"] = experiment_group_name unless experiment_group_name.nil?
305
+ payload["baselineExperimentId"] = baseline_experiment_id unless baseline_experiment_id.nil?
303
306
  payload["includeDbBranchLease"] = true if include_db_branch_lease
304
307
  payload["lazyDbBranchLease"] = true if include_db_branch_lease
305
308
  unless dataset_ids.nil?
data/lib/bitfab/replay.rb CHANGED
@@ -256,6 +256,11 @@ module Bitfab
256
256
  # experiment group, applied when this replay creates the group or the group
257
257
  # has no name yet. A group that already has a name keeps it. Requires
258
258
  # experiment_group_id; passing it alone raises ArgumentError
259
+ # @param baseline_experiment_id [String, nil] optional UUID of an earlier
260
+ # experiment this run is compared against. It takes precedence over the
261
+ # first finished experiment in the group and over the original traces.
262
+ # It must belong to the caller's organization or the server rejects the
263
+ # replay
259
264
  # @param dataset_id [String, nil] optional UUID of the dataset this replay
260
265
  # runs against, stored on the resulting experiment for durable attribution;
261
266
  # mutually exclusive with dataset_ids. Alone it replays the dataset's full
@@ -321,7 +326,7 @@ module Bitfab
321
326
  # The experiment fields are nil for a dry run, which creates no experiment,
322
327
  # and progress events then carry a nil :experiment_id too.
323
328
  def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
324
- concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil,
329
+ concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil,
325
330
  dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
326
331
  adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
327
332
  on_experiment_start: nil, experimental_selective_replay: nil)
@@ -420,6 +425,7 @@ module Bitfab
420
425
  code_change_files:,
421
426
  experiment_group_id:,
422
427
  experiment_group_name:,
428
+ baseline_experiment_id:,
423
429
  include_db_branch_lease:,
424
430
  dataset_ids: resolved_dataset_ids,
425
431
  grader_ids:,
@@ -19,7 +19,7 @@ module Bitfab
19
19
  # Project-owned registry of production replay roots.
20
20
  class ReplayRegistry
21
21
  OPTION_NAMES = %i[
22
- limit trace_ids name notes metadata concurrency max_concurrency experiment_group_id experiment_group_name dataset_id dataset_ids grader_ids mock
22
+ limit trace_ids name notes metadata concurrency max_concurrency experiment_group_id experiment_group_name baseline_experiment_id dataset_id dataset_ids grader_ids mock
23
23
  mock_override adapt_inputs db_branch attempts only_with_assertions category_ids skip_assertion_judging judge_assertions dry_run on_item_finish
24
24
  ].freeze
25
25
 
@@ -206,7 +206,7 @@ module Bitfab
206
206
  options[:limit] = bound || 10
207
207
  end
208
208
 
209
- %i[name notes metadata max_concurrency experiment_group_id experiment_group_name dataset_ids grader_ids mock attempts only_with_assertions category_ids skip_assertion_judging dry_run].each do |key|
209
+ %i[name notes metadata max_concurrency experiment_group_id experiment_group_name baseline_experiment_id dataset_ids grader_ids mock attempts only_with_assertions category_ids skip_assertion_judging dry_run].each do |key|
210
210
  options[key] = args[key] unless args[key].nil?
211
211
  end
212
212
 
@@ -375,6 +375,9 @@ module Bitfab
375
375
  options.on("--experiment-group-name NAME",
376
376
  "Name the experiment group when this run creates it or it has no name yet. " \
377
377
  "A group that already has a name keeps it. Requires --experiment-group-id.") { |value| args[:experiment_group_name] = value }
378
+ options.on("--baseline-experiment-id UUID",
379
+ "Compare this run against an earlier experiment instead of the group's first finished run " \
380
+ "or the original traces.") { |value| args[:baseline_experiment_id] = value }
378
381
  options.on("--dataset-ids IDS", "--dataset-id IDS") { |value| args[:dataset_ids] = comma_separated("--dataset-ids", value) }
379
382
  options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
380
383
  options.on("--attempts N", Integer) { |value| args[:attempts] = positive_integer("--attempts", value) }
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Bitfab
4
- VERSION = "0.76.1"
4
+ VERSION = "0.76.2"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bitfab
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.76.1
4
+ version: 0.76.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Harvest Team