bitfab 0.62.7 → 0.63.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e4a1e837039a5679b239d2c976e5a0ed657b2270c317a9101ded3416acc48fa
-  data.tar.gz: c9168929101f6b1bea5c853a80384a242d64f69dec8c99fb3b4a202ae62bd7b8
+  metadata.gz: 1294a30cd222eeeb8abc13f1f5cf9b14e0b19ebdc1fe9de174aa146587f28edc
+  data.tar.gz: 2636b235d5359ebc72fe0aead7121e5b3ac3363302da51f8b0b5f270713702b5
 SHA512:
-  metadata.gz: 3474b103e94a7cce451ab5955652836bf84c0300c3719ece113a14b0114ee41ff322314da45f4d9015b7040672fa84439ba3bc268712c6fab7f58af3f3cda026
-  data.tar.gz: 442acb8bb6b2a72154c40ae99ec3d34a55d0d212eaba2e27f60bebd000d8453b7dbc0e683c872d9bbf9689e016866c90f3afb2d312ff846020f1d886b5d85565
+  metadata.gz: e4695438f4fed3999fa5f0da499f639d94efaa9fcdf084b522c037a5a5129420e28895c9be10f8059cca89f80a26bbbb95f9e0d23d667cf419103b60ad28d951
+  data.tar.gz: 5ab3ffe5e5bdc8b826d25d2d809c3f5f2acb53910a7294fc0dc6380e2262711e57df10e5e15974c46fa9fc605967326d70e79203ae4ce7c875d105e2ee67e924

data/lib/bitfab/client.rb

@@ -198,7 +198,7 @@ module Bitfab
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
-      name: nil, notes: nil, metadata: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
+      name: nil, notes: nil, metadata: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
       Replay.run(
@@ -215,6 +215,7 @@ module Bitfab
         max_concurrency:,
         attempts:,
         only_with_assertions:,
+        skip_assertion_judging:,
         judge_assertions:,
         dry_run:,
         code_change_description:,

data/lib/bitfab/http_client.rb

@@ -262,7 +262,7 @@ module Bitfab
     #   same org and trace function or the server rejects the replay
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_description: nil,
       code_change_files: nil, experiment_group_id: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
-      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, judge_assertions: false, include_original_metadata: false, experimental_selective_replay: nil)
+      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
       }
@@ -288,7 +288,7 @@ module Bitfab
       payload["graderIds"] = grader_ids unless grader_ids.nil?
       payload["attempts"] = attempts if attempts > 1
       payload["onlyWithAssertions"] = true if only_with_assertions
-      payload["judgeAssertions"] = true if judge_assertions
+      payload["judgeAssertions"] = true unless skip_assertion_judging
       payload["includeOriginalMetadata"] = true if include_original_metadata
       payload["experimentalSelectiveReplay"] = experimental_selective_replay if experimental_selective_replay
       payload["dbBranchSettings"] = db_branch_settings unless db_branch_settings.nil?

data/lib/bitfab/replay.rb

@@ -144,6 +144,8 @@ module Bitfab
     CODE_CHANGE_UNSET = Object.new.freeze
     PERSISTENCE_TIMEOUT_SECONDS = 30.0
     PERSISTENCE_POLL_SECONDS = 0.1
+    JUDGE_ASSERTIONS_DEPRECATION = "judge_assertions is deprecated. Assertion judging is on by default; " \
+      "pass skip_assertion_judging: true (--skip-assertion-judging) to turn it off."
 
     module_function
 
@@ -282,9 +284,10 @@ module Bitfab
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
       concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change_description: CODE_CHANGE_UNSET, code_change_files: CODE_CHANGE_UNSET, experiment_group_id: nil,
-      dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
+      dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
+      skip_assertion_judging = resolve_skip_assertion_judging(skip_assertion_judging, judge_assertions)
       concurrency_options = ReplayConcurrency.resolve(concurrency, attempts, max_concurrency)
       attempts = concurrency_options.attempts
       max_concurrency = concurrency_options.max_concurrency
@@ -387,7 +390,7 @@ module Bitfab
         grader_ids:,
         attempts:,
         only_with_assertions:,
-        judge_assertions:,
+        skip_assertion_judging:,
         include_original_metadata: !adapt_inputs.nil?,
         db_branch_settings: resolved_db_branch_settings,
         **(experimental_selective_replay ? {experimental_selective_replay: SelectiveReplay.wire_options(experimental_selective_replay)} : {})
@@ -566,6 +569,13 @@ module Bitfab
     CC_MAX_FILE_BYTES = 500_000
     CC_MAX_TOTAL_BYTES = 2_000_000
 
+    def resolve_skip_assertion_judging(skip_assertion_judging, judge_assertions)
+      return skip_assertion_judging if judge_assertions.nil?
+
+      Bitfab.warn_once("judge-assertions-deprecated", JUDGE_ASSERTIONS_DEPRECATION)
+      skip_assertion_judging || judge_assertions == false
+    end
+
     def resolve_dataset_ids(dataset_id, dataset_ids)
       if dataset_id && dataset_ids
         raise ArgumentError,

data/lib/bitfab/replay_registry.rb

@@ -21,7 +21,7 @@ module Bitfab
     OPTION_NAMES = %i[
       limit trace_ids name notes metadata concurrency max_concurrency code_change_description
       code_change_files experiment_group_id dataset_id dataset_ids grader_ids mock
-      mock_override adapt_inputs db_branch attempts only_with_assertions judge_assertions dry_run on_item_finish
+      mock_override adapt_inputs db_branch attempts only_with_assertions skip_assertion_judging judge_assertions dry_run on_item_finish
     ].freeze
 
     def initialize
@@ -140,6 +140,7 @@ module Bitfab
     def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
       args = parse(registry, argv.dup)
       return SeedCli.run(registry, argv:, stdout:, stderr:) if args[:seed] || args[:from_trace]
+      stderr.puts "Bitfab: #{Replay::JUDGE_ASSERTIONS_DEPRECATION}" if args[:judge_assertions]
 
       registration = registry.fetch(args.fetch(:pipeline))
       options = registration.options.dup
@@ -188,7 +189,7 @@ module Bitfab
         options[:limit] = bound || 10
       end
 
-      %i[name notes metadata max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions judge_assertions dry_run].each do |key|
+      %i[name notes metadata max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions skip_assertion_judging dry_run].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -325,8 +326,9 @@ module Bitfab
         options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
         options.on("--attempts N", Integer) { |value| args[:attempts] = positive_integer("--attempts", value) }
         options.on("--only-with-assertions") { args[:only_with_assertions] = true }
-        options.on("--judge-assertions",
-          "Judge each replay's approved assertions as it finishes (costs model calls)") { args[:judge_assertions] = true }
+        options.on("--skip-assertion-judging",
+          "Don't judge approved assertions on each replay; judging is on by default and costs model calls") { args[:skip_assertion_judging] = true }
+        options.on("--judge-assertions", "Deprecated: judging is on by default") { args[:judge_assertions] = true }
         options.on("--dry-run") { args[:dry_run] = true }
         options.on("--fail-on-error",
           "Exit 1 after printing the result when any replayed item errored. " \

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.62.7"
+  VERSION = "0.63.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.62.7
+  version: 0.63.0
 platform: ruby
 authors:
 - Harvest Team