bitfab 0.62.6 → 0.63.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b300b5a65bacd9fd60231bcab3ba96751c111ae7e5f12075a6149c163927cf65
-  data.tar.gz: b3b61e0146d500bebc4719ca485e42e175bd0e8f78053d3fa451bb256d494407
+  metadata.gz: 1294a30cd222eeeb8abc13f1f5cf9b14e0b19ebdc1fe9de174aa146587f28edc
+  data.tar.gz: 2636b235d5359ebc72fe0aead7121e5b3ac3363302da51f8b0b5f270713702b5
 SHA512:
-  metadata.gz: f16224c7b475fdfea0e847780b18d7e503b100250ec66523382a3f1bfa45e8b60281d52042de6dd0ac5f5b90bfea4a5585481fdaaa94500d019ce346ec81cb1d
-  data.tar.gz: 836b9a8bdcfd24612213eefef798713e758b337b283ee1ff89ee62b1d17e60f12e8ba883b3532cd2a47430ee3968637e96375f9912f3c35d7df8290f9a7f460d
+  metadata.gz: e4695438f4fed3999fa5f0da499f639d94efaa9fcdf084b522c037a5a5129420e28895c9be10f8059cca89f80a26bbbb95f9e0d23d667cf419103b60ad28d951
+  data.tar.gz: 5ab3ffe5e5bdc8b826d25d2d809c3f5f2acb53910a7294fc0dc6380e2262711e57df10e5e15974c46fa9fc605967326d70e79203ae4ce7c875d105e2ee67e924

data/lib/bitfab/client.rb

@@ -198,7 +198,7 @@ module Bitfab
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
-      name: nil, notes: nil, metadata: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
+      name: nil, notes: nil, metadata: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
       Replay.run(
@@ -215,6 +215,7 @@ module Bitfab
         max_concurrency:,
         attempts:,
         only_with_assertions:,
+        skip_assertion_judging:,
         judge_assertions:,
         dry_run:,
         code_change_description:,

data/lib/bitfab/http_client.rb

@@ -262,7 +262,7 @@ module Bitfab
     #   same org and trace function or the server rejects the replay
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_description: nil,
       code_change_files: nil, experiment_group_id: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
-      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, judge_assertions: false, include_original_metadata: false, experimental_selective_replay: nil)
+      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
       }
@@ -288,7 +288,7 @@ module Bitfab
       payload["graderIds"] = grader_ids unless grader_ids.nil?
       payload["attempts"] = attempts if attempts > 1
       payload["onlyWithAssertions"] = true if only_with_assertions
-      payload["judgeAssertions"] = true if judge_assertions
+      payload["judgeAssertions"] = true unless skip_assertion_judging
       payload["includeOriginalMetadata"] = true if include_original_metadata
       payload["experimentalSelectiveReplay"] = experimental_selective_replay if experimental_selective_replay
       payload["dbBranchSettings"] = db_branch_settings unless db_branch_settings.nil?

data/lib/bitfab/replay.rb

@@ -144,6 +144,8 @@ module Bitfab
     CODE_CHANGE_UNSET = Object.new.freeze
     PERSISTENCE_TIMEOUT_SECONDS = 30.0
     PERSISTENCE_POLL_SECONDS = 0.1
+    JUDGE_ASSERTIONS_DEPRECATION = "judge_assertions is deprecated. Assertion judging is on by default; " \
+      "pass skip_assertion_judging: true (--skip-assertion-judging) to turn it off."
 
     module_function
 
@@ -282,9 +284,10 @@ module Bitfab
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
       concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change_description: CODE_CHANGE_UNSET, code_change_files: CODE_CHANGE_UNSET, experiment_group_id: nil,
-      dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
+      dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
+      skip_assertion_judging = resolve_skip_assertion_judging(skip_assertion_judging, judge_assertions)
       concurrency_options = ReplayConcurrency.resolve(concurrency, attempts, max_concurrency)
       attempts = concurrency_options.attempts
       max_concurrency = concurrency_options.max_concurrency
@@ -387,7 +390,7 @@ module Bitfab
         grader_ids:,
         attempts:,
         only_with_assertions:,
-        judge_assertions:,
+        skip_assertion_judging:,
         include_original_metadata: !adapt_inputs.nil?,
         db_branch_settings: resolved_db_branch_settings,
         **(experimental_selective_replay ? {experimental_selective_replay: SelectiveReplay.wire_options(experimental_selective_replay)} : {})
@@ -562,11 +565,17 @@ module Bitfab
     # via BITFAB_CODE_CHANGE_PATH. Returns [description, files] or nil.
     # Best-effort: any error (no env var, missing file, bad JSON) yields nil so
     # the replay proceeds with no code change rather than raising.
-    TRUNK_CANDIDATES = ["origin/HEAD", "origin/main", "origin/master", "main", "master"].freeze
     CC_MAX_FILES = 60
     CC_MAX_FILE_BYTES = 500_000
     CC_MAX_TOTAL_BYTES = 2_000_000
 
+    def resolve_skip_assertion_judging(skip_assertion_judging, judge_assertions)
+      return skip_assertion_judging if judge_assertions.nil?
+
+      Bitfab.warn_once("judge-assertions-deprecated", JUDGE_ASSERTIONS_DEPRECATION)
+      skip_assertion_judging || judge_assertions == false
+    end
+
     def resolve_dataset_ids(dataset_id, dataset_ids)
       if dataset_id && dataset_ids
         raise ArgumentError,
@@ -618,7 +627,8 @@ module Bitfab
     # Auto-capture the code change to attach when the caller passed none. Lives
     # in replay() (the one call guaranteed to run on every replay) so it works no
     # matter how the replay was launched. Precedence: a BITFAB_CODE_CHANGE_PATH
-    # override, else the git diff vs trunk. Best-effort: any failure yields nil.
+    # override, else the git diff of the working tree vs HEAD (or vs the
+    # merge-base with BITFAB_CODE_CHANGE_BASE). Best-effort: any failure yields nil.
     # An explicit code_change_files always wins over this.
     def resolve_auto_code_change(label = nil)
       return nil if ENV["BITFAB_DISABLE_CODE_CHANGE_CAPTURE"]
@@ -657,23 +667,18 @@ module Bitfab
       !git(root, ["rev-parse", "--verify", "#{ref}^{object}"]).nil?
     end
 
-    # Returns [base, from_trunk]; from_trunk is false on the HEAD fallback.
+    # Returns [base, against]. HEAD by default, so only uncommitted edits show;
+    # BITFAB_CODE_CHANGE_BASE moves the base to the merge-base with that ref.
     def cc_resolve_base(root)
       forced = ENV["BITFAB_CODE_CHANGE_BASE"]
       if forced && cc_ref_exists?(root, forced)
         mb = git(root, ["merge-base", "HEAD", forced])
-        return [mb.strip, true] if mb && !mb.strip.empty?
+        return [mb.strip, "vs #{forced}"] if mb && !mb.strip.empty?
 
         rp = git(root, ["rev-parse", "--verify", forced])
-        return (rp && !rp.strip.empty?) ? [rp.strip, true] : nil
-      end
-      TRUNK_CANDIDATES.each do |candidate|
-        next unless cc_ref_exists?(root, candidate)
-
-        mb = git(root, ["merge-base", "HEAD", candidate])
-        return [mb.strip, true] if mb && !mb.strip.empty?
+        return (rp && !rp.strip.empty?) ? [rp.strip, "vs #{forced}"] : nil
       end
-      cc_ref_exists?(root, "HEAD") ? ["HEAD", false] : nil
+      cc_ref_exists?(root, "HEAD") ? ["HEAD", "uncommitted (vs HEAD)"] : nil
     end
 
     def cc_blob_bytes(root, base, path)
@@ -697,7 +702,7 @@ module Bitfab
       resolved = cc_resolve_base(root)
       return nil if resolved.nil?
 
-      base, from_trunk = resolved
+      base, against = resolved
 
       tracked = git(root, ["diff", "--name-status", "--find-renames", "-z", base, "--", ":!.bitfab"]) || ""
       untracked = git(root, ["ls-files", "--others", "--exclude-standard", "-z", "--", ":!.bitfab"]) || ""
@@ -737,7 +742,6 @@ module Bitfab
       subject = subject ? subject.strip : ""
       head = ((label && !label.strip.empty?) ? label.strip : nil) || (subject.empty? ? nil : subject) || "Working-tree change"
       word = (files.length == 1) ? "file" : "files"
-      against = from_trunk ? "vs trunk" : "uncommitted (vs HEAD)"
       ["#{head} (#{files.length} #{word} changed #{against})", files]
     rescue
       nil

data/lib/bitfab/replay_registry.rb

@@ -21,7 +21,7 @@ module Bitfab
     OPTION_NAMES = %i[
       limit trace_ids name notes metadata concurrency max_concurrency code_change_description
       code_change_files experiment_group_id dataset_id dataset_ids grader_ids mock
-      mock_override adapt_inputs db_branch attempts only_with_assertions judge_assertions dry_run on_item_finish
+      mock_override adapt_inputs db_branch attempts only_with_assertions skip_assertion_judging judge_assertions dry_run on_item_finish
     ].freeze
 
     def initialize
@@ -140,6 +140,7 @@ module Bitfab
     def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
       args = parse(registry, argv.dup)
       return SeedCli.run(registry, argv:, stdout:, stderr:) if args[:seed] || args[:from_trace]
+      stderr.puts "Bitfab: #{Replay::JUDGE_ASSERTIONS_DEPRECATION}" if args[:judge_assertions]
 
       registration = registry.fetch(args.fetch(:pipeline))
       options = registration.options.dup
@@ -188,7 +189,7 @@ module Bitfab
         options[:limit] = bound || 10
       end
 
-      %i[name notes metadata max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions judge_assertions dry_run].each do |key|
+      %i[name notes metadata max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions skip_assertion_judging dry_run].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -325,8 +326,9 @@ module Bitfab
         options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
         options.on("--attempts N", Integer) { |value| args[:attempts] = positive_integer("--attempts", value) }
         options.on("--only-with-assertions") { args[:only_with_assertions] = true }
-        options.on("--judge-assertions",
-          "Judge each replay's approved assertions as it finishes (costs model calls)") { args[:judge_assertions] = true }
+        options.on("--skip-assertion-judging",
+          "Don't judge approved assertions on each replay; judging is on by default and costs model calls") { args[:skip_assertion_judging] = true }
+        options.on("--judge-assertions", "Deprecated: judging is on by default") { args[:judge_assertions] = true }
         options.on("--dry-run") { args[:dry_run] = true }
         options.on("--fail-on-error",
           "Exit 1 after printing the result when any replayed item errored. " \

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.62.6"
+  VERSION = "0.63.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.62.6
+  version: 0.63.0
 platform: ruby
 authors:
 - Harvest Team