bitfab 0.62.7 → 0.63.1

data/lib/bitfab/cloud_replay_cli.rb

@@ -2,13 +2,15 @@
 
 require "json"
 require "open3"
+require "rbconfig"
 require "tempfile"
 
 module Bitfab
   module CloudReplayCli
-    HELP = "Direct GitHub replay: --cloud PIPELINE --trace-ids UUID[,UUID] [--registry PATH] [--max-concurrency 1..32] [--cloud-include FILE] [--cloud-dry-run] [--cloud-detach] [--cloud-request-id UUID] [--cloud-timeout MINUTES] [--cloud-check] [--fail-on-error] [replay options such as --name, --dataset-ids, --attempts, or --resume, passed to the replay on the runner]. Lifecycle: --cloud-status|--cloud-watch|--cloud-cancel|--cloud-cleanup UUID. Setup: --cloud-init [--config FILE] (creates a setup or updates it in place), --cloud-secrets --env-file FILE [NAME ...]. Requires git, gh auth login, Python 3.10+, and bitfab:setup cloud."
+    HINT = "Add --cloud to run the same replay on GitHub Actions; bitfab-replay --cloud --help explains it."
 
     HELPER = File.expand_path("cloudReplay.py", __dir__)
+    REPLAY_EXECUTABLE = File.expand_path("../../exe/bitfab-replay", __dir__)
 
     module_function
 
@@ -17,13 +19,13 @@ module Bitfab
     end
 
     def run(argv:, stdout:, stderr:, helper: HELPER)
-      if argv.include?("--help") || argv.include?("-h")
-        stdout.puts HELP
-        return {}
-      end
+      env = {
+        "BITFAB_REPLAY_COMMAND" => JSON.generate([RbConfig.ruby, REPLAY_EXECUTABLE]),
+        "BITFAB_SDK_LANGUAGE" => "ruby"
+      }
       Tempfile.create("bitfab-cloud-output") do |file|
         result_status = nil
-        Open3.popen3("python3", helper, *argv) do |input, output_stream, error_stream, thread|
+        Open3.popen3(env, "python3", helper, *argv) do |input, output_stream, error_stream, thread|
           input.close
           reader = Thread.new { IO.copy_stream(output_stream, file) }
           error_stream.each_line { |line| stderr.write(line) }
@@ -36,6 +38,10 @@ module Bitfab
           stdout.puts result_output unless result_output.strip.empty?
           raise ArgumentError, "Cloud replay failed. See diagnostics; use the execution UUID to recover a dispatched job."
         end
+        if argv.include?("--help") || argv.include?("-h")
+          stdout.write result_output
+          return {}
+        end
         result = JSON.parse(result_output)
         stdout.puts JSON.pretty_generate(result)
         result

data/lib/bitfab/http_client.rb

@@ -262,7 +262,7 @@ module Bitfab
     #   same org and trace function or the server rejects the replay
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_description: nil,
       code_change_files: nil, experiment_group_id: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
-      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, judge_assertions: false, include_original_metadata: false, experimental_selective_replay: nil)
+      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
       }
@@ -288,7 +288,7 @@ module Bitfab
       payload["graderIds"] = grader_ids unless grader_ids.nil?
       payload["attempts"] = attempts if attempts > 1
       payload["onlyWithAssertions"] = true if only_with_assertions
-      payload["judgeAssertions"] = true if judge_assertions
+      payload["judgeAssertions"] = true unless skip_assertion_judging
       payload["includeOriginalMetadata"] = true if include_original_metadata
       payload["experimentalSelectiveReplay"] = experimental_selective_replay if experimental_selective_replay
       payload["dbBranchSettings"] = db_branch_settings unless db_branch_settings.nil?

data/lib/bitfab/replay.rb

@@ -144,6 +144,8 @@ module Bitfab
     CODE_CHANGE_UNSET = Object.new.freeze
     PERSISTENCE_TIMEOUT_SECONDS = 30.0
     PERSISTENCE_POLL_SECONDS = 0.1
+    JUDGE_ASSERTIONS_DEPRECATION = "judge_assertions is deprecated. Assertion judging is on by default; " \
+      "pass skip_assertion_judging: true (--skip-assertion-judging) to turn it off."
 
     module_function
 
@@ -282,9 +284,10 @@ module Bitfab
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
       concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change_description: CODE_CHANGE_UNSET, code_change_files: CODE_CHANGE_UNSET, experiment_group_id: nil,
-      dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, judge_assertions: false, dry_run: false,
+      dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
+      skip_assertion_judging = resolve_skip_assertion_judging(skip_assertion_judging, judge_assertions)
       concurrency_options = ReplayConcurrency.resolve(concurrency, attempts, max_concurrency)
       attempts = concurrency_options.attempts
       max_concurrency = concurrency_options.max_concurrency
@@ -387,7 +390,7 @@ module Bitfab
         grader_ids:,
         attempts:,
         only_with_assertions:,
-        judge_assertions:,
+        skip_assertion_judging:,
         include_original_metadata: !adapt_inputs.nil?,
         db_branch_settings: resolved_db_branch_settings,
         **(experimental_selective_replay ? {experimental_selective_replay: SelectiveReplay.wire_options(experimental_selective_replay)} : {})
@@ -566,6 +569,13 @@ module Bitfab
     CC_MAX_FILE_BYTES = 500_000
     CC_MAX_TOTAL_BYTES = 2_000_000
 
+    def resolve_skip_assertion_judging(skip_assertion_judging, judge_assertions)
+      return skip_assertion_judging if judge_assertions.nil?
+
+      Bitfab.warn_once("judge-assertions-deprecated", JUDGE_ASSERTIONS_DEPRECATION)
+      skip_assertion_judging || judge_assertions == false
+    end
+
     def resolve_dataset_ids(dataset_id, dataset_ids)
       if dataset_id && dataset_ids
         raise ArgumentError,

data/lib/bitfab/replay_cli.rb

@@ -7,7 +7,7 @@ module Bitfab
   # Installed command that loads a project-owned registry module.
   module ReplayCommand
     USAGE = "Usage: bitfab-replay --registry <path> <pipeline> [replay options]"
-    HELP = "#{USAGE}\nRun bitfab-replay --registry <path> --help to list replay options.\n#{CloudReplayCli::HELP}"
+    HELP = "#{USAGE}\nRun bitfab-replay --registry <path> --help to list replay options.\n#{CloudReplayCli::HINT}"
 
     module_function
 
@@ -26,7 +26,7 @@ module Bitfab
     end
 
     def main(argv: ARGV, stdout: $stdout, stderr: $stderr)
-      if (argv.include?("--help") || argv.include?("-h")) && !argv.include?("--registry")
+      if (argv.include?("--help") || argv.include?("-h")) && !argv.include?("--registry") && !CloudReplayCli.cloud?(argv)
         stdout.puts HELP
         return 0
       end

data/lib/bitfab/replay_registry.rb

@@ -21,7 +21,7 @@ module Bitfab
     OPTION_NAMES = %i[
       limit trace_ids name notes metadata concurrency max_concurrency code_change_description
       code_change_files experiment_group_id dataset_id dataset_ids grader_ids mock
-      mock_override adapt_inputs db_branch attempts only_with_assertions judge_assertions dry_run on_item_finish
+      mock_override adapt_inputs db_branch attempts only_with_assertions skip_assertion_judging judge_assertions dry_run on_item_finish
     ].freeze
 
     def initialize
@@ -140,6 +140,7 @@ module Bitfab
     def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
       args = parse(registry, argv.dup)
       return SeedCli.run(registry, argv:, stdout:, stderr:) if args[:seed] || args[:from_trace]
+      stderr.puts "Bitfab: #{Replay::JUDGE_ASSERTIONS_DEPRECATION}" if args[:judge_assertions]
 
       registration = registry.fetch(args.fetch(:pipeline))
       options = registration.options.dup
@@ -188,7 +189,7 @@ module Bitfab
         options[:limit] = bound || 10
       end
 
-      %i[name notes metadata max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions judge_assertions dry_run].each do |key|
+      %i[name notes metadata max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions skip_assertion_judging dry_run].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -325,8 +326,9 @@ module Bitfab
         options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
         options.on("--attempts N", Integer) { |value| args[:attempts] = positive_integer("--attempts", value) }
         options.on("--only-with-assertions") { args[:only_with_assertions] = true }
-        options.on("--judge-assertions",
-          "Judge each replay's approved assertions as it finishes (costs model calls)") { args[:judge_assertions] = true }
+        options.on("--skip-assertion-judging",
+          "Don't judge approved assertions on each replay; judging is on by default and costs model calls") { args[:skip_assertion_judging] = true }
+        options.on("--judge-assertions", "Deprecated: judging is on by default") { args[:judge_assertions] = true }
         options.on("--dry-run") { args[:dry_run] = true }
         options.on("--fail-on-error",
           "Exit 1 after printing the result when any replayed item errored. " \

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.62.7"
+  VERSION = "0.63.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.62.7
+  version: 0.63.1
 platform: ruby
 authors:
 - Harvest Team