bitfab 0.64.6 → 0.64.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb774e94257b1eb736023bb05b20605e97d86ba24218b55288957919789450df
-  data.tar.gz: 6f63ab2372787edbb1961b964532738fac437ab353c701597187e7e42b696fd8
+  metadata.gz: 6999d067c0ee15b7098bd4a432de082c0993b403178de77fd3a71c00ec6d1558
+  data.tar.gz: 5fe41dfe83313ca0413369680a25aaf09244a25dd40944b7bb0bc035267e7cfd
 SHA512:
-  metadata.gz: 74ec4a06f27ddc8e76832bbf60fa44d60edfbe45d5f8d8b29c82a8892b95d656245327a6c0eb4de1ccadcef0e1b29740cfeb82fae8f0bcc9af2283a0ad40c299
-  data.tar.gz: 832554fe31418799c0face53a8426eac016214babb6faa8c5be197dbfaa9f4276b69e8de90ae0289ccc061348aaee9e9b6b2254737dd785635722c858014b953
+  metadata.gz: 691bf9d5734641198986549b3888d0b488374a21fc58daf80d6afe2e19a803b6f8e76ccc7974cd4abe6969993ff6655a515384f73aeaa4e26a4044af6bc93f7d
+  data.tar.gz: bd9a1aa20a2fcf7937daacb44022e1f6b4211bda69f145e075c5ad16c408417e5571f948016901118be5cb211d094c11d7c349f4dca6ecf82ba7d011a6aeca26

data/lib/bitfab/cloudReplay.py

@@ -33,6 +33,8 @@ API_VERSION = "2026-03-10"
 REPLAY_COMMAND_ENV = "BITFAB_REPLAY_COMMAND"
 SDK_LANGUAGE_ENV = "BITFAB_SDK_LANGUAGE"
 CHECK_COMMAND_ENV = "BITFAB_REPLAY_CHECK"
+EXECUTION_TARGET_ENV = "BITFAB_REPLAY_EXECUTION_TARGET"
+HOSTED_EXECUTION_TARGET = "hosted"
 REQUEST_VERSION = 3
 OLDEST_DISPATCHED_VERSION = 2
 RESULT_LINE = "bitfab-replay-result "
@@ -1776,6 +1778,7 @@ def run_command(directory, args, timeout, experiment, *, summarize=True):
         with subprocess.Popen(
             args,
             cwd=directory,
+            env={**os.environ, EXECUTION_TARGET_ENV: HOSTED_EXECUTION_TARGET},
             stdout=output,
             stderr=subprocess.PIPE,
             stdin=subprocess.DEVNULL,

data/lib/bitfab/http_client.rb

@@ -5,6 +5,7 @@ require "json"
 require "uri"
 
 require_relative "git_state"
+require_relative "replay_invocation"
 require_relative "compress"
 require_relative "connection_pool"
 require_relative "constants"
@@ -262,7 +263,7 @@ module Bitfab
     #   same org and trace function or the server rejects the replay
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_description: nil,
       code_change_files: nil, experiment_group_id: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
-      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false)
+      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
       }
@@ -293,6 +294,7 @@ module Bitfab
       payload["includeOriginalMetadata"] = true if include_original_metadata
       payload["experimentalSelectiveReplay"] = experimental_selective_replay if experimental_selective_replay
       payload["dbBranchSettings"] = db_branch_settings unless db_branch_settings.nil?
+      payload["invocation"] = invocation unless invocation.nil?
       git_state = GitState.resolved
       payload["git"] = git_state if git_state
 

data/lib/bitfab/labels.rb

@@ -14,8 +14,8 @@ module Bitfab
       @http_client = http_client
     end
 
-    def save(label:, annotation:, trace_id: nil, original_trace_id: nil, attempt: nil,
-      confidence: nil, experiment_id: nil, assertion_id: nil, evidence: EVIDENCE_UNSET, test_run_id: nil)
+    def save(label:, annotation:, assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil,
+      confidence: nil, experiment_id: nil, evidence: EVIDENCE_UNSET, test_run_id: nil)
       experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
       update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
       update["label"] = label
@@ -27,6 +27,7 @@ module Bitfab
 
     def save_all(labels, experiment_id: nil, test_run_id: nil)
       experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
+      require_assertion_ids(labels)
       payload = {"labels" => labels}
       unless experiment_id.nil?
         payload["experimentId"] = experiment_id
@@ -35,7 +36,7 @@ module Bitfab
       @http_client.request(LABELS_PATH, payload)["labels"]
     end
 
-    def skip(trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil, assertion_id: nil,
+    def skip(assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil,
       test_run_id: nil, annotation: nil)
       experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
       update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:).merge("skip" => true)
@@ -49,16 +50,15 @@ module Bitfab
       save_all([update.merge("archive" => true)], experiment_id:).first
     end
 
-    def save_human(label:, annotation:, trace_id:, confidence: nil, assertion_id: nil,
-      evidence: EVIDENCE_UNSET)
-      update = {"traceId" => trace_id, "label" => label, "annotation" => annotation}
-      update["assertionId"] = assertion_id unless assertion_id.nil?
+    def save_human(label:, annotation:, trace_id:, assertion_id:, confidence: nil, evidence: EVIDENCE_UNSET)
+      update = {"traceId" => trace_id, "assertionId" => assertion_id, "label" => label, "annotation" => annotation}
       update["confidence"] = confidence unless confidence.nil?
       update["evidence"] = evidence unless evidence.equal?(EVIDENCE_UNSET)
       save_human_all([update]).first
     end
 
     def save_human_all(labels)
+      require_human_assertion_ids(labels)
       @http_client.request(HUMAN_LABELS_PATH, {"labels" => labels})["labels"]
     end
 
@@ -83,6 +83,35 @@ module Bitfab
 
     private
 
+    def require_assertion_ids(labels)
+      unscoped = labels.reject do |entry|
+        entry_value(entry, "archive") == true || !blank?(entry_value(entry, "assertionId"))
+      end
+      return if unscoped.empty?
+
+      targets = unscoped.map { |entry| entry_value(entry, "traceId") || entry_value(entry, "originalTraceId") }
+      raise ArgumentError, "Agent verdicts must name an assertion: #{targets.join(", ")}. " \
+        "Pass the assertion_id of the assertion each verdict scores to labels.save, save_all, and skip. " \
+        "Only archive may omit it, to clear an old whole-trace verdict."
+    end
+
+    def require_human_assertion_ids(labels)
+      unscoped = labels.select { |entry| blank?(entry_value(entry, "assertionId")) }
+      return if unscoped.empty?
+
+      targets = unscoped.map { |entry| entry_value(entry, "traceId") }
+      raise ArgumentError, "Human verdicts must name an assertion: #{targets.join(", ")}. " \
+        "Pass the assertion_id of the assertion each verdict scores to labels.save_human and save_human_all."
+    end
+
+    def entry_value(entry, key)
+      entry.key?(key) ? entry[key] : entry[key.to_sym]
+    end
+
+    def blank?(value)
+      value.nil? || value.to_s.empty?
+    end
+
     def label_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
       if trace_id.nil? == original_trace_id.nil?
         raise ArgumentError, "Pass exactly one of trace_id or original_trace_id. " \

data/lib/bitfab/replay.rb

@@ -8,6 +8,7 @@ require "timeout"
 require_relative "constants"
 require_relative "git_command"
 require_relative "mock_override"
+require_relative "replay_invocation"
 require_relative "serialize"
 require_relative "selective_replay"
 require_relative "traceable"
@@ -142,6 +143,8 @@ module Bitfab
   # Replay historical traces through a traced method and create an experiment.
   module Replay
     CODE_CHANGE_UNSET = Object.new.freeze
+    UNSET_DEFAULT = Object.new.freeze
+    OPTION_DEFAULTS = {mock: "marked"}.freeze
     PERSISTENCE_TIMEOUT_SECONDS = 30.0
     PERSISTENCE_POLL_SECONDS = 0.1
     JUDGE_ASSERTIONS_DEPRECATION = "judge_assertions is deprecated. Assertion judging is on by default; " \
@@ -193,6 +196,13 @@ module Bitfab
       raise error, cause:
     end
 
+    def explicit_replay_options(options)
+      options.reject do |option, value|
+        value.nil? || value == false || value.equal?(ReplayConcurrency::UNSET) ||
+          value.equal?(CODE_CHANGE_UNSET) || value == OPTION_DEFAULTS.fetch(option, UNSET_DEFAULT)
+      end
+    end
+
     # Replay historical traces through a method and create an experiment.
     #
     # Fetches the last N traces for the given trace function key, re-runs each
@@ -289,6 +299,7 @@ module Bitfab
       dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
+      explicit_options = method(:run).parameters.filter_map { |type, option| [option, binding.local_variable_get(option)] if type == :key }
       skip_assertion_judging = resolve_skip_assertion_judging(skip_assertion_judging, judge_assertions)
       concurrency_options = ReplayConcurrency.resolve(concurrency, attempts, max_concurrency)
       attempts = concurrency_options.attempts
@@ -396,6 +407,7 @@ module Bitfab
         include_original_metadata: !adapt_inputs.nil?,
         db_branch_settings: resolved_db_branch_settings,
         dry_run:,
+        invocation: ReplayInvocation.safe_build { ReplayInvocation.build(ReplayInvocation.current_cli_flags || ReplayInvocation.option_flag_entries(explicit_replay_options(explicit_options))) },
         **(experimental_selective_replay ? {experimental_selective_replay: SelectiveReplay.wire_options(experimental_selective_replay)} : {})
       )
       experiment_id = replay_data["experimentId"] || replay_data["testRunId"]

data/lib/bitfab/replay_invocation.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require_relative "version"
+
+module Bitfab
+  module ReplayInvocation
+    EXECUTION_TARGET_ENV = "BITFAB_REPLAY_EXECUTION_TARGET"
+    HOSTED_EXECUTION_TARGET = "hosted"
+    CI_ENV = "CI"
+    CI_TRUE_VALUES = %w[true 1].freeze
+    CI_PRESENCE_ENV_VARS = %w[GITHUB_ACTIONS GITLAB_CI CIRCLECI BUILDKITE JENKINS_URL TF_BUILD].freeze
+    MAX_FLAGS = 100
+    MAX_FLAG_NAME_LENGTH = 100
+    MAX_FLAG_STRING_LENGTH = 1_000
+    MAX_FLAG_LIST_LENGTH = 100
+    SECRET_FLAG_NAME = /key|token|secret|password/i
+    CLI_FLAGS_KEY = :bitfab_replay_cli_flags
+    UNSUPPORTED = Object.new.freeze
+    METADATA_FLAG = "metadata"
+    OPTION_FLAG_NAMES = {
+      max_concurrency: "concurrency",
+      dataset_id: "dataset-ids",
+      dataset_ids: "dataset-ids"
+    }.freeze
+
+    module_function
+
+    def environment
+      return "cloud_replay" if ENV[EXECUTION_TARGET_ENV] == HOSTED_EXECUTION_TARGET
+      return "ci" if CI_TRUE_VALUES.include?(ENV.fetch(CI_ENV, "").downcase)
+      return "ci" if CI_PRESENCE_ENV_VARS.any? { |name| !ENV.fetch(name, "").empty? }
+
+      "local"
+    end
+
+    def flag_scalar(value)
+      case value
+      when String then value[0, MAX_FLAG_STRING_LENGTH]
+      when true, false, nil then value
+      when Integer then value
+      when Float then value.finite? ? value : UNSUPPORTED
+      when Symbol then value.to_s[0, MAX_FLAG_STRING_LENGTH]
+      else UNSUPPORTED
+      end
+    end
+
+    def flag_value(value)
+      if value.is_a?(Array)
+        items = value.first(MAX_FLAG_LIST_LENGTH).map { |item| flag_scalar(item) }
+        return (items.any? { |item| item.equal?(UNSUPPORTED) }) ? true : items
+      end
+      scalar = flag_scalar(value)
+      scalar.equal?(UNSUPPORTED) || scalar
+    end
+
+    def option_flag_name(option)
+      OPTION_FLAG_NAMES.fetch(option.to_sym) { option.to_s.tr("_", "-") }
+    end
+
+    def option_flag_entries(options)
+      mapped, rest = options.to_a.partition { |option, _value| OPTION_FLAG_NAMES.key?(option.to_sym) }
+      (mapped + rest).map { |option, value| [option_flag_name(option), value] }
+    end
+
+    def metadata_keys(value)
+      keys = case value
+      when Hash then value.keys
+      when Array then value.map { |item| item.is_a?(Array) ? item.first : item.to_s.split("=", 2).first }
+      else return true
+      end
+      keys.map(&:to_s).uniq.sort
+    end
+
+    def flags(entries)
+      result = {}
+      entries.each do |name, value|
+        break if result.length >= MAX_FLAGS
+
+        name = name.to_s
+        next if name.empty? || result.key?(name) || name.length > MAX_FLAG_NAME_LENGTH || SECRET_FLAG_NAME.match?(name)
+
+        result[name] = flag_value((name == METADATA_FLAG) ? metadata_keys(value) : value)
+      end
+      result
+    end
+
+    def build(entries = {})
+      {
+        "flags" => flags(entries),
+        "sdk" => {"language" => "ruby", "version" => VERSION},
+        "environment" => environment
+      }
+    end
+
+    def safe_build
+      yield
+    rescue
+      nil
+    end
+
+    def current_cli_flags
+      Thread.current[CLI_FLAGS_KEY]
+    end
+
+    def with_cli_flags(cli_flags)
+      previous = Thread.current[CLI_FLAGS_KEY]
+      Thread.current[CLI_FLAGS_KEY] = cli_flags
+      yield
+    ensure
+      Thread.current[CLI_FLAGS_KEY] = previous
+    end
+  end
+end

data/lib/bitfab/replay_registry.rb

@@ -88,6 +88,7 @@ module Bitfab
   # Argument parsing and output for the SDK-owned replay command.
   module ReplayCli
     MAX_PINNED_TRACE_IDS = 100
+    CLI_INVOCATION_SKIPPED = %i[pipeline seed run from_trace].freeze
 
     class ItemsErroredError < StandardError; end
 
@@ -247,12 +248,14 @@ module Bitfab
         stderr.puts "[replay] Experiment #{experiment[:experiment_id]}: #{experiment[:experiment_url]}"
       end
       result = begin
-        registration.client.replay(
-          registration.receiver,
-          registration.method_name,
-          trace_function_key: registration.trace_function_key,
-          **options
-        )
+        ReplayInvocation.with_cli_flags(cli_invocation_flags(args)) do
+          registration.client.replay(
+            registration.receiver,
+            registration.method_name,
+            trace_function_key: registration.trace_function_key,
+            **options
+          )
+        end
       rescue ReplayError => error
         error.items.each do |item|
           item_error = item[:trace_error] || item[:replay_error] || item[:error]
@@ -273,6 +276,21 @@ module Bitfab
       result
     end
 
+    def cli_invocation_flags(args)
+      args.except(*CLI_INVOCATION_SKIPPED).each_with_object({}) do |(key, value), flags|
+        case key
+        when :param
+          flags["param"] = value.map { |raw| raw.split("=", 2).first }
+        when :max_concurrency
+          flags["concurrency"] = value
+        when :db_branch, :memory_throttle
+          flags["#{"no-" unless value}#{key.to_s.tr("_", "-")}"] = true
+        else
+          flags[key.to_s.tr("_", "-")] = value
+        end
+      end
+    end
+
     def fail_if_items_errored(result, dry_run:)
       items = result[:items].reject { |item| item[:carried_over] }
       errored = items.count { |item| item[:error] }

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.64.6"
+  VERSION = "0.64.8"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.64.6
+  version: 0.64.8
 platform: ruby
 authors:
 - Harvest Team
@@ -184,6 +184,7 @@ files:
 - lib/bitfab/replay_branch.rb
 - lib/bitfab/replay_cli.rb
 - lib/bitfab/replay_concurrency.rb
+- lib/bitfab/replay_invocation.rb
 - lib/bitfab/replay_memory.rb
 - lib/bitfab/replay_processes.rb
 - lib/bitfab/replay_registry.rb