bitfab 0.70.0 → 0.71.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1db22eed10f5c7fd7e24777507be907e01a1a65f7df1a10ff388bc347e17b5f7
-  data.tar.gz: e30cd6c045153777120348af94e6884ee77fef98cb433e14c470886b6085842a
+  metadata.gz: 6a9158d8ba5528a0445a8a84471b6556f1c068b9e34d1b11d9a7759e2a339dae
+  data.tar.gz: 1c00b3b0e7de1d47c34598752047772fc604f045715dd14b28a3d9c191986e21
 SHA512:
-  metadata.gz: 838efe0f9650542d604b48c330aa1dbd033f96091dd7a24ad050d1b8929854cee0af10342d2a3882aaef97c33ab4fb492b1e8a3f1237e74bfdc0ac0a51546064
-  data.tar.gz: 6244270e97642c69cad5fc9c08bb26fa7492149d2d98f04ec6432c4c9f314ba577649f5af2d13913ba39704f0fca46c0d03d435845b647532f9068d97da323aa
+  metadata.gz: ef4585c3380e0ae2370fbb9f9e7a736bad89e7c04c7d2526e1f758a6583fcf82ae6ba84dd0720e99b7525dac11fa0331aa58e13606648f9ef288b1fae2b129de
+  data.tar.gz: ab972e7e48cb60881d62fe597513ae46f6e1649d7ee950d81fe79b7d0d5f523e43761f5381eca2222ec54fc75f940961d0e6d2b9e451fd39a293c9791c9e749a

data/lib/bitfab/baml.rb

@@ -111,6 +111,15 @@ module Bitfab
       nil
     end
 
+    def reported_cost(call)
+      body = call&.http_response&.body&.json
+      usage = body["usage"] if body.is_a?(Hash)
+      cost = usage["cost"] if usage.is_a?(Hash)
+      (cost.is_a?(Numeric) && cost.finite?) ? cost : nil
+    rescue
+      nil
+    end
+
     def enrich_span(collector)
       log = collector.last
       calls = log&.calls || []
@@ -131,6 +140,8 @@ module Bitfab
         model: body["model"] || call&.http_request&.url&.match(%r{/models/([^/:]+)})&.[](1),
         inputTokens: usage&.input_tokens || call&.usage&.input_tokens,
         outputTokens: usage&.output_tokens || call&.usage&.output_tokens,
+        cachedInputTokens: usage&.cached_input_tokens || call&.usage&.cached_input_tokens,
+        costUsd: reported_cost(call),
         durationMs: log&.timing&.duration_ms
       }.compact
       Bitfab.current_span.add_context(metadata) unless metadata.empty?

data/lib/bitfab/client.rb

@@ -147,16 +147,22 @@ module Bitfab
     #   diff against the branch this one was cut from, such as the branch below
     #   it in a stack, else the default branch; "default-branch" the diff
     #   against where HEAD split from the default branch (origin/HEAD, else
-    #   origin/main, origin/master, main, master); "skip" no code change, such
-    #   as for a baseline run; "manual" the files in code_change_files. The
-    #   three git choices include uncommitted edits and untracked files, and
-    #   warn and attach nothing when they find no changed files.
+    #   origin/main, origin/master, main, master); "ref" the diff against where
+    #   HEAD split from the branch or commit in code_change_base; "skip" no code
+    #   change, such as for a baseline run; "manual" the files in
+    #   code_change_files. The four git choices include uncommitted edits and
+    #   untracked files, and warn and attach nothing when they find no changed
+    #   files.
     # @param code_change_description [String, nil] description of the code
     #   change, stored on the experiment. Defaults to one generated from the
     #   captured files. Not accepted with "skip".
     # @param code_change_files [Array<Hash>, nil] the edited files when
     #   code_change is "manual", and only then, each as { path:, before:, after: }
     #   (use "" for new/deleted files).
+    # @param code_change_base [String, nil] the branch or commit to diff
+    #   against when code_change is "ref", and only then, such as "dev",
+    #   "release/2.3", or a commit SHA. A branch that exists only on the remote
+    #   is read as origin/<name>. A name git cannot resolve raises ArgumentError.
     # @param experiment_group_id [String, nil] optional UUID grouping multiple
     #   replay runs into a single experiment batch. The run joins that group,
     #   and an id no group has yet creates it
@@ -222,7 +228,7 @@ module Bitfab
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
-      name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
+      name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
       Replay.run(
@@ -245,6 +251,7 @@ module Bitfab
         code_change:,
         code_change_description:,
         code_change_files:,
+        code_change_base:,
         experiment_group_id:,
         experiment_group_name:,
         dataset_id:,

data/lib/bitfab/cloudReplay.py

@@ -149,7 +149,10 @@ RENAMED_FLAGS = {
 SEED_FLAGS = ("--seed", "--cases", "--from-trace", "--run")
 PATH_FLAGS = ("--registry", "--params", "--code-change-file")
 CODE_CHANGE_FLAG = "--code-change"
-BRANCH_CODE_CHANGES = ("last-branch", "default-branch")
+CODE_CHANGE_BASE_FLAG = "--code-change-base"
+BRANCH_CODE_CHANGES = ("last-branch", "default-branch", "ref")
+GIT_CODE_CHANGES = ("working-tree", *BRANCH_CODE_CHANGES)
+DESCRIBED_REF = re.compile(r"[^\x00-\x1f\x7f]{1,200}")
 CHECKOUT_DEPTH = 2
 CODE_CHANGE_PATH_ENV = "BITFAB_CODE_CHANGE_PATH"
 CODE_CHANGE_MAX_FILES = 60
@@ -220,7 +223,9 @@ Options added by --cloud:
                          checkout of the repository, with GitHub access that can
                          read it and run Actions (a job token needs contents: read
                          and actions: write, passed as GH_TOKEN), and a workflow on
-                         BRANCH that declares a request input.
+                         BRANCH that declares a request input. With no snapshot
+                         there is nothing to diff against, so it takes
+                         --code-change skip or manual, as does a dispatched run.
 
 A workflow can also dispatch bitfab-replay.yml itself on any branch other than a
 bitfab-replay/ snapshot branch, passing the replay as JSON in the request input:
@@ -1029,8 +1034,30 @@ def default_branch_point(root):
     return {"sha": sha, "ref": default} if sha else None
 
 
+def chosen_base_point(root, chosen):
+    if not chosen or chosen.startswith("-"):
+        return None
+    for ref in (chosen, f"origin/{chosen}"):
+        if quiet_git(root, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}"):
+            sha = quiet_git(root, "merge-base", "HEAD", ref)
+            return {"sha": sha, "ref": chosen} if sha else None
+    raise ValueError(
+        f"{CODE_CHANGE_BASE_FLAG} {chosen} names no branch or commit in this repository; pass a branch name or a commit SHA, and fetch it first if it only exists on the remote"
+    )
+
+
+def refuse_git_code_change(replay, where):
+    kind = flag_value(replay, CODE_CHANGE_FLAG)
+    if kind in GIT_CODE_CHANGES:
+        raise ValueError(
+            f"{where} as it is on GitHub, with no snapshot of a checkout, so --code-change {kind} has nothing to compare against there; pass --code-change skip, or --code-change manual with --code-change-file, or start it with bitfab-replay --cloud and no --cloud-ref, which resolves {kind} from your checkout before upload"
+        )
+
+
 def code_change_base(root, replay):
     kind = flag_value(replay, CODE_CHANGE_FLAG)
+    if kind == "ref":
+        return chosen_base_point(root, flag_value(replay, CODE_CHANGE_BASE_FLAG))
     if kind == "last-branch":
         return branch_point(root)
     if kind == "default-branch":
@@ -1839,6 +1866,7 @@ def submit_as_is(root, repo, workflow, parsed, path):
     branch = parsed["cloud"]["--cloud-ref"]
     request, _ = replay_request(root, parsed)
     check_arguments(parsed["replay"])
+    refuse_git_code_change(parsed["replay"], f"--cloud-ref replays {branch}")
     sha = branch_head(repo, branch)
     if "--cloud-preview" in parsed["switches"]:
         return {
@@ -2089,6 +2117,7 @@ def dispatched_request():
         raise ValueError("The request id does not match the execution_id input")
     args = request.get("args")
     validate_arguments(args)
+    refuse_git_code_change(args, f"A dispatched replay runs {branch}")
     if request.get("check") is True and "--dry-run" not in args:
         args = [*args, "--dry-run"]
     normalized = {
@@ -2227,7 +2256,7 @@ def branch_code_change(root, base, commit, request):
         isinstance(sha, str)
         and SHA.fullmatch(sha)
         and isinstance(ref, str)
-        and BRANCH_NAME.fullmatch(ref)
+        and DESCRIBED_REF.fullmatch(ref)
         and (
             sha == base
             or quiet_git(root, "rev-parse", "--verify", "-q", f"{commit}^2") == sha

data/lib/bitfab/replay.rb

@@ -143,11 +143,11 @@ module Bitfab
   # Replay historical traces through a traced method and create an experiment.
   module Replay
     # The values Client#replay accepts for +code_change:+.
-    CODE_CHANGE_TYPES = %w[working-tree last-branch default-branch skip manual].freeze
+    CODE_CHANGE_TYPES = %w[working-tree last-branch default-branch ref skip manual].freeze
     # The +code_change:+ choices, as named in the errors that refuse a replay.
     CODE_CHANGE_CHOICES = '"working-tree" (uncommitted changes vs HEAD), "last-branch" (the diff against the ' \
-      'branch this one was cut from), "default-branch" (the diff against the default branch), "skip" (no code ' \
-      'change), or "manual" (pass the files yourself)'
+      'branch this one was cut from), "default-branch" (the diff against the default branch), "ref" (the diff ' \
+      'against a branch or commit you name), "skip" (no code change), or "manual" (pass the files yourself)'
     UNSET_DEFAULT = Object.new.freeze
     OPTION_DEFAULTS = {mock: "marked"}.freeze
     PERSISTENCE_TIMEOUT_SECONDS = 30.0
@@ -237,16 +237,22 @@ module Bitfab
     #   diff against the branch this one was cut from, such as the branch below
     #   it in a stack, else the default branch; "default-branch" the diff
     #   against where HEAD split from the default branch (origin/HEAD, else
-    #   origin/main, origin/master, main, master); "skip" no code change, such
-    #   as for a baseline run; "manual" the files in code_change_files. The
-    #   three git choices include uncommitted edits and untracked files, and
-    #   warn and attach nothing when they find no changed files.
+    #   origin/main, origin/master, main, master); "ref" the diff against where
+    #   HEAD split from the branch or commit in code_change_base; "skip" no code
+    #   change, such as for a baseline run; "manual" the files in
+    #   code_change_files. The four git choices include uncommitted edits and
+    #   untracked files, and warn and attach nothing when they find no changed
+    #   files.
     # @param code_change_description [String, nil] description of the code
     #   change, stored on the experiment. Defaults to one generated from the
     #   captured files. Not accepted with "skip".
     # @param code_change_files [Array<Hash>, nil] the edited files when
     #   code_change is "manual", and only then, each as { path:, before:, after: }
     #   (empty string for new/deleted files).
+    # @param code_change_base [String, nil] the branch or commit to diff
+    #   against when code_change is "ref", and only then, such as "dev",
+    #   "release/2.3", or a commit SHA. A branch that exists only on the remote
+    #   is read as origin/<name>. A name git cannot resolve raises ArgumentError.
     # @param experiment_group_id [String, nil] optional UUID grouping multiple
     #   replay runs into a single experiment batch. The run joins that group,
     #   and an id no group has yet creates it
@@ -313,11 +319,11 @@ module Bitfab
     #   The experiment fields are nil for a dry run, which creates no experiment,
     #   and progress events then carry a nil :experiment_id too.
     def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
-      concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change: nil, code_change_description: nil, code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil,
+      concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil,
       dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
-      validate_code_change(code_change, code_change_description, code_change_files)
+      validate_code_change(code_change, code_change_description, code_change_files, code_change_base)
       explicit_options = method(:run).parameters.filter_map { |type, option| [option, binding.local_variable_get(option)] if type == :key }
       skip_assertion_judging = resolve_skip_assertion_judging(skip_assertion_judging, judge_assertions)
       concurrency_options = ReplayConcurrency.resolve(concurrency, attempts, max_concurrency)
@@ -396,7 +402,7 @@ module Bitfab
 
       include_db_branch_lease = db_branch_enabled?(db_branch) && !dry_run
 
-      code_change_description, code_change_files = resolve_code_change(code_change, code_change_description, code_change_files, name)
+      code_change_description, code_change_files = resolve_code_change(code_change, code_change_description, code_change_files, name, code_change_base)
 
       resolved_db_branch_settings = db_branch_settings(db_branch)
       replay_data = http_client.start_replay(
@@ -655,7 +661,7 @@ module Bitfab
       settings.empty? ? nil : settings
     end
 
-    def validate_code_change(code_change, description, files)
+    def validate_code_change(code_change, description, files, base = nil)
       if code_change.nil?
         raise ArgumentError, "replay needs a code_change naming the code this run tests: #{CODE_CHANGE_CHOICES}."
       end
@@ -669,17 +675,24 @@ module Bitfab
       if code_change != "manual" && !files.nil?
         raise ArgumentError, "code_change_files only applies with code_change \"manual\" (got \"#{code_change}\")."
       end
+      if code_change == "ref" && !(base.is_a?(String) && !base.strip.empty? && !base.strip.start_with?("-"))
+        raise ArgumentError,
+          'code_change "ref" needs code_change_base: the branch or commit to diff against, such as "dev" or a commit SHA.'
+      end
+      if code_change != "ref" && !base.nil?
+        raise ArgumentError, "code_change_base only applies with code_change \"ref\" (got \"#{code_change}\")."
+      end
       if code_change == "skip" && !description.nil?
         raise ArgumentError, 'code_change "skip" attaches no code change, so it cannot take a code_change_description.'
       end
       code_change
     end
 
-    def resolve_code_change(code_change, description, files, label = nil)
+    def resolve_code_change(code_change, description, files, label = nil, base = nil)
       return [nil, nil] if code_change == "skip"
       return [description, files] if code_change == "manual"
 
-      captured = read_code_change_file || capture_code_change_from_git(Dir.pwd, code_change, label)
+      captured = read_code_change_file || capture_code_change_from_git(Dir.pwd, code_change, label, base&.strip)
       if captured.nil? || captured[1].nil? || captured[1].empty?
         warn "Bitfab: code_change \"#{code_change}\" found no changed files, so this replay has no code change attached."
         return [description, nil]
@@ -713,8 +726,21 @@ module Bitfab
       !git(root, ["rev-parse", "--verify", "#{ref}^{object}"]).nil?
     end
 
-    def cc_resolve_base(root, type)
+    def cc_resolve_chosen_base(root, chosen)
+      [chosen, "origin/#{chosen}"].each do |ref|
+        next unless cc_ref_exists?(root, "#{ref}^{commit}")
+
+        base = (git(root, ["merge-base", "HEAD", ref]) || "").strip
+        return base.empty? ? nil : [base, "vs #{chosen}"]
+      end
+      raise ArgumentError,
+        "code_change_base \"#{chosen}\" names no branch or commit in this repository. Pass a branch name, such as " \
+          "\"dev\", or a commit SHA; fetch it first if it only exists on the remote."
+    end
+
+    def cc_resolve_base(root, type, chosen = nil)
       case type
+      when "ref" then chosen ? cc_resolve_chosen_base(root, chosen) : nil
       when "last-branch" then cc_resolve_branch_point(root)
       when "default-branch" then cc_resolve_default_branch_point(root)
       else cc_ref_exists?(root, "HEAD") ? ["HEAD", "uncommitted (vs HEAD)"] : nil
@@ -809,12 +835,12 @@ module Bitfab
       0
     end
 
-    def capture_code_change_from_git(cwd, type, label = nil)
+    def capture_code_change_from_git(cwd, type, label = nil, chosen = nil)
       root = git(cwd, ["rev-parse", "--show-toplevel"])
       return nil if root.nil? || root.strip.empty?
 
       root = root.strip
-      resolved = cc_resolve_base(root, type)
+      resolved = cc_resolve_base(root, type, chosen)
       return nil if resolved.nil?
 
       entries = cc_list_changes(root, resolved[0])
@@ -849,6 +875,8 @@ module Bitfab
       head = ((label && !label.strip.empty?) ? label.strip : nil) || (subject.empty? ? nil : subject) || "Working-tree change"
       word = (files.length == 1) ? "file" : "files"
       ["#{head} (#{files.length} #{word} changed #{against})", files]
+    rescue ArgumentError
+      raise
     rescue
       nil
     end

data/lib/bitfab/replay_registry.rb

@@ -204,6 +204,7 @@ module Bitfab
       end
 
       options[:code_change] = args.fetch(:code_change)
+      options[:code_change_base] = args[:code_change_base] if args[:code_change_base]
       if args[:code_change_file]
         code_change = load_code_change(args[:code_change_file])
         options[:code_change_description] = code_change.fetch("description")
@@ -337,10 +338,11 @@ module Bitfab
         options.on("--max-concurrency N", Integer) do |value|
           args[:max_concurrency] = positive_integer("--max-concurrency", value)
         end
-        options.on("--code-change working-tree|last-branch|default-branch|skip|manual",
+        options.on("--code-change working-tree|last-branch|default-branch|ref|skip|manual",
           "Required: the code change this run tests; working-tree is the uncommitted changes vs HEAD, " \
             "last-branch the diff against the branch this one was cut from, default-branch the diff against the " \
-            "default branch, skip none, manual the edits in --code-change-file.") do |value|
+            "default branch, ref the diff against --code-change-base, skip none, manual the edits in " \
+            "--code-change-file.") do |value|
           unless Replay::CODE_CHANGE_TYPES.include?(value)
             raise ArgumentError,
               "Unknown --code-change '#{value}'. Pass #{Replay::CODE_CHANGE_CHOICES}. " \
@@ -350,6 +352,8 @@ module Bitfab
         end
         options.on("--code-change-file PATH",
           "With --code-change manual: a JSON file of { description, files }.") { |value| args[:code_change_file] = value }
+        options.on("--code-change-base BRANCH|COMMIT",
+          "With --code-change ref: the branch or commit to diff against.") { |value| args[:code_change_base] = value }
         options.on("--experiment-group-id UUID",
           "Add this run to an experiment group. A UUID no group has yet creates the group.") { |value| args[:experiment_group_id] = value }
         options.on("--experiment-group-name NAME",
@@ -390,12 +394,17 @@ module Bitfab
     def require_code_change(args)
       if args[:code_change].nil?
         raise ArgumentError,
-          "--code-change is required: #{Replay::CODE_CHANGE_CHOICES}. \"manual\" takes the edits from --code-change-file PATH."
+          "--code-change is required: #{Replay::CODE_CHANGE_CHOICES}. \"ref\" takes the branch or commit from " \
+            "--code-change-base, and \"manual\" takes the edits from --code-change-file PATH."
       end
-      return args if (args[:code_change] == "manual") == !args[:code_change_file].nil?
+      unless (args[:code_change] == "manual") == !args[:code_change_file].nil?
+        raise ArgumentError,
+          "--code-change-file goes with --code-change manual, and --code-change manual needs --code-change-file PATH."
+      end
+      return args if (args[:code_change] == "ref") == !args[:code_change_base].nil?
 
       raise ArgumentError,
-        "--code-change-file goes with --code-change manual, and --code-change manual needs --code-change-file PATH."
+        "--code-change-base goes with --code-change ref, and --code-change ref needs --code-change-base BRANCH|COMMIT."
     end
 
     def registry_dataset_ids(options)

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.70.0"
+  VERSION = "0.71.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.70.0
+  version: 0.71.1
 platform: ruby
 authors:
 - Harvest Team