bitfab 0.51.0 → 0.51.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ece2db3c44be909ec097e539479143d0c4138663809c26a05a968076e681066
-  data.tar.gz: 6d68e352c7e484586147d1700998de7936fe904c5d04cabfe07df80e302104b1
+  metadata.gz: ea799ee55cf7c1ac7167300d800d9593162ddc6b906557c12c3079607fcfc4f4
+  data.tar.gz: f7985925a6a8b3cf8c5e2ea4389bd8122646f5bf2eb5bf59e6769d9f533636b1
 SHA512:
-  metadata.gz: bbb4750e80b63864d612592e1bebaac664463059e2d139fe7d556f68ce73b46868001daa54391d1a4562767a2e7e16355fdb8eefc89c64ace8f864dffffce86b
-  data.tar.gz: 02d731c682b787b8233a1c9a43135fc8b6d17d5959806ff65bec3be9979dbda95d1bfb6d01373b427405a5fb9fccd976c87706d8d488ea7733fe70ab0f8fa15b
+  metadata.gz: 10c67dc35cf10ed1c445108ce1801993b8f4115afda9411fe1dc3b9e12213ab090441e91c6a59a0864925822a09d344360e4d10ab15cac0a1a183c56d007972e
+  data.tar.gz: 2f968a4c58b66fd51d49c74199b20207a44aecd56285656b2ece8df04a76e9070b9f87c7dcbb53f2eeedfe61ba0a271d68660b934b4a56eb1f8fca278221f452

data/README.md

@@ -399,6 +399,8 @@ batch is never re-encoded to measure its size. If Bitfab accepts only part of a
 the standard OTLP `partialSuccess` response and the SDK warns with the rejected-span count and
 reason.
 
+Every root trace also records the commit its code ran at as `commit_ref`. Deploy platforms such as Vercel, GitHub Actions, Railway, Render, Heroku, Cloudflare Pages, GitLab CI, Azure Pipelines, and CircleCI supply it through their build variables, a plain checkout resolves it from `git` once per process in the background, and a container built without either should set `BITFAB_COMMIT_SHA` at build time. Set `BITFAB_DISABLE_COMMIT_REF` to send no `commit_ref` at all.
+
 Before finalizing a replay, the SDK flushes OTel and polls Bitfab's replay-status API until every
 expected trace completion and span count is persisted.
 

data/lib/bitfab/client.rb

@@ -51,6 +51,7 @@ module Bitfab
       # replay on this client (after any per-call mock_override). Instance
       # state, no global; clear_mock_overrides resets it.
       @mock_overrides = []
+      CommitRef.start_resolution
     end
 
     # The configured API key (a proc is resolved on read). Reflects what was
@@ -95,9 +96,12 @@ module Bitfab
     # @param limit [Integer, nil] maximum number of traces to replay (default: 5;
     #   maximum: 5,000). Ignored when trace_ids or a dataset is passed because
     #   either source already determines how many traces replay. Supplying
-    #   trace_ids also emits a warning.
-    # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100),
-    #   mutually exclusive with dataset_id and dataset_ids
+    #   trace_ids also emits a warning. To replay part of a dataset selection,
+    #   name the members to run in trace_ids rather than bounding it here.
+    # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100).
+    #   Passed alongside dataset_id or dataset_ids it pins which members of that
+    #   selection replay, and the server rejects any ID none of those datasets
+    #   contains
     # @param max_concurrency [Integer, nil] max threads for parallel replay (default: 10)
     # @param code_change_description [String, nil] optional rationale for the
     #   code change being tested in this replay (stored on the experiment).
@@ -112,10 +116,12 @@ module Bitfab
     #   replay runs into a single experiment batch
     # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
     #   replay runs against, for benchmarking one method against several corpora
-    #   in a single run; mutually exclusive with dataset_id
+    #   in a single run; mutually exclusive with dataset_id. Passing trace_ids
+    #   alongside it narrows the run to those members
     # @param dataset_id [String, nil] optional UUID of the dataset this replay
-    #   runs against, mutually exclusive with trace_ids and stored on the
-    #   resulting experiment for durable attribution
+    #   runs against, mutually exclusive with dataset_ids and stored on the
+    #   resulting experiment for durable attribution. Alone it replays the
+    #   dataset's full membership, and with trace_ids only those members
     # @param grader_ids [Array<String>, nil] optional UUIDs of graders attached
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
@@ -668,6 +674,8 @@ module Bitfab
       if trace_state&.dig(:db_snapshot_ref)
         raw_trace["db_snapshot_ref"] = trace_state[:db_snapshot_ref]
       end
+      commit_ref = CommitRef.current
+      raw_trace["commit_ref"] = commit_ref if commit_ref
       if db_snapshot_usage
         usage = {"neon_branch_id" => db_snapshot_usage[:neon_branch_id]}
         if db_snapshot_usage[:snapshot_timestamp]

data/lib/bitfab/commit_ref.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require "uri"
+require_relative "git_command"
+
+module Bitfab
+  module CommitRef
+    EXPLICIT_SHA_ENV = "BITFAB_COMMIT_SHA"
+    DISABLE_ENV = "BITFAB_DISABLE_COMMIT_REF"
+    GIT_TIMEOUT_SECONDS = 2
+
+    VERCEL_PROVIDER_HOSTS = {
+      "github" => "github.com",
+      "gitlab" => "gitlab.com",
+      "bitbucket" => "bitbucket.org"
+    }.freeze
+
+    PLATFORM_ENVS = [
+      ["VERCEL_GIT_COMMIT_SHA", "VERCEL_GIT_COMMIT_REF", :vercel],
+      ["GITHUB_SHA", "GITHUB_REF_NAME", ["GITHUB_SERVER_URL", "GITHUB_REPOSITORY"]],
+      ["RAILWAY_GIT_COMMIT_SHA", "RAILWAY_GIT_BRANCH", nil],
+      ["RENDER_GIT_COMMIT", "RENDER_GIT_BRANCH", nil],
+      ["SOURCE_VERSION", nil, nil],
+      ["CF_PAGES_COMMIT_SHA", "CF_PAGES_BRANCH", nil],
+      ["CI_COMMIT_SHA", "CI_COMMIT_REF_NAME", ["CI_REPOSITORY_URL"]],
+      ["BUILD_SOURCEVERSION", "BUILD_SOURCEBRANCHNAME", ["BUILD_REPOSITORY_URI"]],
+      ["CIRCLE_SHA1", "CIRCLE_BRANCH", ["CIRCLE_REPOSITORY_URL"]]
+    ].freeze
+
+    REMOTE_ENVS = %w[
+      GITHUB_SERVER_URL GITHUB_REPOSITORY CI_REPOSITORY_URL BUILD_REPOSITORY_URI
+      CIRCLE_REPOSITORY_URL VERCEL_GIT_PROVIDER VERCEL_GIT_REPO_OWNER VERCEL_GIT_REPO_SLUG
+    ].freeze
+
+    ENV_NAMES = ([EXPLICIT_SHA_ENV, DISABLE_ENV] + PLATFORM_ENVS.flat_map { |sha, branch, _| [sha, branch].compact } + REMOTE_ENVS).freeze
+
+    @lock = Mutex.new
+    @resolved = false
+    @ref = nil
+    @git_thread = nil
+
+    module_function
+
+    def read(env, name)
+      return nil if name.nil?
+
+      value = env[name].to_s.strip
+      value.empty? ? nil : value
+    end
+
+    def remote_from_env(env, spec)
+      case spec
+      when :vercel
+        host = VERCEL_PROVIDER_HOSTS[read(env, "VERCEL_GIT_PROVIDER").to_s.downcase]
+        owner = read(env, "VERCEL_GIT_REPO_OWNER")
+        slug = read(env, "VERCEL_GIT_REPO_SLUG")
+        (host && owner && slug) ? "#{host}/#{owner}/#{slug}" : nil
+      when Array
+        parts = spec.map { |name| read(env, name) }
+        parts.all? ? normalize_remote(parts.join("/")) : nil
+      end
+    end
+
+    def normalize_remote(raw)
+      value = raw.to_s.strip
+      return nil if value.empty?
+
+      if value.include?("://")
+        parsed = begin
+          URI.parse(value)
+        rescue URI::InvalidURIError
+          return nil
+        end
+        host = parsed.host.to_s.downcase
+        path = parsed.path.to_s
+      else
+        head, separator, tail = value.partition(":")
+        host = separator.empty? ? "" : head.rpartition("@").last
+        path = separator.empty? ? value : tail
+      end
+      path = path.gsub(%r{\A/+|/+\z}, "")
+      path = path.delete_suffix(".git").sub(%r{/+\z}, "") if path.end_with?(".git")
+      return (path.empty? ? nil : path) if host.empty?
+
+      path.empty? ? host : "#{host}/#{path}"
+    end
+
+    def resolve_from_env(env)
+      explicit = read(env, EXPLICIT_SHA_ENV)
+      platform = PLATFORM_ENVS.find { |sha_env, _, _| read(env, sha_env) }
+      if platform.nil?
+        return nil if explicit.nil?
+
+        return {"sha" => explicit, "branch" => nil, "dirty" => nil, "remote" => nil, "root_sha" => nil}
+      end
+
+      sha_env, branch_env, remote_spec = platform
+      {
+        "sha" => explicit || read(env, sha_env),
+        "branch" => read(env, branch_env),
+        "dirty" => nil,
+        "remote" => remote_from_env(env, remote_spec),
+        "root_sha" => nil
+      }
+    end
+
+    def git(cwd, *args)
+      GitCommand.run(cwd, args, timeout: GIT_TIMEOUT_SECONDS)&.strip
+    end
+
+    def resolve_from_git(cwd)
+      sha = git(cwd, "rev-parse", "HEAD")
+      return nil if sha.nil? || sha.empty?
+
+      branch = git(cwd, "symbolic-ref", "--short", "-q", "HEAD")
+      status = git(cwd, "status", "--porcelain")
+      roots = git(cwd, "rev-list", "--max-parents=0", "HEAD")
+      {
+        "sha" => sha,
+        "branch" => (branch.nil? || branch.empty?) ? nil : branch,
+        "dirty" => status.nil? ? nil : !status.empty?,
+        "remote" => normalize_remote(git(cwd, "remote", "get-url", "origin")),
+        "root_sha" => (roots.nil? || roots.empty?) ? nil : roots.split.min
+      }
+    end
+
+    def start_resolution
+      @lock.synchronize do
+        return if @resolved || @git_thread
+
+        if read(ENV, DISABLE_ENV)
+          @resolved = true
+          return
+        end
+
+        from_env = resolve_from_env(ENV)
+        if from_env
+          @ref = from_env
+          @resolved = true
+          return
+        end
+
+        cwd = begin
+          Dir.pwd
+        rescue SystemCallError
+          @resolved = true
+          return
+        end
+        @git_thread = Thread.new { resolve_git_in_background(cwd) }
+      end
+    end
+
+    def resolve_git_in_background(cwd)
+      ref = resolve_from_git(cwd)
+      @lock.synchronize do
+        unless @resolved
+          @ref = ref
+          @resolved = true
+        end
+      end
+    rescue
+      @lock.synchronize { @resolved = true }
+    end
+
+    def current
+      start_resolution
+      @lock.synchronize { @ref }
+    end
+
+    def settle(timeout = 5)
+      thread = @lock.synchronize { @git_thread }
+      thread&.join(timeout)
+      nil
+    end
+
+    def reset!
+      @lock.synchronize do
+        @ref = nil
+        @resolved = false
+        @git_thread = nil
+      end
+    end
+  end
+end

data/lib/bitfab/git_command.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module Bitfab
+  module GitCommand
+    KILL_GRACE_SECONDS = 1
+
+    module_function
+
+    def run(cwd, args, timeout:)
+      Open3.popen3("git", *args, chdir: cwd, **spawn_options) do |stdin, stdout, stderr, wait_thread|
+        stdin.close
+        out = +""
+        reader = Thread.new { out = read_all(stdout) }
+        drain = Thread.new { read_all(stderr) }
+        finished = wait_thread.join(timeout)
+        stop(wait_thread) unless finished
+        settle(reader, stdout)
+        settle(drain, stderr)
+        (finished && wait_thread.value.success?) ? out.force_encoding("UTF-8").scrub : nil
+      end
+    rescue
+      nil
+    end
+
+    def spawn_options
+      Gem.win_platform? ? {} : {pgroup: true}
+    end
+
+    def kill_target(wait_thread)
+      Gem.win_platform? ? wait_thread.pid : -wait_thread.pid
+    end
+
+    def read_all(io)
+      io.read.to_s
+    rescue
+      ""
+    end
+
+    def settle(thread, io)
+      return if thread.join(KILL_GRACE_SECONDS)
+
+      io.close
+      thread.join
+    end
+
+    def stop(wait_thread)
+      target = kill_target(wait_thread)
+      if Gem.win_platform?
+        signal("KILL", target)
+      else
+        signal("TERM", target)
+        signal("KILL", target) unless wait_thread.join(KILL_GRACE_SECONDS)
+      end
+      wait_thread.join
+    end
+
+    def signal(name, target)
+      Process.kill(name, target)
+    rescue
+      nil
+    end
+  end
+end

data/lib/bitfab/replay.rb

@@ -6,6 +6,7 @@ require "open3"
 require "timeout"
 
 require_relative "constants"
+require_relative "git_command"
 require_relative "mock_override"
 require_relative "serialize"
 require_relative "traceable"
@@ -206,12 +207,14 @@ module Bitfab
     #   replay runs into a single experiment batch
     # @param dataset_id [String, nil] optional UUID of the dataset this replay
     #   runs against, stored on the resulting experiment for durable attribution;
-    #   mutually exclusive with dataset_ids
+    #   mutually exclusive with dataset_ids. Alone it replays the dataset's full
+    #   membership, and with trace_ids it replays only those members
     # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
     #   replay runs against, for benchmarking one method against several corpora
     #   in a single run; mutually exclusive with dataset_id. The run replays the
     #   union of their traces, graded by the union of their graders, and is
-    #   attributed to every one of them
+    #   attributed to every one of them. Passing trace_ids alongside it narrows
+    #   the run to those members
     # @param grader_ids [Array<String>, nil] optional UUIDs of graders attached
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
@@ -267,10 +270,6 @@ module Bitfab
         end
       end
       resolved_dataset_ids = resolve_dataset_ids(dataset_id, dataset_ids)
-      if trace_ids && resolved_dataset_ids
-        raise ArgumentError,
-          "trace_ids and dataset_ids select different replay sources and cannot be used together."
-      end
       if limit && trace_ids
         warn "Bitfab: limit is ignored when trace_ids is passed: the explicit trace ID list already " \
           "determines how many traces replay."
@@ -562,20 +561,7 @@ module Bitfab
     end
 
     def git(cwd, args)
-      # capture3 so git's stderr (e.g. "not a git repository") is swallowed, not
-      # leaked to the host process's console.
-      out, _err, status = Timeout.timeout(30) do
-        Open3.capture3("git", *args, chdir: cwd)
-      end
-      return nil unless status.success?
-
-      # Tag as UTF-8 and scrub invalid bytes so blob contents compare cleanly
-      # against the UTF-8 working-file reads and never break JSON.generate later
-      # (a non-UTF-8 file that slips past the null-byte check would otherwise
-      # abort the whole start-replay request).
-      out.force_encoding("UTF-8").scrub
-    rescue
-      nil
+      GitCommand.run(cwd, args, timeout: 30)
     end
 
     def cc_ref_exists?(root, ref)

data/lib/bitfab/replay_registry.rb

@@ -81,8 +81,26 @@ module Bitfab
 
   # Argument parsing and output for the SDK-owned replay command.
   module ReplayCli
+    MAX_PINNED_TRACE_IDS = 100
+
     module_function
 
+    def pinned_dataset_members(client, dataset_ids, limit)
+      members = dataset_ids.flat_map { |dataset_id|
+        client.datasets.list_traces(dataset_id).fetch("traceIds")
+      }.uniq.sort
+      return nil if limit >= members.length
+
+      if limit > MAX_PINNED_TRACE_IDS
+        raise ArgumentError,
+          "Bounding a dataset replay to #{limit} names one trace ID per replayed item, and " \
+            "a replay accepts at most #{MAX_PINNED_TRACE_IDS}. Drop the limit to replay " \
+            "every selected dataset in full."
+      end
+
+      members.first(limit)
+    end
+
     def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
       args = parse(registry, argv.dup)
       registration = registry.fetch(args.fetch(:pipeline))
@@ -108,20 +126,25 @@ module Bitfab
       end
 
       if args[:trace_ids]
-        options.delete(:limit)
         options.delete(:dataset_ids)
         options[:trace_ids] = args[:trace_ids]
       elsif args[:dataset_ids]
         options.delete(:trace_ids)
-        options.delete(:limit)
-      elsif args[:limit]
-        options.delete(:trace_ids)
-        options.delete(:dataset_ids)
-        options[:limit] = args[:limit]
-      elsif options[:trace_ids] || selected_dataset_ids
-        options.delete(:limit)
+      end
+
+      bound = args[:limit] || options[:limit]
+      options.delete(:limit)
+      bounded_dataset_ids = args[:dataset_ids] || options[:dataset_ids]
+
+      if options[:trace_ids]
+        options[:trace_ids] = options[:trace_ids].first(bound) if bound
+      elsif bounded_dataset_ids
+        if bound
+          pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound)
+          options[:trace_ids] = pinned if pinned
+        end
       else
-        options[:limit] = options.fetch(:limit, 10)
+        options[:limit] = bound || 10
       end
 
       %i[name max_concurrency experiment_group_id dataset_ids grader_ids mock].each do |key|
@@ -179,7 +202,10 @@ module Bitfab
       args = {}
       parser = OptionParser.new do |options|
         options.banner = "Usage: bitfab-replay <#{registry.names.join("|")}> [options]"
-        options.on("--limit N", Integer) { |value| args[:limit] = positive_integer("--limit", value) }
+        options.on("--limit N", Integer,
+          "How many traces to replay. It bounds a trace ID or dataset selection instead of replacing it") do |value|
+          args[:limit] = positive_integer("--limit", value)
+        end
         options.on("--trace-ids IDS") { |value| args[:trace_ids] = comma_separated("--trace-ids", value) }
         options.on("--name NAME") { |value| args[:name] = value }
         options.on("--concurrency N", Integer) do |value|

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.51.0"
+  VERSION = "0.51.2"
 end

data/lib/bitfab.rb

@@ -8,6 +8,8 @@ require_relative "bitfab/warn_once"
 require_relative "bitfab/payload_budget"
 require_relative "bitfab/serialize"
 require_relative "bitfab/db_snapshot"
+require_relative "bitfab/git_command"
+require_relative "bitfab/commit_ref"
 require_relative "bitfab/span_context"
 require_relative "bitfab/http_client"
 require_relative "bitfab/datasets"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.51.0
+  version: 0.51.2
 platform: ruby
 authors:
 - Harvest Team
@@ -140,10 +140,12 @@ files:
 - exe/bitfab-replay
 - lib/bitfab.rb
 - lib/bitfab/client.rb
+- lib/bitfab/commit_ref.rb
 - lib/bitfab/compress.rb
 - lib/bitfab/constants.rb
 - lib/bitfab/datasets.rb
 - lib/bitfab/db_snapshot.rb
+- lib/bitfab/git_command.rb
 - lib/bitfab/http_client.rb
 - lib/bitfab/mock_override.rb
 - lib/bitfab/otel.rb