bitfab 0.38.0 → 0.38.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e4bb59de5ab0ec989d85e8be14ee08ad935ada5145e01826b34657fc8c9e6623
-  data.tar.gz: 78b0255c5a3062470f4149baf632c94a5b18b430567485a1ba58f5b9cf9b9534
+  metadata.gz: 2816ce4e3bcd11bb1335ba7dac6b36376277d17075cf853d26c9befa231a758b
+  data.tar.gz: 7e923c2efb8dfd38edbdd592972a8f236a3fa81cdfabcbd46b8303ba4160b01e
 SHA512:
-  metadata.gz: e127f198a39372bebaff5a0cf395b4b6b00f2ede4bf9768b8e3a4556ee00271ddf44fe4a40f16fcffac72d3def4d3c83c3f756801ba156edbe58ecbfd189f2e5
-  data.tar.gz: ca803b3df3783f6bb01d5eb5a518300334d0ccd4f31d43f3870c981779cd69710d076a1ded3f058c3b9e8aa65f643b3872dd60f99ac8e4394cd5f97855ec1970
+  metadata.gz: 15a2df82260f61733b90da3c110b195c92373c187c3eddce5faa041f44499bb008ec5a14cd27dab22b63e2973540010cc2e2f10fcc2dcd75481405d018a107f6
+  data.tar.gz: 13968f6a9f8d3c88ea2845d0d59e0cc1d31ffadc1054311addeedc1e4ba92d94aea504f0f5d11e9891fd0a233e92d285e635e5d8e85e46ebfdf553cdedc45005

data/exe/bitfab-replay

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bitfab"
+
+exit Bitfab::ReplayCommand.main

data/lib/bitfab/client.rb

@@ -91,7 +91,8 @@ module Bitfab
     #   maximum: 5,000). Ignored when trace_ids or dataset_id is passed because
     #   either source already determines how many traces replay. Supplying
     #   trace_ids also emits a warning.
-    # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100)
+    # @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100),
+    #   mutually exclusive with dataset_id
     # @param max_concurrency [Integer, nil] max threads for parallel replay (default: 10)
     # @param code_change_description [String, nil] optional rationale for the
     #   code change being tested in this replay (stored on the experiment).
@@ -105,7 +106,8 @@ module Bitfab
     # @param experiment_group_id [String, nil] optional UUID grouping multiple
     #   replay runs into a single experiment batch
     # @param dataset_id [String, nil] optional UUID of the dataset this replay
-    #   runs against, stored on the resulting experiment for durable attribution
+    #   runs against, mutually exclusive with trace_ids and stored on the
+    #   resulting experiment for durable attribution
     # @param grader_ids [Array<String>, nil] optional UUIDs of graders attached
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the

data/lib/bitfab/replay.rb

@@ -260,11 +260,14 @@ module Bitfab
           raise ArgumentError, "trace_ids supports at most 100 trace IDs per replay (got #{trace_ids.length})."
         end
       end
+      if trace_ids && dataset_id
+        raise ArgumentError,
+          "trace_ids and dataset_id select different replay sources and cannot be used together."
+      end
       if limit && trace_ids
         warn "Bitfab: limit is ignored when trace_ids is passed: the explicit trace ID list already " \
           "determines how many traces replay."
       end
-
       # Reject a trace_function_key that contradicts the method's declared key.
       # replay() fetches historical traces by trace_function_key but records the
       # replayed spans under the method's own bitfab_span key (via send below),

data/lib/bitfab/replay_cli.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module Bitfab
+  # Installed command that loads a project-owned registry module.
+  module ReplayCommand
+    USAGE = "Usage: bitfab-replay --registry <path> <pipeline> [replay options]"
+    HELP = "#{USAGE}\nRun bitfab-replay --registry <path> --help to list replay options."
+
+    module_function
+
+    def run(argv: ARGV, stdout: $stdout, stderr: $stderr, registry_loader: nil)
+      registry_path, replay_argv = parse(argv)
+      loader = registry_loader || method(:load_registry)
+      registry = loader.call(registry_path)
+      ReplayCli.run(registry, argv: replay_argv, stdout:, stderr:)
+    end
+
+    def main(argv: ARGV, stdout: $stdout, stderr: $stderr)
+      if (argv.include?("--help") || argv.include?("-h")) && !argv.include?("--registry")
+        stdout.puts HELP
+        return 0
+      end
+      run(argv:, stdout:, stderr:)
+      0
+    rescue => error
+      stderr.puts error.message
+      1
+    end
+
+    def parse(argv)
+      values = argv.dup
+      registry_index = values.index("--registry")
+      registry_path = values[registry_index + 1] if registry_index
+      unless registry_path && !registry_path.start_with?("--")
+        raise OptionParser::MissingArgument, USAGE
+      end
+
+      values.slice!(registry_index, 2)
+      raise OptionParser::MissingArgument, USAGE if values.empty?
+
+      [File.expand_path(registry_path), values]
+    end
+
+    def load_registry(path)
+      container = Module.new
+      container.module_eval(File.read(path), path)
+      registry = container.const_get(:REGISTRY, false) if container.const_defined?(:REGISTRY, false)
+      unless registry.is_a?(ReplayRegistry)
+        raise ArgumentError,
+          "Replay registry '#{path}' must define REGISTRY as a Bitfab::ReplayRegistry."
+      end
+
+      registry
+    end
+  end
+end

data/lib/bitfab/replay_registry.rb

@@ -0,0 +1,290 @@
+# frozen_string_literal: true
+
+require "json"
+require "optparse"
+
+module Bitfab
+  # One project function exposed through the SDK-owned replay command.
+  ReplayRegistration = Data.define(
+    :client,
+    :receiver,
+    :method_name,
+    :trace_function_key,
+    :options,
+    :options_factory
+  )
+
+  ReplayRegistryContext = Data.define(:params)
+
+  # Project-owned registry of production replay roots.
+  class ReplayRegistry
+    OPTION_NAMES = %i[
+      limit trace_ids name max_concurrency code_change_description
+      code_change_files experiment_group_id dataset_id grader_ids mock
+      mock_override adapt_inputs db_branch
+    ].freeze
+
+    def initialize
+      @entries = {}
+    end
+
+    def names
+      @entries.keys.freeze
+    end
+
+    # Register the exact traced receiver method production invokes.
+    #
+    # A +bitfab_span+ method carries its trace function key automatically.
+    # Handler-instrumented or otherwise plain methods must pass
+    # +trace_function_key:+ explicitly. Pass executable per-function replay
+    # behavior such as +mock_override:+ and +adapt_inputs:+ as keyword options;
+    # the installed command forwards them to +Bitfab::Client#replay+.
+    def register(name, receiver, method_name, client: nil, trace_function_key: nil, options_factory: nil, **options)
+      raise ArgumentError, "Replay registry names cannot be empty." if name.to_s.empty?
+      raise ArgumentError, "Replay registry already contains '#{name}'." if @entries.key?(name.to_s)
+      if options_factory && !options_factory.respond_to?(:call)
+        raise ArgumentError, "Replay registry options_factory must be callable."
+      end
+
+      key = trace_function_key || Traceable.trace_function_key_for(receiver, method_name)
+      unless key
+        raise ArgumentError,
+          "Replay registry entry uses a plain method. Set trace_function_key: " \
+            "to the key its production handler records."
+      end
+      validate_options(options)
+
+      @entries[name.to_s] = ReplayRegistration.new(
+        client: client || Bitfab.client,
+        receiver:,
+        method_name: method_name.to_sym,
+        trace_function_key: key,
+        options: options.freeze,
+        options_factory:
+      )
+      self
+    end
+
+    def fetch(name)
+      @entries.fetch(name)
+    end
+
+    def validate_options(options)
+      unknown = options.keys - OPTION_NAMES
+      return if unknown.empty?
+
+      raise ArgumentError,
+        "Unknown replay registry option(s): #{unknown.sort.join(", ")}. " \
+          "Allowed options: #{OPTION_NAMES.sort.join(", ")}."
+    end
+  end
+
+  # Argument parsing and output for the SDK-owned replay command.
+  module ReplayCli
+    module_function
+
+    def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
+      args = parse(registry, argv.dup)
+      registration = registry.fetch(args.fetch(:pipeline))
+      options = registration.options.dup
+      params = load_parameters(args[:params], args.fetch(:param, []))
+      if registration.options_factory
+        dynamic_options = registration.options_factory.call(ReplayRegistryContext.new(params:))
+        unless dynamic_options.is_a?(Hash)
+          raise ArgumentError, "Replay registry options_factory must return a Hash."
+        end
+        registry.validate_options(dynamic_options)
+        options.merge!(dynamic_options)
+      end
+
+      if options[:trace_ids] && options[:dataset_id]
+        raise ArgumentError,
+          "Replay registry options trace_ids and dataset_id select different sources and cannot be used together."
+      end
+
+      if args[:trace_ids]
+        options.delete(:limit)
+        options.delete(:dataset_id)
+        options[:trace_ids] = args[:trace_ids]
+      elsif args[:dataset_id]
+        options.delete(:trace_ids)
+        options.delete(:limit)
+      elsif args[:limit]
+        options.delete(:trace_ids)
+        options.delete(:dataset_id)
+        options[:limit] = args[:limit]
+      elsif options[:trace_ids] || options[:dataset_id]
+        options.delete(:limit)
+      else
+        options[:limit] = options.fetch(:limit, 10)
+      end
+
+      %i[name max_concurrency experiment_group_id dataset_id grader_ids mock].each do |key|
+        options[key] = args[key] unless args[key].nil?
+      end
+
+      unless args[:db_branch].nil?
+        if args[:db_branch]
+          configured = options[:db_branch]
+          options[:db_branch] = (configured.nil? || configured == false) ? true : configured
+        else
+          options[:db_branch] = false
+        end
+      end
+
+      if args[:code_change]
+        code_change = load_code_change(args[:code_change])
+        options[:code_change_description] = code_change.fetch("description")
+        options[:code_change_files] = code_change.fetch("files")
+      elsif args[:no_code_change]
+        options[:code_change_description] = nil
+        options[:code_change_files] = nil
+      end
+
+      reporter = Bitfab.method(:report_replay_progress)
+      options[:on_item_start] = reporter
+      options[:on_item_finish] = reporter
+
+      count = options[:trace_ids]&.length || options[:limit] || "dataset"
+      stderr.puts "[replay] Replaying #{count} traces from \"#{registration.trace_function_key}\"..."
+
+      result = begin
+        registration.client.replay(
+          registration.receiver,
+          registration.method_name,
+          trace_function_key: registration.trace_function_key,
+          **options
+        )
+      rescue ReplayError => error
+        error.items.each do |item|
+          item_error = item[:trace_error] || item[:replay_error] || item[:error]
+          stderr.puts "#{item[:original_trace_id]}: #{item_error}"
+        end
+        raise
+      end
+
+      render_summary(args.fetch(:pipeline), result, stderr)
+      stdout.puts Bitfab.serialize_replay_result(result)
+      result
+    end
+
+    def parse(registry, argv)
+      if argv.include?("--db-branch") && argv.include?("--no-db-branch")
+        raise OptionParser::InvalidArgument, "--db-branch and --no-db-branch cannot be used together"
+      end
+      args = {}
+      parser = OptionParser.new do |options|
+        options.banner = "Usage: bitfab-replay <#{registry.names.join("|")}> [options]"
+        options.on("--limit N", Integer) { |value| args[:limit] = positive_integer("--limit", value) }
+        options.on("--trace-ids IDS") { |value| args[:trace_ids] = comma_separated("--trace-ids", value) }
+        options.on("--name NAME") { |value| args[:name] = value }
+        options.on("--concurrency N", Integer) do |value|
+          args[:max_concurrency] = positive_integer("--concurrency", value)
+        end
+        options.on("--max-concurrency N", Integer) do |value|
+          args[:max_concurrency] = positive_integer("--max-concurrency", value)
+        end
+        options.on("--code-change PATH") { |value| args[:code_change] = value }
+        options.on("--experiment-group-id UUID") { |value| args[:experiment_group_id] = value }
+        options.on("--dataset-id UUID") { |value| args[:dataset_id] = value }
+        options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
+        options.on("--mock STRATEGY", %w[none all marked]) { |value| args[:mock] = value }
+        options.on("--db-branch") { args[:db_branch] = true }
+        options.on("--no-db-branch") { args[:db_branch] = false }
+        options.on("--no-code-change") { args[:no_code_change] = true }
+        options.on("--params PATH") { |value| args[:params] = value }
+        options.on("--param NAME=VALUE") { |value| (args[:param] ||= []) << value }
+      end
+      parser.parse!(argv)
+      if args[:trace_ids] && args[:dataset_id]
+        raise OptionParser::InvalidArgument,
+          "--trace-ids and --dataset-id select different replay sources and cannot be used together"
+      end
+      if args[:code_change] && args[:no_code_change]
+        raise OptionParser::InvalidArgument, "--code-change and --no-code-change cannot be used together"
+      end
+      pipeline = argv.shift
+      raise OptionParser::MissingArgument, parser.banner unless pipeline && registry.names.include?(pipeline)
+      raise OptionParser::InvalidArgument, "unexpected arguments: #{argv.join(" ")}" unless argv.empty?
+
+      args.merge(pipeline:)
+    end
+
+    def positive_integer(flag, value)
+      raise OptionParser::InvalidArgument, "#{flag} must be a positive integer" if value < 1
+
+      value
+    end
+
+    def comma_separated(flag, value)
+      values = value.split(",").map(&:strip).reject(&:empty?)
+      raise OptionParser::InvalidArgument, "#{flag} must contain at least one value" if values.empty?
+
+      values
+    end
+
+    def load_code_change(path)
+      value = JSON.parse(File.read(path))
+      unless value.is_a?(Hash) && value["description"].is_a?(String) && value["files"].is_a?(Array)
+        raise ArgumentError, "Invalid --code-change file '#{path}': expected { description, files }."
+      end
+
+      value
+    end
+
+    def parse_parameter(raw)
+      key, separator, value = raw.partition("=")
+      key = key.strip
+      if separator.empty? || key.empty?
+        raise OptionParser::InvalidArgument,
+          "Invalid --param '#{raw}': expected a non-empty name=value pair"
+      end
+
+      parsed = begin
+        JSON.parse(value)
+      rescue JSON::ParserError
+        value
+      end
+      [key, parsed]
+    end
+
+    def load_parameters(path, raw_parameters)
+      params = {}
+      if path
+        value = JSON.parse(File.read(path))
+        unless value.is_a?(Hash)
+          raise ArgumentError, "Invalid --params file '#{path}': expected a JSON object."
+        end
+        params.merge!(value)
+      end
+      raw_parameters.each do |raw|
+        key, value = parse_parameter(raw)
+        params[key] = value
+      end
+      params.freeze
+    end
+
+    def render_summary(pipeline, result, stderr)
+      same = 0
+      changed = 0
+      errors = 0
+      result[:items].each do |item|
+        if item[:error]
+          errors += 1
+        elsif item[:result] == item[:original_output]
+          same += 1
+        else
+          changed += 1
+        end
+      end
+
+      stderr.puts "\n─── Summary ───"
+      stderr.puts "  Pipeline: #{pipeline}"
+      stderr.puts "  Replayed: #{result[:items].length}"
+      stderr.puts "  Same:     #{same}"
+      stderr.puts "  Changed:  #{changed}"
+      stderr.puts "  Errors:   #{errors}" if errors > 0
+      stderr.puts "\n  #{result[:test_run_url]}"
+    end
+  end
+end

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.38.0"
+  VERSION = "0.38.1"
 end

data/lib/bitfab.rb

@@ -15,6 +15,8 @@ require_relative "bitfab/replay"
 require_relative "bitfab/replay_branch"
 require_relative "bitfab/client"
 require_relative "bitfab/traceable"
+require_relative "bitfab/replay_registry"
+require_relative "bitfab/replay_cli"
 
 module Bitfab
   # No-op span handle returned when outside a span context.

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.38.0
+  version: 0.38.1
 platform: ruby
 authors:
 - Harvest Team
-bindir: bin
+bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
@@ -131,11 +131,13 @@ description: Client library for sending function execution spans to the Bitfab A
   Provides automatic tracing with nested span support.
 email:
 - team@goharvest.ai
-executables: []
+executables:
+- bitfab-replay
 extensions: []
 extra_rdoc_files: []
 files:
 - README.md
+- exe/bitfab-replay
 - lib/bitfab.rb
 - lib/bitfab/client.rb
 - lib/bitfab/compress.rb
@@ -147,6 +149,8 @@ files:
 - lib/bitfab/payload_budget.rb
 - lib/bitfab/replay.rb
 - lib/bitfab/replay_branch.rb
+- lib/bitfab/replay_cli.rb
+- lib/bitfab/replay_registry.rb
 - lib/bitfab/serialize.rb
 - lib/bitfab/span_context.rb
 - lib/bitfab/traceable.rb