bitfab 0.73.2 → 0.74.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00d7ec6b274939de9c460f01d0e1c2fc662739aa4bd95e7a18f26d36d4697009
-  data.tar.gz: 55dc958c35be2de32dd532a87910d0fca80fdbfd840b0f46a9c2a2c6428d820c
+  metadata.gz: b1def1a5df451e3c7b1afa22248c2c54b2b135f5712eb19a989b41209a8c5622
+  data.tar.gz: 16a9741bbda86c16c78aafec91688d962c9ae58e9b032681a07d7b633497c258
 SHA512:
-  metadata.gz: c4d616bbc59c9bb76eaa405104af8b4b1e1011d048bea5476eb1ac3e6dcfcd6832d69e11632334b3b9b458813877867e1cea85c519e59458466564a5600249df
-  data.tar.gz: 63d3c26eef686ace5ac8578d90bd64f737ae1fcbf3b1d3c3aee134d49afc98bf676d86c0f5bcfa6d6757a790597e4e3e06c8dc75051fde29e9b11caeccde37c2
+  metadata.gz: 2be36b8b49c5b7c6d6525364c6f10a24d696eb40d8dae2343bdf1b1017e2f2fd47e6c52d8cdf200b4f1cf22a75fdd0ea41bf217e533090a9a42fa725ee7018a8
+  data.tar.gz: 5a4fe53c914d211714bbd2a31d44af37743edf1007f4599f52c6dfcce9e3bf970d90778588b1ee17267394017ed554752882f63d45a06fa7ce693786b97dc152

data/lib/bitfab/assertions.rb

@@ -4,9 +4,11 @@ require "uri"
 
 module Bitfab
   class Traces
+    CODE_DRAFT_SAVE_TIMEOUT_SECONDS = 300
+    CODED_KINDS = ["code", "hybrid", :code, :hybrid].freeze
     CATEGORY_ID_KEYS = ["category_id", :category_id].freeze
     DEPRECATED_CATEGORY_ID_KEYS = ["category_assertion_id", :category_assertion_id].freeze
-    private_constant :CATEGORY_ID_KEYS, :DEPRECATED_CATEGORY_ID_KEYS
+    private_constant :CODE_DRAFT_SAVE_TIMEOUT_SECONDS, :CODED_KINDS, :CATEGORY_ID_KEYS, :DEPRECATED_CATEGORY_ID_KEYS
 
     def initialize(http_client)
       @http_client = http_client
@@ -44,6 +46,17 @@ module Bitfab
     # pass "kind" => "judge" to remove the code. Returned assertions carry
     # "kind", "code", and "codeBrokenReason".
     #
+    # Send "kind" => "code" or "hybrid" without "code", on an assertion that
+    # has no code yet, to have Bitfab write the code: the text is checked by
+    # the guided rewrite (unless skipped), code of that kind is written and run
+    # on the trace, and the save waits for both, so that entry's "rewrite" is
+    # already final. Its returned "codeDraft" hash says what happened:
+    # "requestedKind", "status" ("written", or "not_written" when no working
+    # code could be written and the assertion was saved as "judge" without
+    # code), and "reason" (why not, or nil). Every other entry's "codeDraft"
+    # is nil. At most 10 assertions per save may ask for code this way, and a
+    # save that asks for any waits up to five minutes.
+    #
     # By default, after the save, Bitfab checks each assertion's text against its
     # trace. When the text bundles two claims, adds a condition, points at a
     # value instead of stating it, names an id from the recorded run, or cannot
@@ -77,7 +90,8 @@ module Bitfab
           "updates" => updates.map { |update| with_new_category_key(update) },
           "source" => source,
           "guidedRewrite" => !skip_guided_rewrite
-        }
+        },
+        timeout: save_timeout(updates)
       )["assertions"]
       saved.is_a?(Array) ? saved.map { |assertion| with_both_category_keys(assertion) } : saved
     end
@@ -88,6 +102,18 @@ module Bitfab
 
     private
 
+    def save_timeout(updates)
+      asks = updates.any? do |update|
+        (update["assertions"] || update[:assertions] || []).any? { |assertion| asks_bitfab_to_write_code?(assertion) }
+      end
+      asks ? CODE_DRAFT_SAVE_TIMEOUT_SECONDS : nil
+    end
+
+    def asks_bitfab_to_write_code?(assertion)
+      kind = assertion["kind"] || assertion[:kind]
+      CODED_KINDS.include?(kind) && !assertion.key?("code") && !assertion.key?(:code)
+    end
+
     def assertions_path(trace_id, suffix = "")
       "/api/sdk/traces/#{URI.encode_www_form_component(trace_id).gsub("+", "%20")}/assertions#{suffix}"
     end

data/lib/bitfab/client.rb

@@ -182,6 +182,12 @@ module Bitfab
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
+    # @param category_ids [Array<String>, nil] optional category UUIDs. With
+    #   trace_ids, dataset_id or dataset_ids, only the selected traces carrying
+    #   an approved assertion in one of these categories replay. It changes
+    #   which traces replay, not which assertions are judged. Raises
+    #   ArgumentError when empty or passed without a trace ID or dataset
+    #   selection.
     # @param mock [String] mock strategy for child spans: "marked" (default),
     #   "none", or "all". "marked" only mocks spans declared with
     #   mock_on_replay: true; "all" mocks every matched recorded child span. A selected
@@ -228,7 +234,7 @@ module Bitfab
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
-      name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
+      name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
       Replay.run(
@@ -245,6 +251,7 @@ module Bitfab
         max_concurrency:,
         attempts:,
         only_with_assertions:,
+        category_ids:,
         skip_assertion_judging:,
         judge_assertions:,
         dry_run:,

data/lib/bitfab/http_client.rb

@@ -276,9 +276,15 @@ module Bitfab
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
+    # @param category_ids [Array<String>, nil] optional category UUIDs. With
+    #   trace_ids, dataset_id or dataset_ids, only the selected traces carrying
+    #   an approved assertion in one of these categories replay. It changes
+    #   which traces replay, not which assertions are judged. Raises
+    #   ArgumentError when empty or passed without a trace ID or dataset
+    #   selection.
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_type: nil, code_change_description: nil,
       code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
-      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil)
+      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
       }
@@ -306,6 +312,7 @@ module Bitfab
       payload["graderIds"] = grader_ids unless grader_ids.nil?
       payload["attempts"] = attempts if attempts > 1
       payload["onlyWithAssertions"] = true if only_with_assertions
+      payload["categoryIds"] = category_ids unless category_ids.nil?
       payload["judgeAssertions"] = true unless skip_assertion_judging
       payload["dryRun"] = true if dry_run
       payload["includeOriginalMetadata"] = true if include_original_metadata

data/lib/bitfab/replay.rb

@@ -274,6 +274,12 @@ module Bitfab
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
+    # @param category_ids [Array<String>, nil] optional category UUIDs. With
+    #   trace_ids, dataset_id or dataset_ids, only the selected traces carrying
+    #   an approved assertion in one of these categories replay. It changes
+    #   which traces replay, not which assertions are judged. Raises
+    #   ArgumentError when empty or passed without a trace ID or dataset
+    #   selection.
     # @param mock [String] mock strategy for child spans: "marked" (default),
     #   "none", or "all". "marked" only mocks spans declared with
     #   mock_on_replay: true; "all" mocks every matched recorded child span.
@@ -320,7 +326,7 @@ module Bitfab
     #   and progress events then carry a nil :experiment_id too.
     def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
       concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil,
-      dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
+      dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
       validate_code_change(code_change, code_change_description, code_change_files, code_change_base)
@@ -355,6 +361,7 @@ module Bitfab
         end
       end
       resolved_dataset_ids = resolve_dataset_ids(dataset_id, dataset_ids)
+      validate_category_ids(category_ids, trace_ids, resolved_dataset_ids)
       if limit && trace_ids
         warn "Bitfab: limit is ignored when trace_ids is passed: the explicit trace ID list already " \
           "determines how many traces replay."
@@ -422,6 +429,7 @@ module Bitfab
         grader_ids:,
         attempts:,
         only_with_assertions:,
+        category_ids:,
         skip_assertion_judging:,
         include_original_metadata: !adapt_inputs.nil?,
         db_branch_settings: resolved_db_branch_settings,
@@ -613,6 +621,17 @@ module Bitfab
       skip_assertion_judging || judge_assertions == false
     end
 
+    def validate_category_ids(category_ids, trace_ids, dataset_ids)
+      return if category_ids.nil?
+      if category_ids.empty?
+        raise ArgumentError, "category_ids must contain at least one category ID."
+      end
+      return if trace_ids || dataset_ids
+
+      raise ArgumentError,
+        "category_ids narrows a trace ID or dataset selection. Pass it with trace_ids, dataset_id or dataset_ids."
+    end
+
     def resolve_dataset_ids(dataset_id, dataset_ids)
       if dataset_id && dataset_ids
         raise ArgumentError,

data/lib/bitfab/replay_registry.rb

@@ -20,7 +20,7 @@ module Bitfab
   class ReplayRegistry
     OPTION_NAMES = %i[
       limit trace_ids name notes metadata concurrency max_concurrency experiment_group_id experiment_group_name dataset_id dataset_ids grader_ids mock
-      mock_override adapt_inputs db_branch attempts only_with_assertions skip_assertion_judging judge_assertions dry_run on_item_finish
+      mock_override adapt_inputs db_branch attempts only_with_assertions category_ids skip_assertion_judging judge_assertions dry_run on_item_finish
     ].freeze
 
     def initialize
@@ -97,31 +97,39 @@ module Bitfab
     # approved assertion is checked on a replay, so narrowing on anything else
     # would pick traces the run cannot be judged against. The server applies the
     # same rule when it revalidates the selection.
-    def approved_assertion?(assertions)
-      assertions.any? { |assertion| assertion["approvalState"] == "approved" }
+    def approved_assertion?(assertions, category_ids = nil)
+      assertions.any? do |assertion|
+        assertion["approvalState"] == "approved" &&
+          (category_ids.nil? || category_ids.include?(assertion["category_id"]))
+      end
     end
 
-    def bound_trace_ids(client, trace_ids, limit, only_with_assertions)
+    def bound_trace_ids(client, trace_ids, limit, only_with_assertions, category_ids = nil)
       unless limit.is_a?(Integer) && limit.positive?
         raise ArgumentError, "Replay limit must be a positive integer."
       end
-      return trace_ids.first(limit) if !only_with_assertions || limit >= trace_ids.length
+      narrows = only_with_assertions || !category_ids.nil?
+      return trace_ids.first(limit) if !narrows || limit >= trace_ids.length
 
       selected = []
       trace_ids.each do |trace_id|
         result = client.traces.get_assertions(trace_id)
-        if result.fetch("inheritedFrom").nil? && approved_assertion?(result.fetch("assertions"))
+        if result.fetch("inheritedFrom").nil? && approved_assertion?(result.fetch("assertions"), category_ids)
           selected << trace_id
           break if selected.length == limit
         end
       end
       if selected.empty?
+        if category_ids
+          raise ArgumentError,
+            "No traces with an approved assertion in the given categories matched this replay selection."
+        end
         raise ArgumentError, "No traces with approved assertions matched this replay selection."
       end
       selected
     end
 
-    def pinned_dataset_members(client, dataset_ids, limit, only_with_assertions = false)
+    def pinned_dataset_members(client, dataset_ids, limit, only_with_assertions = false, category_ids = nil)
       members = dataset_ids.flat_map { |dataset_id|
         client.datasets.list_traces(dataset_id).fetch("traceIds")
       }.uniq.sort
@@ -134,7 +142,7 @@ module Bitfab
             "every selected dataset in full."
       end
 
-      bound_trace_ids(client, members, limit, only_with_assertions)
+      bound_trace_ids(client, members, limit, only_with_assertions, category_ids)
     end
 
     def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
@@ -173,24 +181,32 @@ module Bitfab
       end
 
       only_with_assertions = args[:only_with_assertions] || options[:only_with_assertions]
+      category_ids = args[:category_ids] || options[:category_ids]
       bound = args[:limit] || options[:limit]
       options.delete(:limit)
       bounded_dataset_ids = args[:dataset_ids] || options[:dataset_ids]
 
+      if category_ids && options[:trace_ids].nil? && bounded_dataset_ids.nil?
+        raise ArgumentError,
+          "--category-ids narrows a trace ID or dataset selection. Pass it with --trace-ids or --dataset-ids."
+      end
+
       if options[:trace_ids]
         if bound
-          options[:trace_ids] = bound_trace_ids(registration.client, options[:trace_ids], bound, only_with_assertions)
+          options[:trace_ids] = bound_trace_ids(registration.client, options[:trace_ids], bound, only_with_assertions,
+            category_ids)
         end
       elsif bounded_dataset_ids
         if bound
-          pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound, only_with_assertions)
+          pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound, only_with_assertions,
+            category_ids)
           options[:trace_ids] = pinned if pinned
         end
       else
         options[:limit] = bound || 10
       end
 
-      %i[name notes metadata max_concurrency experiment_group_id experiment_group_name dataset_ids grader_ids mock attempts only_with_assertions skip_assertion_judging dry_run].each do |key|
+      %i[name notes metadata max_concurrency experiment_group_id experiment_group_name dataset_ids grader_ids mock attempts only_with_assertions category_ids skip_assertion_judging dry_run].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -363,6 +379,11 @@ module Bitfab
         options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
         options.on("--attempts N", Integer) { |value| args[:attempts] = positive_integer("--attempts", value) }
         options.on("--only-with-assertions") { args[:only_with_assertions] = true }
+        options.on("--category-ids id1,id2",
+          "With --trace-ids or --dataset-ids: replay only the traces carrying an approved assertion in one of " \
+            "these categories; which assertions are judged stays the same") do |value|
+          args[:category_ids] = comma_separated("--category-ids", value)
+        end
         options.on("--skip-assertion-judging",
           "Don't judge approved assertions on each replay; judging is on by default and costs model calls") { args[:skip_assertion_judging] = true }
         options.on("--judge-assertions", "Deprecated: judging is on by default") { args[:judge_assertions] = true }

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.73.2"
+  VERSION = "0.74.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.73.2
+  version: 0.74.1
 platform: ruby
 authors:
 - Harvest Team