bitfab 0.73.1 → 0.74.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23076d3ce8682540fa7b9104949899c4292d5ac8aab6cd3b8a8adb77ff379f5e
-  data.tar.gz: 1464557fde7f948ac8aeb34ab4c66d9c947dd30b352d8a2862d74a835ca3d061
+  metadata.gz: 3c5d6ffd1ec89aa3d5e4bd2cbb3a261d5e4007065c3481a4f33898255ab34d37
+  data.tar.gz: 0c662b87145b14ce88ab61aa62e28c8e473de63d42a8335c687df9586353fb44
 SHA512:
-  metadata.gz: 652fd2e9a0285090fff5055b615f04ba48d48b49de4ba27fd4256f59f660d58ea2d4afb6d128eaf9b4f50cbe0a61d0ca022cc90ab59db76d1e0206f5ac11c3c7
-  data.tar.gz: b40ea515566dcae4643ad52b9732244b6a8663a43db9b092cebc7af5360de7df95a77e15a91231ce06b5dc1acb801f3933f69a0cb2072ec0c03d74017c79bc05
+  metadata.gz: 3ff0a584aa0928967b16865aa5e107b895180cde0ea03fa4146f0bff1ebf0b53ba869310ce6c9658bec06583e35c11b897a5e996816f619389b1a8019c8f7909
+  data.tar.gz: d5d1783b9c5af65a670757dd445ea937b339a6faeadc42b564f78e143e392bc359f394bfbc586d613df9f3a20457f345f71e842bc89497fed8c9caba1eb20f08

data/lib/bitfab/client.rb

@@ -182,6 +182,12 @@ module Bitfab
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
+    # @param category_ids [Array<String>, nil] optional category UUIDs. With
+    #   trace_ids, dataset_id or dataset_ids, only the selected traces carrying
+    #   an approved assertion in one of these categories replay. It changes
+    #   which traces replay, not which assertions are judged. Raises
+    #   ArgumentError when empty or passed without a trace ID or dataset
+    #   selection.
     # @param mock [String] mock strategy for child spans: "marked" (default),
     #   "none", or "all". "marked" only mocks spans declared with
     #   mock_on_replay: true; "all" mocks every matched recorded child span. A selected
@@ -228,7 +234,7 @@ module Bitfab
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
-      name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
+      name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
       Replay.run(
@@ -245,6 +251,7 @@ module Bitfab
         max_concurrency:,
         attempts:,
         only_with_assertions:,
+        category_ids:,
         skip_assertion_judging:,
         judge_assertions:,
         dry_run:,

data/lib/bitfab/http_client.rb

@@ -276,9 +276,15 @@ module Bitfab
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
+    # @param category_ids [Array<String>, nil] optional category UUIDs. With
+    #   trace_ids, dataset_id or dataset_ids, only the selected traces carrying
+    #   an approved assertion in one of these categories replay. It changes
+    #   which traces replay, not which assertions are judged. Raises
+    #   ArgumentError when empty or passed without a trace ID or dataset
+    #   selection.
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_type: nil, code_change_description: nil,
       code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
-      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil)
+      grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
       }
@@ -306,6 +312,7 @@ module Bitfab
       payload["graderIds"] = grader_ids unless grader_ids.nil?
       payload["attempts"] = attempts if attempts > 1
       payload["onlyWithAssertions"] = true if only_with_assertions
+      payload["categoryIds"] = category_ids unless category_ids.nil?
       payload["judgeAssertions"] = true unless skip_assertion_judging
       payload["dryRun"] = true if dry_run
       payload["includeOriginalMetadata"] = true if include_original_metadata

data/lib/bitfab/replay.rb

@@ -274,6 +274,12 @@ module Bitfab
     #   directly to this experiment, graded as the union with the dataset's
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
+    # @param category_ids [Array<String>, nil] optional category UUIDs. With
+    #   trace_ids, dataset_id or dataset_ids, only the selected traces carrying
+    #   an approved assertion in one of these categories replay. It changes
+    #   which traces replay, not which assertions are judged. Raises
+    #   ArgumentError when empty or passed without a trace ID or dataset
+    #   selection.
     # @param mock [String] mock strategy for child spans: "marked" (default),
     #   "none", or "all". "marked" only mocks spans declared with
     #   mock_on_replay: true; "all" mocks every matched recorded child span.
@@ -320,7 +326,7 @@ module Bitfab
     #   and progress events then carry a nil :experiment_id too.
     def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
       concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil,
-      dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
+      dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
       validate_code_change(code_change, code_change_description, code_change_files, code_change_base)
@@ -355,6 +361,7 @@ module Bitfab
         end
       end
       resolved_dataset_ids = resolve_dataset_ids(dataset_id, dataset_ids)
+      validate_category_ids(category_ids, trace_ids, resolved_dataset_ids)
       if limit && trace_ids
         warn "Bitfab: limit is ignored when trace_ids is passed: the explicit trace ID list already " \
           "determines how many traces replay."
@@ -422,6 +429,7 @@ module Bitfab
         grader_ids:,
         attempts:,
         only_with_assertions:,
+        category_ids:,
         skip_assertion_judging:,
         include_original_metadata: !adapt_inputs.nil?,
         db_branch_settings: resolved_db_branch_settings,
@@ -613,6 +621,17 @@ module Bitfab
       skip_assertion_judging || judge_assertions == false
     end
 
+    def validate_category_ids(category_ids, trace_ids, dataset_ids)
+      return if category_ids.nil?
+      if category_ids.empty?
+        raise ArgumentError, "category_ids must contain at least one category ID."
+      end
+      return if trace_ids || dataset_ids
+
+      raise ArgumentError,
+        "category_ids narrows a trace ID or dataset selection. Pass it with trace_ids, dataset_id or dataset_ids."
+    end
+
     def resolve_dataset_ids(dataset_id, dataset_ids)
       if dataset_id && dataset_ids
         raise ArgumentError,

data/lib/bitfab/replay_registry.rb

@@ -20,7 +20,7 @@ module Bitfab
   class ReplayRegistry
     OPTION_NAMES = %i[
       limit trace_ids name notes metadata concurrency max_concurrency experiment_group_id experiment_group_name dataset_id dataset_ids grader_ids mock
-      mock_override adapt_inputs db_branch attempts only_with_assertions skip_assertion_judging judge_assertions dry_run on_item_finish
+      mock_override adapt_inputs db_branch attempts only_with_assertions category_ids skip_assertion_judging judge_assertions dry_run on_item_finish
     ].freeze
 
     def initialize
@@ -97,31 +97,39 @@ module Bitfab
     # approved assertion is checked on a replay, so narrowing on anything else
     # would pick traces the run cannot be judged against. The server applies the
     # same rule when it revalidates the selection.
-    def approved_assertion?(assertions)
-      assertions.any? { |assertion| assertion["approvalState"] == "approved" }
+    def approved_assertion?(assertions, category_ids = nil)
+      assertions.any? do |assertion|
+        assertion["approvalState"] == "approved" &&
+          (category_ids.nil? || category_ids.include?(assertion["category_id"]))
+      end
     end
 
-    def bound_trace_ids(client, trace_ids, limit, only_with_assertions)
+    def bound_trace_ids(client, trace_ids, limit, only_with_assertions, category_ids = nil)
       unless limit.is_a?(Integer) && limit.positive?
         raise ArgumentError, "Replay limit must be a positive integer."
       end
-      return trace_ids.first(limit) if !only_with_assertions || limit >= trace_ids.length
+      narrows = only_with_assertions || !category_ids.nil?
+      return trace_ids.first(limit) if !narrows || limit >= trace_ids.length
 
       selected = []
       trace_ids.each do |trace_id|
         result = client.traces.get_assertions(trace_id)
-        if result.fetch("inheritedFrom").nil? && approved_assertion?(result.fetch("assertions"))
+        if result.fetch("inheritedFrom").nil? && approved_assertion?(result.fetch("assertions"), category_ids)
           selected << trace_id
           break if selected.length == limit
         end
       end
       if selected.empty?
+        if category_ids
+          raise ArgumentError,
+            "No traces with an approved assertion in the given categories matched this replay selection."
+        end
         raise ArgumentError, "No traces with approved assertions matched this replay selection."
       end
       selected
     end
 
-    def pinned_dataset_members(client, dataset_ids, limit, only_with_assertions = false)
+    def pinned_dataset_members(client, dataset_ids, limit, only_with_assertions = false, category_ids = nil)
       members = dataset_ids.flat_map { |dataset_id|
         client.datasets.list_traces(dataset_id).fetch("traceIds")
       }.uniq.sort
@@ -134,7 +142,7 @@ module Bitfab
             "every selected dataset in full."
       end
 
-      bound_trace_ids(client, members, limit, only_with_assertions)
+      bound_trace_ids(client, members, limit, only_with_assertions, category_ids)
     end
 
     def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
@@ -173,24 +181,32 @@ module Bitfab
       end
 
       only_with_assertions = args[:only_with_assertions] || options[:only_with_assertions]
+      category_ids = args[:category_ids] || options[:category_ids]
       bound = args[:limit] || options[:limit]
       options.delete(:limit)
       bounded_dataset_ids = args[:dataset_ids] || options[:dataset_ids]
 
+      if category_ids && options[:trace_ids].nil? && bounded_dataset_ids.nil?
+        raise ArgumentError,
+          "--category-ids narrows a trace ID or dataset selection. Pass it with --trace-ids or --dataset-ids."
+      end
+
       if options[:trace_ids]
         if bound
-          options[:trace_ids] = bound_trace_ids(registration.client, options[:trace_ids], bound, only_with_assertions)
+          options[:trace_ids] = bound_trace_ids(registration.client, options[:trace_ids], bound, only_with_assertions,
+            category_ids)
         end
       elsif bounded_dataset_ids
         if bound
-          pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound, only_with_assertions)
+          pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound, only_with_assertions,
+            category_ids)
           options[:trace_ids] = pinned if pinned
         end
       else
         options[:limit] = bound || 10
       end
 
-      %i[name notes metadata max_concurrency experiment_group_id experiment_group_name dataset_ids grader_ids mock attempts only_with_assertions skip_assertion_judging dry_run].each do |key|
+      %i[name notes metadata max_concurrency experiment_group_id experiment_group_name dataset_ids grader_ids mock attempts only_with_assertions category_ids skip_assertion_judging dry_run].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -363,6 +379,11 @@ module Bitfab
         options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
         options.on("--attempts N", Integer) { |value| args[:attempts] = positive_integer("--attempts", value) }
         options.on("--only-with-assertions") { args[:only_with_assertions] = true }
+        options.on("--category-ids id1,id2",
+          "With --trace-ids or --dataset-ids: replay only the traces carrying an approved assertion in one of " \
+            "these categories; which assertions are judged stays the same") do |value|
+          args[:category_ids] = comma_separated("--category-ids", value)
+        end
         options.on("--skip-assertion-judging",
           "Don't judge approved assertions on each replay; judging is on by default and costs model calls") { args[:skip_assertion_judging] = true }
         options.on("--judge-assertions", "Deprecated: judging is on by default") { args[:judge_assertions] = true }

data/lib/bitfab/scenarios.rb

@@ -26,11 +26,17 @@ module Bitfab
     # again. "scenarioId" is nil while the trace has no scenario yet, and "creationStatus" is the scenario's creation
     # status right after the call.
     #
+    # regenerate: true builds a scenario the trace already has again: its name, description, trigger, data, and
+    # actual outcome are cleared and built over, and the organization's scenario callback, when one is set up, is
+    # asked again. Its categories, review status, labels, and assertions are kept, and asking while it is
+    # generating starts nothing more. Leaving it out leaves an existing scenario as it is.
+    #
     # A replay, a trace outside the organization, or a trace Bitfab cannot build a scenario for (not
     # noah-agent-turn and no scenario callback) raises an error. Adding a trace to a dataset starts the same build
     # only when the organization has a scenario callback.
-    def generate_scenario(trace_id)
-      @http_client.request("#{scenario_path(trace_id)}/generate", {})["generation"]
+    def generate_scenario(trace_id, regenerate: nil)
+      payload = regenerate.nil? ? {} : {regenerate:}
+      @http_client.request("#{scenario_path(trace_id)}/generate", payload)["generation"]
     end
 
     # Updates the scenario built from trace_id and returns the saved hash.

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.73.1"
+  VERSION = "0.74.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.73.1
+  version: 0.74.0
 platform: ruby
 authors:
 - Harvest Team