bitfab 0.64.22 → 0.64.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0289a1a1ca3c00414731bf4e8052c8966cf9b4aef9d2b9e09272d4520df89fa1'
-  data.tar.gz: 3481391fdf05d30f59ac7fcea57fd15c8e63d5dbdbdfa1ba1bcf770dd44b661a
+  metadata.gz: 6be3e712951d897d0cd849ae85155eb7276e170799228d7d84b190d40f376b3c
+  data.tar.gz: fd8221b81baa82065582036be59042b2b0faff64e09e173b6f754ca4b3c592c5
 SHA512:
-  metadata.gz: a6cf514e01eb409810fc2814bceb71ee1f5bbe073b97a0d248bd8c2d63ab3e9ef87cb204aa9fe3bf0eccc331c5b2b03b03f9bfe3cf94d8a38dcdec6e55ac72da
-  data.tar.gz: 29b0530278abc08a7da0800d3a940abaceeac61cc0cf08e769d52f0a12e3efa808cbbb5ae4f2a20c85c142dbfa2debdeafb20f19f0d7bc689a7b63743c4d3082
+  metadata.gz: de63a2a365d14b3bc59d6cf3b4e427203f56d2963d760732444cee78bdc03d45e57f4da9b7ffd26fb30ad71f7f0d5d9d7df35fad9873bf5a7d8e59de12ba1f13
+  data.tar.gz: fd21103ff0706e5e3200c35a8cd1fa65b4df2404882f9d6a35c4676d790e9d512bb2dae4aabb0aec4c2ff8820901b74bd0632d953367fc3c029bdcd26495f8fc

data/lib/bitfab/client.rb

@@ -137,7 +137,12 @@ module Bitfab
     #   each as { path:, before:, after: } (use "" for new/deleted files).
     #   Omit to capture automatically, or pass nil to suppress file capture.
     # @param experiment_group_id [String, nil] optional UUID grouping multiple
-    #   replay runs into a single experiment batch
+    #   replay runs into a single experiment batch. The run joins that group,
+    #   and an id no group has yet creates it
+    # @param experiment_group_name [String, nil] optional display name for the
+    #   experiment group, applied when this replay creates the group or the group
+    #   has no name yet. A group that already has a name keeps it. Requires
+    #   experiment_group_id; passing it alone raises ArgumentError
     # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
     #   replay runs against, for benchmarking one method against several corpora
     #   in a single run; mutually exclusive with dataset_id. Passing trace_ids
@@ -196,7 +201,7 @@ module Bitfab
     # @return [Hash] with :items, :experiment_id, :experiment_url, and the
     #   deprecated aliases :test_run_id and :test_run_url carrying the same values
     def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
-      name: nil, notes: nil, metadata: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
+      name: nil, notes: nil, metadata: nil, code_change_description: Replay::CODE_CHANGE_UNSET, code_change_files: Replay::CODE_CHANGE_UNSET, experiment_group_id: nil, experiment_group_name: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
       Replay.run(
@@ -219,6 +224,7 @@ module Bitfab
         code_change_description:,
         code_change_files:,
         experiment_group_id:,
+        experiment_group_name:,
         dataset_id:,
         dataset_ids:,
         grader_ids:,

data/lib/bitfab/cloudReplay.py

@@ -1896,7 +1896,7 @@ def check_out_snapshot_base(root, commit, branch):
         base = git(root, "rev-parse", "--verify", "-q", f"{commit}^")
     except CommandError:
         print(
-            "This checkout does not include the commit the snapshot was taken on, so the experiment records no code change; set fetch-depth: 2 on the workflow's actions/checkout step, or run bitfab-replay --cloud-update",
+            "This checkout does not include the commit the snapshot was taken on, so the experiment records no code change and the snapshot commit instead of your branch; set fetch-depth: 2 on the workflow's checkout step, or run bitfab-replay --cloud-update",
             flush=True,
         )
         return None
@@ -2900,7 +2900,7 @@ def raise_checkout_depth(lines, start, end):
 def add_checkout_depth(lines):
     for index, line in enumerate(lines):
         match = re.match(
-            r'([ \t]*)(- +)?"?uses"?[ \t]*:[ \t]*"?actions/checkout@',
+            r'([ \t]*)(- +)?"?uses"?[ \t]*:[ \t]*"?[\w.-]+/checkout@',
             without_yaml_comment(line),
         )
         if not match:
@@ -3063,9 +3063,7 @@ def update_workflow(root, path, *, remove_old_key_secrets=False):
             "Checked out the commit each snapshot is taken on, so experiments record the code change"
         )
     elif depth == "by hand":
-        by_hand.append(
-            f"fetch-depth: {CHECKOUT_DEPTH} in the actions/checkout step's with"
-        )
+        by_hand.append(f"fetch-depth: {CHECKOUT_DEPTH} in the checkout step's with")
     if updated:
         target.write_text("\n".join(lines))
     steps = []

data/lib/bitfab/http_client.rb

@@ -254,6 +254,9 @@ module Bitfab
     #   each as { path:, before:, after: } (use "" for new/deleted files)
     # @param experiment_group_id [String, nil] optional UUID grouping multiple
     #   replay runs into a single experiment batch
+    # @param experiment_group_name [String, nil] optional display name for the
+    #   experiment group, applied when this replay creates the group or the group
+    #   has no name yet; only sent together with experiment_group_id
     # @param name [String, nil] optional display name for the resulting
     #   experiment
     # @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
@@ -266,7 +269,7 @@ module Bitfab
     #   runnable graders at completion; each must be an active/live grader in the
     #   same org and trace function or the server rejects the replay
     def start_replay(trace_function_key, limit, trace_ids: nil, code_change_description: nil,
-      code_change_files: nil, experiment_group_id: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
+      code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
       grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil)
       payload = {
         "traceFunctionKey" => trace_function_key
@@ -281,6 +284,7 @@ module Bitfab
       payload["codeChangeDescription"] = code_change_description unless code_change_description.nil?
       payload["codeChangeFiles"] = normalize_code_change_files(code_change_files) unless code_change_files.nil?
       payload["experimentGroupId"] = experiment_group_id unless experiment_group_id.nil?
+      payload["experimentGroupName"] = experiment_group_name unless experiment_group_name.nil?
       payload["includeDbBranchLease"] = true if include_db_branch_lease
       payload["lazyDbBranchLease"] = true if include_db_branch_lease
       unless dataset_ids.nil?

data/lib/bitfab/labels.rb

@@ -44,6 +44,17 @@ module Bitfab
       save_all([update], experiment_id:).first
     end
 
+    # Records that the evidence does not show the part of the run this assertion checks, so there was
+    # nothing to judge. Counted as a fail in every pass rate, but stored and read back as an abstention,
+    # apart from a fail. +annotation:+ says what the evidence would have had to show.
+    def abstain(assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil,
+      test_run_id: nil, annotation: nil)
+      experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
+      update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:).merge("abstain" => true)
+      update["annotation"] = annotation unless annotation.nil?
+      save_all([update], experiment_id:).first
+    end
+
     def archive(trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil, assertion_id: nil, test_run_id: nil)
       experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
       update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
@@ -91,7 +102,7 @@ module Bitfab
 
       targets = unscoped.map { |entry| entry_value(entry, "traceId") || entry_value(entry, "originalTraceId") }
       raise ArgumentError, "Agent verdicts must name an assertion: #{targets.join(", ")}. " \
-        "Pass the assertion_id of the assertion each verdict scores to labels.save, save_all, and skip. " \
+        "Pass the assertion_id of the assertion each verdict scores to labels.save, save_all, skip, and abstain. " \
         "Only archive may omit it, to clear an old whole-trace verdict."
     end
 

data/lib/bitfab/replay.rb

@@ -235,7 +235,12 @@ module Bitfab
     #   each as { path:, before:, after: } (empty string for new/deleted files).
     #   Omit to capture automatically, or pass nil to suppress file capture.
     # @param experiment_group_id [String, nil] optional UUID grouping multiple
-    #   replay runs into a single experiment batch
+    #   replay runs into a single experiment batch. The run joins that group,
+    #   and an id no group has yet creates it
+    # @param experiment_group_name [String, nil] optional display name for the
+    #   experiment group, applied when this replay creates the group or the group
+    #   has no name yet. A group that already has a name keeps it. Requires
+    #   experiment_group_id; passing it alone raises ArgumentError
     # @param dataset_id [String, nil] optional UUID of the dataset this replay
     #   runs against, stored on the resulting experiment for durable attribution;
     #   mutually exclusive with dataset_ids. Alone it replays the dataset's full
@@ -295,7 +300,7 @@ module Bitfab
     #   The experiment fields are nil for a dry run, which creates no experiment,
     #   and progress events then carry a nil :experiment_id too.
     def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
-      concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change_description: CODE_CHANGE_UNSET, code_change_files: CODE_CHANGE_UNSET, experiment_group_id: nil,
+      concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change_description: CODE_CHANGE_UNSET, code_change_files: CODE_CHANGE_UNSET, experiment_group_id: nil, experiment_group_name: nil,
       dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
       adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
       on_experiment_start: nil, experimental_selective_replay: nil)
@@ -319,6 +324,9 @@ module Bitfab
       unless attempts.is_a?(Integer) && attempts.between?(1, 100)
         raise ArgumentError, "attempts must be an integer from 1 to 100."
       end
+      if experiment_group_name && experiment_group_id.nil?
+        raise ArgumentError, "experiment_group_name only applies with experiment_group_id. Pass it together with experiment_group_id."
+      end
       item_finish_callback = on_item_finish || on_progress
       if trace_ids
         raise ArgumentError, "trace_ids must contain at least one trace ID." if trace_ids.empty?
@@ -398,6 +406,7 @@ module Bitfab
         code_change_description:,
         code_change_files:,
         experiment_group_id:,
+        experiment_group_name:,
         include_db_branch_lease:,
         dataset_ids: resolved_dataset_ids,
         grader_ids:,

data/lib/bitfab/replay_registry.rb

@@ -20,7 +20,7 @@ module Bitfab
   class ReplayRegistry
     OPTION_NAMES = %i[
       limit trace_ids name notes metadata concurrency max_concurrency code_change_description
-      code_change_files experiment_group_id dataset_id dataset_ids grader_ids mock
+      code_change_files experiment_group_id experiment_group_name dataset_id dataset_ids grader_ids mock
       mock_override adapt_inputs db_branch attempts only_with_assertions skip_assertion_judging judge_assertions dry_run on_item_finish
     ].freeze
 
@@ -190,7 +190,7 @@ module Bitfab
         options[:limit] = bound || 10
       end
 
-      %i[name notes metadata max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions skip_assertion_judging dry_run].each do |key|
+      %i[name notes metadata max_concurrency experiment_group_id experiment_group_name dataset_ids grader_ids mock attempts only_with_assertions skip_assertion_judging dry_run].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -339,7 +339,11 @@ module Bitfab
           args[:max_concurrency] = positive_integer("--max-concurrency", value)
         end
         options.on("--code-change PATH") { |value| args[:code_change] = value }
-        options.on("--experiment-group-id UUID") { |value| args[:experiment_group_id] = value }
+        options.on("--experiment-group-id UUID",
+          "Add this run to an experiment group. A UUID no group has yet creates the group.") { |value| args[:experiment_group_id] = value }
+        options.on("--experiment-group-name NAME",
+          "Name the experiment group when this run creates it or it has no name yet. " \
+            "A group that already has a name keeps it. Requires --experiment-group-id.") { |value| args[:experiment_group_name] = value }
         options.on("--dataset-ids IDS", "--dataset-id IDS") { |value| args[:dataset_ids] = comma_separated("--dataset-ids", value) }
         options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
         options.on("--attempts N", Integer) { |value| args[:attempts] = positive_integer("--attempts", value) }

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.64.22"
+  VERSION = "0.64.25"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.64.22
+  version: 0.64.25
 platform: ruby
 authors:
 - Harvest Team