bitfab 0.64.23 → 0.65.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea8473d1108c410cab9e21e334b155b36be99ddf83207260dfa510815bb8dc64
-  data.tar.gz: f6ac0d40b0a6941ab06428882ac5a66a12ed2b0c4f9f83401f315cd2592dc8c3
+  metadata.gz: aa27d322488c12e530f14a5f85a7f9978208edf332ab91f7bf156f4ccb7b0b25
+  data.tar.gz: f8cadf7e8693aa155c254b7866b41f1781d5de4493dad8b0aad6643d285469a6
 SHA512:
-  metadata.gz: f4acbd07a27cfdb78e158193580c937346d56bcc8bf9d3c5489c71be6ac4af55d25813c01e7d359091261634020bd7a6fc852a003e176c0d107191194acfc063
-  data.tar.gz: f0005a0fc2fa952f93b8488881dbb8e7806eae1b1fa86e81dc5656949d48e035374cb9bacec418ce39650af2b0f658d514745007e1ca398b3d353f9a1df1251a
+  metadata.gz: b274fec88de7c05f84977a60cc146290e89e2e8778fec9a498967304cc2c46a7cb65efa27f65210ec76ba1a2b58dbd3738e4f969d6ac126a9c3a87b25f80c30c
+  data.tar.gz: 8ddf0c559dc1dd33139497e18acf48a065017a59ea05b81e708ee9d3aa4a8eae1b202fb1b5243e7027131b27bcaedd5a2c3065df2fa964793c388609f199e3ad

data/lib/bitfab/assertions.rb

@@ -4,22 +4,39 @@ require "uri"
 
 module Bitfab
   class Traces
+    CATEGORY_ID_KEYS = ["category_id", :category_id].freeze
+    DEPRECATED_CATEGORY_ID_KEYS = ["category_assertion_id", :category_assertion_id].freeze
+    private_constant :CATEGORY_ID_KEYS, :DEPRECATED_CATEGORY_ID_KEYS
+
     def initialize(http_client)
       @http_client = http_client
     end
 
+    # Each returned assertion carries "category_id" and, with the same value,
+    # the deprecated "category_assertion_id".
     def get_assertions(trace_id)
-      @http_client.get(assertions_path(trace_id))
+      response = @http_client.get(assertions_path(trace_id))
+      return response unless response.is_a?(Hash) && response["assertions"].is_a?(Array)
+
+      response.merge("assertions" => response["assertions"].map { |assertion| with_both_category_keys(assertion) })
     end
 
+    # Assertion hashes take "category_id": omit it to keep the assignment, or
+    # pass nil to clear it. The deprecated "category_assertion_id" is still
+    # accepted and sent as "category_id"; "category_id" wins when both are set.
     def save_assertions(trace_id, assertions, source: "agent")
       save_assertions_all([{"traceId" => trace_id, "assertions" => assertions}], source:)
     end
 
+    # Same assertion hash keys as {#save_assertions}, for several traces at once.
     def save_assertions_all(updates, source: "agent")
       return [] if updates.empty?
 
-      @http_client.request("/api/sdk/traces/assertions", {"updates" => updates, "source" => source})["assertions"]
+      saved = @http_client.request(
+        "/api/sdk/traces/assertions",
+        {"updates" => updates.map { |update| with_new_category_key(update) }, "source" => source}
+      )["assertions"]
+      saved.is_a?(Array) ? saved.map { |assertion| with_both_category_keys(assertion) } : saved
     end
 
     def archive_assertions(trace_id, assertion_ids)
@@ -31,5 +48,39 @@ module Bitfab
     def assertions_path(trace_id, suffix = "")
       "/api/sdk/traces/#{URI.encode_www_form_component(trace_id).gsub("+", "%20")}/assertions#{suffix}"
     end
+
+    def with_new_category_key(update)
+      return update unless update.is_a?(Hash)
+
+      key = ["assertions", :assertions].find { |candidate| update[candidate].is_a?(Array) }
+      return update if key.nil?
+
+      update.merge(key => update[key].map { |assertion| rename_category_key(assertion) })
+    end
+
+    def rename_category_key(assertion)
+      return assertion unless assertion.is_a?(Hash)
+
+      old_key = DEPRECATED_CATEGORY_ID_KEYS.find { |candidate| assertion.key?(candidate) }
+      return assertion if old_key.nil?
+
+      Bitfab.warn_once("category-assertion-id-deprecated",
+        "category_assertion_id is deprecated; use category_id instead.")
+      renamed = assertion.except(*DEPRECATED_CATEGORY_ID_KEYS)
+      renamed["category_id"] = assertion[old_key] unless CATEGORY_ID_KEYS.any? { |candidate| assertion.key?(candidate) }
+      renamed
+    end
+
+    def with_both_category_keys(assertion)
+      return assertion unless assertion.is_a?(Hash)
+
+      if assertion.key?("category_id")
+        assertion.key?("category_assertion_id") ? assertion : assertion.merge("category_assertion_id" => assertion["category_id"])
+      elsif assertion.key?("category_assertion_id")
+        assertion.merge("category_id" => assertion["category_assertion_id"])
+      else
+        assertion
+      end
+    end
   end
 end

data/lib/bitfab/{assertion_categories.rb → categories.rb}

@@ -3,8 +3,9 @@
 require "uri"
 
 module Bitfab
-  class AssertionCategories
-    PATH = "/api/sdk/assertionCategories"
+  # The organization's categories, which group assertions and scenarios.
+  class Categories
+    PATH = "/api/sdk/categories"
     private_constant :PATH
 
     def initialize(http_client)
@@ -36,4 +37,7 @@ module Bitfab
       URI.encode_www_form_component(value).gsub("+", "%20")
     end
   end
+
+  # @deprecated Use {Categories}.
+  AssertionCategories = Categories
 end

data/lib/bitfab/client.rb

@@ -37,7 +37,13 @@ module Bitfab
 
     attr_reader :service_url
 
-    attr_reader :datasets, :assertion_categories, :organization_members, :traces, :spans, :labels, :graders, :experiments
+    attr_reader :datasets, :categories, :organization_members, :traces, :spans, :labels, :graders, :experiments
+
+    # @deprecated Use {#categories}. Returns the same object.
+    def assertion_categories
+      Bitfab.warn_once("assertion-categories-deprecated", "assertion_categories is deprecated; use categories instead.")
+      @categories
+    end
 
     def initialize(api_key: nil, service_url: nil, capture_enabled: nil, enabled: nil, strict: false, simulation_plan: true, timeout: 120, trace_across_threads: nil, env_vars: nil, baml_client: nil, db_snapshot: nil)
       @db_snapshot = DbSnapshot.normalize_config(db_snapshot)
@@ -54,7 +60,7 @@ module Bitfab
       @http_client = HttpClient.new(api_key: -> { resolve_api_key }, service_url: @service_url, timeout:)
       @http_client.configure_simulation_plan(enabled: simulation_plan, start: !api_key_resolved_lazily?)
       @datasets = Datasets.new(@http_client)
-      @assertion_categories = AssertionCategories.new(@http_client)
+      @categories = Categories.new(@http_client)
       @organization_members = OrganizationMembers.new(@http_client)
       @traces = Traces.new(@http_client)
       @spans = Spans.new(@http_client)

data/lib/bitfab/cloudReplay.py

@@ -1896,7 +1896,7 @@ def check_out_snapshot_base(root, commit, branch):
         base = git(root, "rev-parse", "--verify", "-q", f"{commit}^")
     except CommandError:
         print(
-            "This checkout does not include the commit the snapshot was taken on, so the experiment records no code change; set fetch-depth: 2 on the workflow's actions/checkout step, or run bitfab-replay --cloud-update",
+            "This checkout does not include the commit the snapshot was taken on, so the experiment records no code change and the snapshot commit instead of your branch; set fetch-depth: 2 on the workflow's checkout step, or run bitfab-replay --cloud-update",
             flush=True,
         )
         return None
@@ -2900,7 +2900,7 @@ def raise_checkout_depth(lines, start, end):
 def add_checkout_depth(lines):
     for index, line in enumerate(lines):
         match = re.match(
-            r'([ \t]*)(- +)?"?uses"?[ \t]*:[ \t]*"?actions/checkout@',
+            r'([ \t]*)(- +)?"?uses"?[ \t]*:[ \t]*"?[\w.-]+/checkout@',
             without_yaml_comment(line),
         )
         if not match:
@@ -3063,9 +3063,7 @@ def update_workflow(root, path, *, remove_old_key_secrets=False):
             "Checked out the commit each snapshot is taken on, so experiments record the code change"
         )
     elif depth == "by hand":
-        by_hand.append(
-            f"fetch-depth: {CHECKOUT_DEPTH} in the actions/checkout step's with"
-        )
+        by_hand.append(f"fetch-depth: {CHECKOUT_DEPTH} in the checkout step's with")
     if updated:
         target.write_text("\n".join(lines))
     steps = []

data/lib/bitfab/labels.rb

@@ -44,6 +44,17 @@ module Bitfab
       save_all([update], experiment_id:).first
     end
 
+    # Records that the evidence does not show the part of the run this assertion checks, so there was
+    # nothing to judge. Counted as a fail in every pass rate, but stored and read back as an abstention,
+    # apart from a fail. +annotation:+ says what the evidence would have had to show.
+    def abstain(assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil,
+      test_run_id: nil, annotation: nil)
+      experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
+      update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:).merge("abstain" => true)
+      update["annotation"] = annotation unless annotation.nil?
+      save_all([update], experiment_id:).first
+    end
+
     def archive(trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil, assertion_id: nil, test_run_id: nil)
       experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
       update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
@@ -91,7 +102,7 @@ module Bitfab
 
       targets = unscoped.map { |entry| entry_value(entry, "traceId") || entry_value(entry, "originalTraceId") }
       raise ArgumentError, "Agent verdicts must name an assertion: #{targets.join(", ")}. " \
-        "Pass the assertion_id of the assertion each verdict scores to labels.save, save_all, and skip. " \
+        "Pass the assertion_id of the assertion each verdict scores to labels.save, save_all, skip, and abstain. " \
         "Only archive may omit it, to clear an old whole-trace verdict."
     end
 

data/lib/bitfab/scenarios.rb

@@ -21,8 +21,13 @@ module Bitfab
     # {"summary" => String, "actions" => [{"name", "details", "result", "failed", "at"}], "source" => Hash or nil}
     # and replaces the saved value whole. Passing nil clears it. Leaving it out keeps it.
     #
+    # category_ids: files the scenario under those categories and replaces its whole set of categories.
+    # Leave it out to keep the current set, and pass an empty array to clear it. The returned hash lists the
+    # scenario's categories under "categories", each {"id", "title", "description"}, sorted by title.
+    #
     # Raises ArgumentError before any request when nothing is passed, or when data: and append_data: are both passed.
-    def save_scenario(trace_id, name: nil, description: nil, trigger: nil, data: nil, append_data: nil, actual_outcome: UNSET)
+    def save_scenario(trace_id, name: nil, description: nil, trigger: nil, data: nil, append_data: nil, actual_outcome: UNSET,
+      category_ids: nil)
       if !data.nil? && !append_data.nil?
         raise ArgumentError, "Pass data or append_data, not both: data replaces the saved data whole."
       end
@@ -34,8 +39,9 @@ module Bitfab
       payload["data"] = data unless data.nil?
       payload["appendData"] = append_data unless append_data.nil?
       payload["actualOutcome"] = actual_outcome unless actual_outcome.equal?(UNSET)
+      payload["categoryIds"] = category_ids.to_a unless category_ids.nil?
       if payload.empty?
-        raise ArgumentError, "Pass at least one of name, description, trigger, data, append_data, or actual_outcome: there is nothing to save otherwise."
+        raise ArgumentError, "Pass at least one of name, description, trigger, data, append_data, actual_outcome, or category_ids: there is nothing to save otherwise."
       end
 
       @http_client.request(scenario_path(trace_id), payload)["scenario"]

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.64.23"
+  VERSION = "0.65.0"
 end

data/lib/bitfab.rb

@@ -15,7 +15,7 @@ require_relative "bitfab/git_state"
 require_relative "bitfab/span_context"
 require_relative "bitfab/http_client"
 require_relative "bitfab/datasets"
-require_relative "bitfab/assertion_categories"
+require_relative "bitfab/categories"
 require_relative "bitfab/organization_members"
 require_relative "bitfab/traces"
 require_relative "bitfab/spans"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.64.23
+  version: 0.65.0
 platform: ruby
 authors:
 - Harvest Team
@@ -155,9 +155,9 @@ files:
 - exe/bitfab-replay
 - exe/bitfab-seed
 - lib/bitfab.rb
-- lib/bitfab/assertion_categories.rb
 - lib/bitfab/assertions.rb
 - lib/bitfab/baml.rb
+- lib/bitfab/categories.rb
 - lib/bitfab/client.rb
 - lib/bitfab/cloudReplay.py
 - lib/bitfab/cloud_replay_cli.rb