bitfab 0.64.16 → 0.64.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 663335d46afd24e88e03d2bb0e8614928b95ebe4f9e24abad0e4f2db68255784
-  data.tar.gz: e0e696b4ed008232aaf21cce80b8f48fa95168500d8bf520874237a5ab54f1fa
+  metadata.gz: 9b213b2d758ad1fbcafc95b9f3704d6fa06c5b288be373dfd9c97565c699eaa4
+  data.tar.gz: 67f58161475956d24a8bac5592e080b42892b277fc0fd132e9a1b679254fddbc
 SHA512:
-  metadata.gz: 52d0d805f4ffe7e66584885c853a9a5a8d26d04af5159ec174ac4c3befa079e33c977bbab0b3804644fd22e415f97bfe48a5358bd3eb51ab791d16861e97e9f9
-  data.tar.gz: 3a345ad2555bff9970cfeae3bf9e965f0cf08c0ec30765526173ef394f4eb4b02c213d50d7b5352d923ca9b5a2d2fe06c46c19d0f494fb1d75ed82242d04b4d6
+  metadata.gz: 54b089f5d3a87132403ec8bff156d0a2bcb596480956c14b9bdb8f158c877589a33a30a8ee1eafaed363ff1ac97b61638997abed639aeb487f16eaba6fd90c81
+  data.tar.gz: c34da0a8b44cb56dad4d788188940fb11197b8377155ff614fb7dd930141dde8911f4bdbdb83480c0a1a4cca73748e2b296459430c3641002a8e1a4a8fe9b6c8

data/lib/bitfab/client.rb

@@ -37,7 +37,7 @@ module Bitfab
 
     attr_reader :service_url
 
-    attr_reader :datasets, :assertion_categories, :organization_members, :traces, :labels, :graders, :experiments
+    attr_reader :datasets, :assertion_categories, :organization_members, :traces, :spans, :labels, :graders, :experiments
 
     def initialize(api_key: nil, service_url: nil, capture_enabled: nil, enabled: nil, strict: false, simulation_plan: true, timeout: 120, trace_across_threads: nil, env_vars: nil, baml_client: nil, db_snapshot: nil)
       @db_snapshot = DbSnapshot.normalize_config(db_snapshot)
@@ -57,6 +57,7 @@ module Bitfab
       @assertion_categories = AssertionCategories.new(@http_client)
       @organization_members = OrganizationMembers.new(@http_client)
       @traces = Traces.new(@http_client)
+      @spans = Spans.new(@http_client)
       @labels = Labels.new(@http_client)
       @graders = Graders.new(@http_client)
       @experiments = Experiments.new(@http_client)
@@ -370,24 +371,28 @@ module Bitfab
     end
 
     # Fetch one persisted span without loading the full trace.
-    # Exactly one of id or name is required. Name lookups return the last
-    # matching span by default; occurrence also accepts "first" or a zero-based
-    # Integer index.
-    def get_trace_span(trace_id, id: nil, name: nil, occurrence: "last")
+    # Pass id alone, or name and/or type (llm, agent, function, guardrail,
+    # handoff, custom). Name and type lookups return the last matching span by
+    # default; occurrence also accepts "first" or a zero-based Integer index.
+    # Raises ArgumentError before sending for any other combination or an
+    # unknown type.
+    def get_trace_span(trace_id, id: nil, name: nil, type: nil, occurrence: "last")
       validate_trace_id(trace_id)
-      raise ArgumentError, "Provide exactly one of id or name" if id.nil? == name.nil?
+      has_position_selector = !name.nil? || !type.nil?
+      raise ArgumentError, "Provide id, or name and/or type" if id.nil? != has_position_selector
 
       validate_span_id(id) unless id.nil?
       if !name.nil? && (!name.is_a?(String) || name.empty?)
         raise ArgumentError, "name must be a non-empty string"
       end
+      validate_span_type!(type) unless type.nil?
 
       valid_occurrence = ["first", "last"].include?(occurrence) || (occurrence.is_a?(Integer) && occurrence >= 0)
       unless valid_occurrence
         raise ArgumentError, 'occurrence must be "first", "last", or a non-negative integer'
       end
 
-      @http_client.get_trace_span(trace_id, id:, name:, occurrence:)
+      @http_client.get_trace_span(trace_id, id:, name:, type:, occurrence:)
     end
 
     def get_span(span_id)

data/lib/bitfab/cloudReplay.py

@@ -137,6 +137,11 @@ RENAMED_FLAGS = {
 SEED_FLAGS = ("--seed", "--cases", "--from-trace", "--run")
 PATH_FLAGS = ("--registry", "--params", "--code-change")
 CODE_CHANGE_FLAGS = ("--code-change", "--no-code-change")
+CHECKOUT_DEPTH = 2
+CODE_CHANGE_PATH_ENV = "BITFAB_CODE_CHANGE_PATH"
+CODE_CHANGE_MAX_FILES = 60
+CODE_CHANGE_MAX_FILE_BYTES = 500_000
+CODE_CHANGE_MAX_TOTAL_BYTES = 2_000_000
 CHECK_ARGUMENTS_SECONDS = 300
 CHECK_ARGUMENTS_FLAG = "--check-arguments"
 REPLAY_ARGUMENTS_FLAG = "--replay-arguments"
@@ -165,6 +170,8 @@ Every replay option works as it does locally, including --dry-run and --fail-on-
 and your SDK checks them before anything is uploaded. The runner replays a snapshot of
 your working tree (changed files, plus new files git does not ignore) from the same
 directory, and this command prints the replay's output and exits with its exit code.
+The experiment records your branch, the commit the snapshot was taken on, and the
+snapshot's changed files as its code change, as it would for a local replay.
 Credential-like files that are not on GitHub yet are refused unless you name them with
 --cloud-allow-file. Requires git and Python 3.10+ on macOS or Linux, with a github.com
 origin.
@@ -887,6 +894,13 @@ def replay_request(root, parsed):
     return request, files
 
 
+def current_branch(root):
+    try:
+        return git(root, "symbolic-ref", "--short", "-q", "HEAD") or None
+    except CommandError:
+        return None
+
+
 def sensitive(path):
     parts = Path(path).parts
     name = Path(path).name.lower()
@@ -1300,6 +1314,9 @@ def allowed_files(root, parsed):
 def submit(root, repo, workflow, parsed, path):
     execution_id = parsed["id"]
     request, files = replay_request(root, parsed)
+    branch = current_branch(root)
+    if branch:
+        request["branch"] = branch
     check_arguments(parsed["replay"])
     allowed = allowed_files(root, parsed)
     if "--cloud-preview" in parsed["switches"]:
@@ -1817,6 +1834,111 @@ def runner_request(root, commit):
     return dispatched_request()
 
 
+def check_out_snapshot_base(root, commit, branch):
+    try:
+        base = git(root, "rev-parse", "--verify", "-q", f"{commit}^")
+    except CommandError:
+        print(
+            "This checkout does not include the commit the snapshot was taken on, so the experiment records no code change; set fetch-depth: 2 on the workflow's actions/checkout step, or run bitfab-replay --cloud-update",
+            flush=True,
+        )
+        return None
+    if isinstance(branch, str) and BRANCH_NAME.fullmatch(branch):
+        try:
+            git(root, "update-ref", f"refs/heads/{branch}", base)
+            git(root, "symbolic-ref", "HEAD", f"refs/heads/{branch}")
+            return base
+        except CommandError:
+            pass
+    git(root, "reset", "-q", "--soft", base)
+    return base
+
+
+def replay_name(args):
+    for index, value in enumerate(args):
+        if value.startswith("--name="):
+            return value.partition("=")[2]
+        if value == "--name" and index + 1 < len(args):
+            return args[index + 1]
+    return None
+
+
+def git_blob(root, ref, path):
+    try:
+        size = int(git(root, "cat-file", "-s", f"{ref}:{path}"))
+    except (CommandError, ValueError):
+        return None
+    if size > CODE_CHANGE_MAX_FILE_BYTES:
+        return None
+    result = subprocess.run(
+        ["git", "-C", str(root), "show", f"{ref}:{path}"],
+        capture_output=True,
+        check=False,
+    )
+    if result.returncode or b"\x00" in result.stdout[:8000]:
+        return None
+    return result.stdout.decode("utf-8", "replace").replace("\r\n", "\n")
+
+
+def changed_paths(root, base, commit):
+    raw = git(
+        root,
+        "diff",
+        "--name-status",
+        "--find-renames",
+        "-z",
+        base,
+        commit,
+        "--",
+        ":!.bitfab",
+    )
+    parts = [part for part in raw.split("\x00") if part]
+    index = 0
+    while index + 1 < len(parts):
+        status, before_path = parts[index][0], parts[index + 1]
+        index += 2
+        path = before_path
+        if status in "RC" and index < len(parts):
+            path = parts[index]
+            index += 1
+        yield status, before_path, path
+
+
+def snapshot_code_change(root, base, commit, name):
+    files, total = [], 0
+    for status, before_path, path in changed_paths(root, base, commit):
+        if len(files) >= CODE_CHANGE_MAX_FILES:
+            break
+        before = "" if status == "A" else git_blob(root, base, before_path)
+        after = "" if status == "D" else git_blob(root, commit, path)
+        if before is None or after is None or before == after:
+            continue
+        size = len(before.encode()) + len(after.encode())
+        if total + size > CODE_CHANGE_MAX_TOTAL_BYTES:
+            continue
+        total += size
+        files.append({"path": path, "before": before, "after": after})
+    if not files:
+        return None
+    head = (name or "").strip() or git(root, "log", "-1", "--format=%s", base)
+    word = "file" if len(files) == 1 else "files"
+    return {
+        "description": f"{head or 'Working-tree change'} ({len(files)} {word} changed uncommitted (vs HEAD))",
+        "files": files,
+    }
+
+
+def write_code_change(change):
+    handle, path = tempfile.mkstemp(
+        prefix="bitfab-code-change-",
+        suffix=".json",
+        dir=os.environ.get("RUNNER_TEMP") or None,
+    )
+    with os.fdopen(handle, "w") as out:
+        json.dump(change, out)
+    return path
+
+
 def execute():
     if os.environ.get("GITHUB_RUN_ATTEMPT") != "1":
         raise ValueError("Submit a new replay instead of rerunning an Actions job")
@@ -1825,6 +1947,11 @@ def execute():
     if git(root, "rev-parse", "HEAD") != commit:
         raise ValueError("Runner checkout does not match the dispatched commit")
     request = runner_request(root, commit)
+    base = (
+        check_out_snapshot_base(root, commit, request.get("branch"))
+        if os.environ.get("GITHUB_REF_NAME", "").startswith(PREFIX)
+        else None
+    )
     directory = within(root, request["cwd"])
     check_secrets(root)
     if os.environ.get(API_KEY_ENV, "").strip():
@@ -1837,10 +1964,19 @@ def execute():
         workflow_environment_conflicts(root, environment)
         print(f"Setting {', '.join(sorted(environment))} from the request", flush=True)
     names = {option_name(value) for value in request["args"]}
+    explicit_change = bool(names & set(CODE_CHANGE_FLAGS))
+    change = (
+        None
+        if explicit_change or base is None
+        else snapshot_code_change(root, base, commit, replay_name(request["args"]))
+    )
+    run_environment = dict(environment)
+    if change:
+        run_environment[CODE_CHANGE_PATH_ENV] = write_code_change(change)
     args = [
         *replay_command(),
         *request["args"],
-        *([] if names & set(CODE_CHANGE_FLAGS) else ["--no-code-change"]),
+        *([] if explicit_change or change else ["--no-code-change"]),
     ]
     if dry_run(request["args"]):
         run_check_command(directory, environment)
@@ -1854,7 +1990,7 @@ def execute():
             request.get("timeoutMinutes"),
             experiment,
             summarize=request.get("fullOutput") is not True,
-            environment=environment,
+            environment=run_environment,
         )
     finally:
         print(OUTPUT_END, flush=True)
@@ -2497,7 +2633,10 @@ def workflow_document(steps, run, directory, env, *, runs_on, environment):
     if environment:
         job["environment"] = environment
     job["steps"] = [
-        {"uses": ACTIONS["checkout"], "with": {"persist-credentials": False}},
+        {
+            "uses": ACTIONS["checkout"],
+            "with": {"persist-credentials": False, "fetch-depth": CHECKOUT_DEPTH},
+        },
         *steps,
         {
             **({} if directory == "." else {"working-directory": directory}),
@@ -2608,6 +2747,57 @@ def add_key_env(lines):
     return True
 
 
+def raise_checkout_depth(lines, start, end):
+    for at in range(start, end):
+        if not re.match(r'[ \t]*"?fetch-depth"?[ \t]*:', lines[at]):
+            continue
+        depth = re.match(
+            r'([ \t]*"?fetch-depth"?[ \t]*:[ \t]*)["\']?(\d+)["\']?(.*)',
+            lines[at],
+        )
+        if depth is None or int(depth[2]) == 0 or int(depth[2]) >= CHECKOUT_DEPTH:
+            return False
+        lines[at] = f"{depth[1]}{CHECKOUT_DEPTH}{depth[3]}"
+        return True
+    return None
+
+
+def add_checkout_depth(lines):
+    for index, line in enumerate(lines):
+        match = re.match(
+            r'([ \t]*)(- +)?"?uses"?[ \t]*:[ \t]*"?actions/checkout@',
+            without_yaml_comment(line),
+        )
+        if not match:
+            continue
+        key = len(match[1]) + len(match[2] or "")
+        start = index
+        while start > 0 and not re.match(r"[ \t]*- ", lines[start]):
+            start -= 1
+        end = block_end(lines, start, key - 1)
+        raised = raise_checkout_depth(lines, start, end)
+        if raised is not None:
+            return "updated" if raised else "unchanged"
+        entry = f"fetch-depth: {CHECKOUT_DEPTH}"
+        for at in range(start, end):
+            with_key = re.fullmatch(
+                r"[ \t]*(- +)?with[ \t]*:(.*)", without_yaml_comment(lines[at])
+            )
+            if not with_key:
+                continue
+            if len(lines[at]) - len(lines[at].lstrip(" \t-")) != key:
+                continue
+            if with_key[2].strip():
+                return "by hand"
+            entries = block_end(lines, at, key)
+            child = indentation(lines[at + 1]) if entries > at + 1 else key + 2
+            lines.insert(entries, " " * child + entry)
+            return "updated"
+        lines[end:end] = [" " * key + "with:", " " * (key + 2) + entry]
+        return "updated"
+    return "unchanged"
+
+
 def update_workflow(root, path):
     target = within(root, path)
     text = target.read_text()
@@ -2633,6 +2823,15 @@ def update_workflow(root, path):
             by_hand.append(
                 f'{API_KEY_ENV}: "{KEY_SECRET_REFERENCE}" in the replay step\'s env'
             )
+    depth = add_checkout_depth(lines)
+    if depth == "updated":
+        updated.append(
+            "Checked out the commit each snapshot is taken on, so experiments record the code change"
+        )
+    elif depth == "by hand":
+        by_hand.append(
+            f"fetch-depth: {CHECKOUT_DEPTH} in the actions/checkout step's with"
+        )
     if updated:
         target.write_text("\n".join(lines))
     steps = []

data/lib/bitfab/http_client.rb

@@ -321,11 +321,11 @@ module Bitfab
       get("/api/sdk/externalSpans/#{span_id}#{query}", timeout: 30)
     end
 
-    def get_trace_span(trace_id, id: nil, name: nil, occurrence: "last")
+    def get_trace_span(trace_id, id: nil, name: nil, type: nil, occurrence: "last")
       query = if id
         {id:}
       else
-        {name:, occurrence: occurrence.to_s}
+        {name:, type: type&.to_s, occurrence: occurrence.to_s}.compact
       end
       encoded_trace_id = URI.encode_www_form_component(trace_id)
       get(

data/lib/bitfab/spans.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Bitfab
+  class Spans
+    def initialize(http_client)
+      @http_client = http_client
+    end
+
+    def list(trace_id, span_type: nil, span_name: nil, fields: nil, cursor: nil, limit: nil)
+      payload = {"traceId" => trace_id}
+      payload["spanType"] = span_type unless span_type.nil?
+      payload["spanName"] = span_name unless span_name.nil?
+      payload["fields"] = fields unless fields.nil?
+      payload["cursor"] = cursor unless cursor.nil?
+      payload["limit"] = limit unless limit.nil?
+      @http_client.request("/api/sdk/spans/list", payload)
+    end
+  end
+end

data/lib/bitfab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bitfab
-  VERSION = "0.64.16"
+  VERSION = "0.64.18"
 end

data/lib/bitfab.rb

@@ -18,6 +18,7 @@ require_relative "bitfab/datasets"
 require_relative "bitfab/assertion_categories"
 require_relative "bitfab/organization_members"
 require_relative "bitfab/traces"
+require_relative "bitfab/spans"
 require_relative "bitfab/assertions"
 require_relative "bitfab/scenarios"
 require_relative "bitfab/labels"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitfab
 version: !ruby/object:Gem::Version
-  version: 0.64.16
+  version: 0.64.18
 platform: ruby
 authors:
 - Harvest Team
@@ -197,6 +197,7 @@ files:
 - lib/bitfab/simulation_plan.rb
 - lib/bitfab/span_context.rb
 - lib/bitfab/span_origin.rb
+- lib/bitfab/spans.rb
 - lib/bitfab/subtree.rb
 - lib/bitfab/thread_propagation.rb
 - lib/bitfab/timestamp.rb