bitfab 0.65.6 → 0.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/bitfab/assertions.rb +34 -6
- data/lib/bitfab/client.rb +8 -2
- data/lib/bitfab/graders.rb +73 -9
- data/lib/bitfab/labels.rb +1 -141
- data/lib/bitfab/replay.rb +1 -1
- data/lib/bitfab/verdicts.rb +276 -0
- data/lib/bitfab/version.rb +1 -1
- data/lib/bitfab.rb +1 -1
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 35b3530a206cb86806f06bd4cbeef8d660b07f118f656dae1bc81d9bb5418038
|
|
4
|
+
data.tar.gz: 3da107dd35bde42d9cd9f4c7f549226cf52624da941a24333c6b2ff7738dfd61
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 191854786921184a3324c8fc7aa5bb620c82310e28129e33986cde9aa77ddbdb9fdf0da917b7829a07ba501f2c3992b8d6e0742a7ca695abaa1e3dffcac69826
|
|
7
|
+
data.tar.gz: 9e642a61b8fb8287f6db422cf32d55a44565e9ff2caffa5716d0bf5800248780e620ba0f41ef1f10a35505079099f0f50fca519172185b559ba5c41256a99191
|
data/lib/bitfab/assertions.rb
CHANGED
|
@@ -13,7 +13,11 @@ module Bitfab
|
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
# Each returned assertion carries "category_id" and, with the same value,
|
|
16
|
-
# the deprecated "category_assertion_id".
|
|
16
|
+
# the deprecated "category_assertion_id". It also carries "rewrite", the
|
|
17
|
+
# latest check of its text (the same hash a save returns), or nil when no
|
|
18
|
+
# check ever ran on it. Read a "pending" check from a save back here once
|
|
19
|
+
# it finishes, and save each of its "separateAssertions" as its own
|
|
20
|
+
# assertion.
|
|
17
21
|
def get_assertions(trace_id)
|
|
18
22
|
response = @http_client.get(assertions_path(trace_id))
|
|
19
23
|
return response unless response.is_a?(Hash) && response["assertions"].is_a?(Array)
|
|
@@ -24,17 +28,41 @@ module Bitfab
|
|
|
24
28
|
# Assertion hashes take "category_id": omit it to keep the assignment, or
|
|
25
29
|
# pass nil to clear it. The deprecated "category_assertion_id" is still
|
|
26
30
|
# accepted and sent as "category_id"; "category_id" wins when both are set.
|
|
27
|
-
|
|
28
|
-
|
|
31
|
+
#
|
|
32
|
+
# By default, after the save, Bitfab checks each assertion's text against its
|
|
33
|
+
# trace. When the text bundles two claims, adds a condition, points at a
|
|
34
|
+
# value instead of stating it, names an id from the recorded run, or cannot
|
|
35
|
+
# be measured, Bitfab rewrites it and saves the rewritten text in place of
|
|
36
|
+
# the text you sent. This method returns as soon as the text is saved: each
|
|
37
|
+
# checked assertion comes back with "status" "pending", and Bitfab rewrites
|
|
38
|
+
# it in place in the background, usually within a minute. Read the result
|
|
39
|
+
# later with {#get_assertions}, whose assertions carry the same "rewrite"
|
|
40
|
+
# hash. Pass skip_guided_rewrite: true to save the text exactly as sent,
|
|
41
|
+
# with no check.
|
|
42
|
+
#
|
|
43
|
+
# Each returned assertion carries a "rewrite" hash: "status" ("pending",
|
|
44
|
+
# "rewritten", "unchanged", "not_checked", or "check_failed"),
|
|
45
|
+
# "sentAssertion", "changes", "separateAssertions", "unresolved", and
|
|
46
|
+
# "error". A check not finished 15 minutes after the save reads as
|
|
47
|
+
# "check_failed", with the text as sent. Save each of
|
|
48
|
+
# "separateAssertions" as its own assertion, since the saved one keeps
|
|
49
|
+
# only one claim.
|
|
50
|
+
def save_assertions(trace_id, assertions, source: "agent", skip_guided_rewrite: false)
|
|
51
|
+
save_assertions_all([{"traceId" => trace_id, "assertions" => assertions}], source:, skip_guided_rewrite:)
|
|
29
52
|
end
|
|
30
53
|
|
|
31
|
-
# Same assertion hash keys
|
|
32
|
-
|
|
54
|
+
# Same assertion hash keys, rewrite, and returned "rewrite" hash as
|
|
55
|
+
# {#save_assertions}, for several traces at once.
|
|
56
|
+
def save_assertions_all(updates, source: "agent", skip_guided_rewrite: false)
|
|
33
57
|
return [] if updates.empty?
|
|
34
58
|
|
|
35
59
|
saved = @http_client.request(
|
|
36
60
|
"/api/sdk/traces/assertions",
|
|
37
|
-
{
|
|
61
|
+
{
|
|
62
|
+
"updates" => updates.map { |update| with_new_category_key(update) },
|
|
63
|
+
"source" => source,
|
|
64
|
+
"guidedRewrite" => !skip_guided_rewrite
|
|
65
|
+
}
|
|
38
66
|
)["assertions"]
|
|
39
67
|
saved.is_a?(Array) ? saved.map { |assertion| with_both_category_keys(assertion) } : saved
|
|
40
68
|
end
|
data/lib/bitfab/client.rb
CHANGED
|
@@ -37,7 +37,7 @@ module Bitfab
|
|
|
37
37
|
|
|
38
38
|
attr_reader :service_url
|
|
39
39
|
|
|
40
|
-
attr_reader :datasets, :categories, :organization_members, :traces, :spans, :
|
|
40
|
+
attr_reader :datasets, :categories, :organization_members, :traces, :spans, :verdicts, :graders, :experiments
|
|
41
41
|
|
|
42
42
|
# @deprecated Use {#categories}. Returns the same object.
|
|
43
43
|
def assertion_categories
|
|
@@ -45,6 +45,12 @@ module Bitfab
|
|
|
45
45
|
@categories
|
|
46
46
|
end
|
|
47
47
|
|
|
48
|
+
# @deprecated Use {#verdicts}. Returns the same object.
|
|
49
|
+
def labels
|
|
50
|
+
Bitfab.warn_once("labels-deprecated", "labels is deprecated; use verdicts instead.")
|
|
51
|
+
@verdicts
|
|
52
|
+
end
|
|
53
|
+
|
|
48
54
|
def initialize(api_key: nil, service_url: nil, capture_enabled: nil, enabled: nil, strict: false, simulation_plan: true, timeout: 120, trace_across_threads: nil, env_vars: nil, baml_client: nil, db_snapshot: nil)
|
|
49
55
|
@db_snapshot = DbSnapshot.normalize_config(db_snapshot)
|
|
50
56
|
@env_vars = env_vars || ENV.to_h
|
|
@@ -65,7 +71,7 @@ module Bitfab
|
|
|
65
71
|
@organization_members = OrganizationMembers.new(@http_client)
|
|
66
72
|
@traces = Traces.new(@http_client)
|
|
67
73
|
@spans = Spans.new(@http_client)
|
|
68
|
-
@
|
|
74
|
+
@verdicts = Verdicts.new(@http_client)
|
|
69
75
|
@graders = Graders.new(@http_client)
|
|
70
76
|
@experiments = Experiments.new(@http_client)
|
|
71
77
|
# Mock overrides registered via register_mock_override, applied to every
|
data/lib/bitfab/graders.rb
CHANGED
|
@@ -2,15 +2,69 @@
|
|
|
2
2
|
|
|
3
3
|
require "uri"
|
|
4
4
|
|
|
5
|
+
require_relative "verdicts"
|
|
6
|
+
require_relative "warn_once"
|
|
7
|
+
|
|
5
8
|
module Bitfab
|
|
9
|
+
# Reads and writes grader verdicts: a grader's pass or fail on a whole trace, with its reason.
|
|
6
10
|
class Graders
|
|
7
|
-
|
|
11
|
+
GRADER_VERDICTS_PATH = "/api/sdk/graderVerdicts"
|
|
12
|
+
# @deprecated Use {GRADER_VERDICTS_PATH}.
|
|
13
|
+
GRADER_LABELS_PATH = GRADER_VERDICTS_PATH
|
|
14
|
+
GRADER_VERDICT_VALUES = ["pass", "fail"].freeze
|
|
15
|
+
LABEL_UNSET = Object.new.freeze
|
|
16
|
+
|
|
17
|
+
private_constant :GRADER_VERDICT_VALUES, :LABEL_UNSET
|
|
8
18
|
|
|
9
19
|
def initialize(http_client)
|
|
10
20
|
@http_client = http_client
|
|
11
21
|
end
|
|
12
22
|
|
|
23
|
+
# Reads grader verdicts for +trace_ids:+, for one +grader_id:+, or both. Each verdict carries
|
|
24
|
+
# +verdict+ ("pass", "fail", or nil), +reason+, +failureDiagnostic+, +confidence+, and +source+.
|
|
25
|
+
# Raises when neither selector is given.
|
|
26
|
+
def get_verdicts(trace_ids: nil, grader_id: nil, limit: nil)
|
|
27
|
+
read(trace_ids:, grader_id:, limit:)["verdicts"]
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Saves an external grader's verdict on one trace. +verdict:+ is "pass" or "fail". The deprecated
|
|
31
|
+
# +label:+ (true or false) is still accepted in place of +verdict:+.
|
|
32
|
+
def save_verdict(grader_id:, trace_id:, verdict: nil, reason: nil, failure_diagnostic: nil, confidence: nil,
|
|
33
|
+
label: LABEL_UNSET)
|
|
34
|
+
save_verdict_all([verdict_update(grader_id:, trace_id:, verdict:, reason:, failure_diagnostic:, confidence:,
|
|
35
|
+
label:)]).first
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Saves several external grader verdict entries (+graderId+, +traceId+, +verdict+, and optional
|
|
39
|
+
# +reason+, +failureDiagnostic+, +confidence+) in one request.
|
|
40
|
+
def save_verdict_all(verdicts)
|
|
41
|
+
write(verdicts)["verdicts"]
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# @deprecated Use {#get_verdicts}. Returns the rows in their old shape, with +label+ as true or false.
|
|
13
45
|
def get_labels(trace_ids: nil, grader_id: nil, limit: nil)
|
|
46
|
+
Bitfab.warn_once("grader-get-labels-deprecated", "graders.get_labels is deprecated; use graders.get_verdicts instead.")
|
|
47
|
+
read(trace_ids:, grader_id:, limit:)["labels"]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# @deprecated Use {#save_verdict}.
|
|
51
|
+
def save_label(grader_id:, trace_id:, label:, reason: nil, failure_diagnostic: nil, confidence: nil)
|
|
52
|
+
Bitfab.warn_once("grader-save-label-deprecated", "graders.save_label is deprecated; use graders.save_verdict instead.")
|
|
53
|
+
update = verdict_update(grader_id:, trace_id:, verdict: VerdictInput.legacy_verdict(label), reason:,
|
|
54
|
+
failure_diagnostic:, confidence:, label: LABEL_UNSET)
|
|
55
|
+
write([update])["labels"].first
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# @deprecated Use {#save_verdict_all}.
|
|
59
|
+
def save_label_all(labels)
|
|
60
|
+
Bitfab.warn_once("grader-save-label-all-deprecated",
|
|
61
|
+
"graders.save_label_all is deprecated; use graders.save_verdict_all instead.")
|
|
62
|
+
write(labels.map { |entry| legacy_entry(entry) })["labels"]
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
def read(trace_ids:, grader_id:, limit:)
|
|
14
68
|
if (trace_ids.nil? || trace_ids.empty?) && grader_id.nil?
|
|
15
69
|
raise ArgumentError, "Pass trace_ids, grader_id, or both: there is nothing to read otherwise."
|
|
16
70
|
end
|
|
@@ -19,19 +73,29 @@ module Bitfab
|
|
|
19
73
|
params["traceIds"] = trace_ids.join(",") unless trace_ids.nil? || trace_ids.empty?
|
|
20
74
|
params["graderId"] = grader_id unless grader_id.nil?
|
|
21
75
|
params["limit"] = limit unless limit.nil?
|
|
22
|
-
@http_client.get("#{
|
|
76
|
+
@http_client.get("#{GRADER_VERDICTS_PATH}?#{URI.encode_www_form(params)}")
|
|
23
77
|
end
|
|
24
78
|
|
|
25
|
-
def
|
|
26
|
-
|
|
79
|
+
def write(verdicts)
|
|
80
|
+
verdicts = verdicts.map { |entry| VerdictInput.normalize_entry(entry, GRADER_VERDICT_VALUES) }
|
|
81
|
+
@http_client.request(GRADER_VERDICTS_PATH, {"verdicts" => verdicts})
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def legacy_entry(entry)
|
|
85
|
+
label_key = [:label, "label"].find { |key| entry.key?(key) }
|
|
86
|
+
verdict_present = entry.key?(:verdict) || entry.key?("verdict")
|
|
87
|
+
return entry if label_key.nil? || verdict_present
|
|
88
|
+
|
|
89
|
+
entry.except(label_key).merge("verdict" => VerdictInput.legacy_verdict(entry[label_key]))
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def verdict_update(grader_id:, trace_id:, verdict:, reason:, failure_diagnostic:, confidence:, label:)
|
|
93
|
+
verdict = VerdictInput.resolve_verdict(verdict, label, GRADER_VERDICT_VALUES, LABEL_UNSET)
|
|
94
|
+
update = {"graderId" => grader_id, "traceId" => trace_id, "verdict" => verdict}
|
|
27
95
|
update["reason"] = reason unless reason.nil?
|
|
28
96
|
update["failureDiagnostic"] = failure_diagnostic unless failure_diagnostic.nil?
|
|
29
97
|
update["confidence"] = confidence unless confidence.nil?
|
|
30
|
-
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
def save_label_all(labels)
|
|
34
|
-
@http_client.request(GRADER_LABELS_PATH, {"labels" => labels})["labels"]
|
|
98
|
+
update
|
|
35
99
|
end
|
|
36
100
|
end
|
|
37
101
|
end
|
data/lib/bitfab/labels.rb
CHANGED
|
@@ -1,143 +1,3 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
module Bitfab
|
|
6
|
-
class Labels
|
|
7
|
-
LABELS_PATH = "/api/sdk/traces/labels"
|
|
8
|
-
HUMAN_LABELS_PATH = "#{LABELS_PATH}/human"
|
|
9
|
-
EVIDENCE_UNSET = Object.new.freeze
|
|
10
|
-
|
|
11
|
-
private_constant :EVIDENCE_UNSET
|
|
12
|
-
|
|
13
|
-
def initialize(http_client)
|
|
14
|
-
@http_client = http_client
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def save(label:, annotation:, assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil,
|
|
18
|
-
confidence: nil, experiment_id: nil, evidence: EVIDENCE_UNSET, test_run_id: nil)
|
|
19
|
-
experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
|
|
20
|
-
update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
|
|
21
|
-
update["label"] = label
|
|
22
|
-
update["annotation"] = annotation
|
|
23
|
-
update["confidence"] = confidence unless confidence.nil?
|
|
24
|
-
update["evidence"] = evidence unless evidence.equal?(EVIDENCE_UNSET)
|
|
25
|
-
save_all([update], experiment_id:).first
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
def save_all(labels, experiment_id: nil, test_run_id: nil)
|
|
29
|
-
experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
|
|
30
|
-
require_assertion_ids(labels)
|
|
31
|
-
payload = {"labels" => labels}
|
|
32
|
-
unless experiment_id.nil?
|
|
33
|
-
payload["experimentId"] = experiment_id
|
|
34
|
-
payload["testRunId"] = experiment_id
|
|
35
|
-
end
|
|
36
|
-
@http_client.request(LABELS_PATH, payload)["labels"]
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
def skip(assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil,
|
|
40
|
-
test_run_id: nil, annotation: nil)
|
|
41
|
-
experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
|
|
42
|
-
update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:).merge("skip" => true)
|
|
43
|
-
update["annotation"] = annotation unless annotation.nil?
|
|
44
|
-
save_all([update], experiment_id:).first
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
# Records that the evidence does not show the part of the run this assertion checks, so there was
|
|
48
|
-
# nothing to judge. Counted as a fail in every pass rate, but stored and read back as an abstention,
|
|
49
|
-
# apart from a fail. +annotation:+ says what the evidence would have had to show.
|
|
50
|
-
def abstain(assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil,
|
|
51
|
-
test_run_id: nil, annotation: nil)
|
|
52
|
-
experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
|
|
53
|
-
update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:).merge("abstain" => true)
|
|
54
|
-
update["annotation"] = annotation unless annotation.nil?
|
|
55
|
-
save_all([update], experiment_id:).first
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
def archive(trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil, assertion_id: nil, test_run_id: nil)
|
|
59
|
-
experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
|
|
60
|
-
update = label_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
|
|
61
|
-
save_all([update.merge("archive" => true)], experiment_id:).first
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
def save_human(label:, annotation:, trace_id:, assertion_id:, confidence: nil, evidence: EVIDENCE_UNSET)
|
|
65
|
-
update = {"traceId" => trace_id, "assertionId" => assertion_id, "label" => label, "annotation" => annotation}
|
|
66
|
-
update["confidence"] = confidence unless confidence.nil?
|
|
67
|
-
update["evidence"] = evidence unless evidence.equal?(EVIDENCE_UNSET)
|
|
68
|
-
save_human_all([update]).first
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
def save_human_all(labels)
|
|
72
|
-
require_human_assertion_ids(labels)
|
|
73
|
-
@http_client.request(HUMAN_LABELS_PATH, {"labels" => labels})["labels"]
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
def get(trace_id)
|
|
77
|
-
get_all([trace_id]).first
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
def get_all(trace_ids)
|
|
81
|
-
return [] if trace_ids.empty?
|
|
82
|
-
|
|
83
|
-
query = URI.encode_www_form("traceIds" => trace_ids.join(","))
|
|
84
|
-
@http_client.get("#{LABELS_PATH}?#{query}")["labels"]
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
def generate_label_evidence(trace_id, assertion_id)
|
|
88
|
-
encoded_trace_id = URI.encode_www_form_component(trace_id).gsub("+", "%20")
|
|
89
|
-
encoded_assertion_id = URI.encode_www_form_component(assertion_id).gsub("+", "%20")
|
|
90
|
-
@http_client.get(
|
|
91
|
-
"/api/sdk/traces/#{encoded_trace_id}/assertions/#{encoded_assertion_id}/evidence"
|
|
92
|
-
)["evidence"]
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
private
|
|
96
|
-
|
|
97
|
-
def require_assertion_ids(labels)
|
|
98
|
-
unscoped = labels.reject do |entry|
|
|
99
|
-
entry_value(entry, "archive") == true || !blank?(entry_value(entry, "assertionId"))
|
|
100
|
-
end
|
|
101
|
-
return if unscoped.empty?
|
|
102
|
-
|
|
103
|
-
targets = unscoped.map { |entry| entry_value(entry, "traceId") || entry_value(entry, "originalTraceId") }
|
|
104
|
-
raise ArgumentError, "Agent verdicts must name an assertion: #{targets.join(", ")}. " \
|
|
105
|
-
"Pass the assertion_id of the assertion each verdict scores to labels.save, save_all, skip, and abstain. " \
|
|
106
|
-
"Only archive may omit it, to clear an old whole-trace verdict."
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
def require_human_assertion_ids(labels)
|
|
110
|
-
unscoped = labels.select { |entry| blank?(entry_value(entry, "assertionId")) }
|
|
111
|
-
return if unscoped.empty?
|
|
112
|
-
|
|
113
|
-
targets = unscoped.map { |entry| entry_value(entry, "traceId") }
|
|
114
|
-
raise ArgumentError, "Human verdicts must name an assertion: #{targets.join(", ")}. " \
|
|
115
|
-
"Pass the assertion_id of the assertion each verdict scores to labels.save_human and save_human_all."
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
def entry_value(entry, key)
|
|
119
|
-
entry.key?(key) ? entry[key] : entry[key.to_sym]
|
|
120
|
-
end
|
|
121
|
-
|
|
122
|
-
def blank?(value)
|
|
123
|
-
value.nil? || value.to_s.empty?
|
|
124
|
-
end
|
|
125
|
-
|
|
126
|
-
def label_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
|
|
127
|
-
if trace_id.nil? == original_trace_id.nil?
|
|
128
|
-
raise ArgumentError, "Pass exactly one of trace_id or original_trace_id. " \
|
|
129
|
-
"Use original_trace_id for a replay verdict, with the experiment_id it ran under."
|
|
130
|
-
end
|
|
131
|
-
|
|
132
|
-
update = {}
|
|
133
|
-
update["assertionId"] = assertion_id unless assertion_id.nil?
|
|
134
|
-
if trace_id.nil?
|
|
135
|
-
update["originalTraceId"] = original_trace_id
|
|
136
|
-
update["attempt"] = attempt unless attempt.nil?
|
|
137
|
-
else
|
|
138
|
-
update["traceId"] = trace_id
|
|
139
|
-
end
|
|
140
|
-
update
|
|
141
|
-
end
|
|
142
|
-
end
|
|
143
|
-
end
|
|
3
|
+
require_relative "verdicts"
|
data/lib/bitfab/replay.rb
CHANGED
|
@@ -549,7 +549,7 @@ module Bitfab
|
|
|
549
549
|
warn "Bitfab: server has no persisted trace for #{missing.length} of " \
|
|
550
550
|
"#{completed_count} completed replay item(s) " \
|
|
551
551
|
"(experiment #{experiment_id}). Their replay token usage is " \
|
|
552
|
-
"unavailable and they cannot be
|
|
552
|
+
"unavailable and they cannot be given a verdict."
|
|
553
553
|
end
|
|
554
554
|
end
|
|
555
555
|
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "uri"
|
|
4
|
+
|
|
5
|
+
require_relative "warn_once"
|
|
6
|
+
|
|
7
|
+
module Bitfab
|
|
8
|
+
module VerdictInput
|
|
9
|
+
JUDGED_VERDICTS = ["pass", "fail"].freeze
|
|
10
|
+
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
def resolve_verdict(verdict, label, allowed, unset)
|
|
14
|
+
unless label.equal?(unset)
|
|
15
|
+
raise ArgumentError, "Pass verdict: or the deprecated label:, not both." unless verdict.nil?
|
|
16
|
+
|
|
17
|
+
Bitfab.warn_once("label-keyword-deprecated", "label: is deprecated; pass verdict: \"pass\" or \"fail\" instead.")
|
|
18
|
+
verdict = legacy_verdict(label)
|
|
19
|
+
end
|
|
20
|
+
check_verdict(verdict, allowed)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def require_annotation(verdict, annotation)
|
|
24
|
+
return unless JUDGED_VERDICTS.include?(verdict)
|
|
25
|
+
return unless annotation.nil? || annotation.to_s.empty?
|
|
26
|
+
|
|
27
|
+
raise ArgumentError, "annotation: is required for a #{verdict.inspect} verdict: say why the assertion " \
|
|
28
|
+
"#{(verdict == "pass") ? "passed" : "failed"}. Only an \"abstained\" verdict may omit it."
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def normalize_entry(entry, allowed)
|
|
32
|
+
label_key = [:label, "label"].find { |key| entry.key?(key) }
|
|
33
|
+
abstain_key = [:abstain, "abstain"].find { |key| entry.key?(key) }
|
|
34
|
+
return entry if label_key.nil? && abstain_key.nil?
|
|
35
|
+
|
|
36
|
+
verdict_key = [:verdict, "verdict"].find { |key| entry.key?(key) }
|
|
37
|
+
unless verdict_key.nil?
|
|
38
|
+
raise ArgumentError, "A verdict entry carries verdict and the deprecated #{label_key || abstain_key}, not both."
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
if !label_key.nil? && entry[abstain_key] == true
|
|
42
|
+
raise ArgumentError, "A verdict entry carries both the deprecated label and abstain. " \
|
|
43
|
+
"Send verdict: \"pass\", \"fail\", or \"abstained\" instead."
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
normalized = entry.except(label_key, abstain_key)
|
|
47
|
+
if label_key.nil?
|
|
48
|
+
Bitfab.warn_once("abstain-entry-deprecated", "abstain: true in a verdict entry is deprecated; " \
|
|
49
|
+
"send verdict: \"abstained\" instead.")
|
|
50
|
+
return normalized if entry[abstain_key] != true && action_entry?(entry)
|
|
51
|
+
|
|
52
|
+
if entry[abstain_key] != true
|
|
53
|
+
raise ArgumentError, "A verdict entry with abstain: #{entry[abstain_key].inspect} has no verdict. " \
|
|
54
|
+
"Send verdict: \"pass\", \"fail\", or \"abstained\" instead of the deprecated abstain key."
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
normalized["verdict"] = check_verdict("abstained", allowed)
|
|
58
|
+
else
|
|
59
|
+
Bitfab.warn_once("label-entry-deprecated", "label in a verdict entry is deprecated; " \
|
|
60
|
+
"send verdict: \"pass\" or \"fail\" instead.")
|
|
61
|
+
normalized["verdict"] = check_verdict(legacy_verdict(entry[label_key]), allowed)
|
|
62
|
+
end
|
|
63
|
+
normalized
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def action_entry?(entry)
|
|
67
|
+
["skip", "archive"].any? { |key| entry[key] == true || entry[key.to_sym] == true }
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def legacy_verdict(label)
|
|
71
|
+
return "pass" if label == true
|
|
72
|
+
return "fail" if label == false
|
|
73
|
+
|
|
74
|
+
raise ArgumentError, "The deprecated label must be true or false, got #{label.inspect}."
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def refuse_abstained_extras(entry)
|
|
78
|
+
return unless [entry["verdict"], entry[:verdict]].include?("abstained")
|
|
79
|
+
|
|
80
|
+
extra = ["confidence", "evidence"].find { |key| entry.key?(key) || entry.key?(key.to_sym) }
|
|
81
|
+
return if extra.nil?
|
|
82
|
+
|
|
83
|
+
raise ArgumentError, "An \"abstained\" verdict carries no #{extra}, since nothing was found to judge."
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def check_verdict(verdict, allowed)
|
|
87
|
+
return verdict if allowed.include?(verdict)
|
|
88
|
+
|
|
89
|
+
raise ArgumentError, "verdict must be #{allowed.map(&:inspect).join(" or ")}, got #{verdict.inspect}."
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
private_constant :VerdictInput
|
|
94
|
+
|
|
95
|
+
# Reads and writes assertion verdicts on traces: an agent's or a person's pass, fail, or abstention
|
|
96
|
+
# on one assertion, with the reason behind it.
|
|
97
|
+
class Verdicts
|
|
98
|
+
VERDICTS_PATH = "/api/sdk/traces/verdicts"
|
|
99
|
+
HUMAN_VERDICTS_PATH = "#{VERDICTS_PATH}/human"
|
|
100
|
+
# @deprecated Use {VERDICTS_PATH}.
|
|
101
|
+
LABELS_PATH = VERDICTS_PATH
|
|
102
|
+
# @deprecated Use {HUMAN_VERDICTS_PATH}.
|
|
103
|
+
HUMAN_LABELS_PATH = HUMAN_VERDICTS_PATH
|
|
104
|
+
AGENT_VERDICTS = ["pass", "fail", "abstained"].freeze
|
|
105
|
+
HUMAN_VERDICTS = ["pass", "fail"].freeze
|
|
106
|
+
EVIDENCE_UNSET = Object.new.freeze
|
|
107
|
+
LABEL_UNSET = Object.new.freeze
|
|
108
|
+
|
|
109
|
+
private_constant :EVIDENCE_UNSET, :LABEL_UNSET
|
|
110
|
+
|
|
111
|
+
def initialize(http_client)
|
|
112
|
+
@http_client = http_client
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Saves an agent's verdict on one assertion of a trace. +verdict:+ is "pass", "fail", or "abstained",
|
|
116
|
+
# and +annotation:+ is the reason (required for pass and fail). Target a trace by +trace_id:+, or a
|
|
117
|
+
# replay by +original_trace_id:+ and +attempt:+ with the +experiment_id:+ it ran under. The deprecated
|
|
118
|
+
# +label:+ (true or false) is still accepted in place of +verdict:+.
|
|
119
|
+
def save(assertion_id:, annotation: nil, verdict: nil, trace_id: nil, original_trace_id: nil, attempt: nil,
|
|
120
|
+
confidence: nil, experiment_id: nil, evidence: EVIDENCE_UNSET, test_run_id: nil, label: LABEL_UNSET)
|
|
121
|
+
verdict = VerdictInput.resolve_verdict(verdict, label, AGENT_VERDICTS, LABEL_UNSET)
|
|
122
|
+
VerdictInput.require_annotation(verdict, annotation)
|
|
123
|
+
experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
|
|
124
|
+
update = verdict_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
|
|
125
|
+
update["verdict"] = verdict
|
|
126
|
+
update["annotation"] = annotation unless annotation.nil?
|
|
127
|
+
update["confidence"] = confidence unless confidence.nil?
|
|
128
|
+
update["evidence"] = evidence unless evidence.equal?(EVIDENCE_UNSET)
|
|
129
|
+
save_all([update], experiment_id:).first
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Saves several agent verdict entries in one request. Each entry is a hash with the wire keys
|
|
133
|
+
# (+traceId+ or +originalTraceId+ and +attempt+, +assertionId+, and one of +verdict+, +skip+, or +archive+).
|
|
134
|
+
def save_all(verdicts, experiment_id: nil, test_run_id: nil)
|
|
135
|
+
experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
|
|
136
|
+
verdicts = verdicts.map { |entry| VerdictInput.normalize_entry(entry, AGENT_VERDICTS) }
|
|
137
|
+
verdicts.each { |entry| VerdictInput.refuse_abstained_extras(entry) }
|
|
138
|
+
require_assertion_ids(verdicts)
|
|
139
|
+
payload = {"verdicts" => verdicts}
|
|
140
|
+
unless experiment_id.nil?
|
|
141
|
+
payload["experimentId"] = experiment_id
|
|
142
|
+
payload["testRunId"] = experiment_id
|
|
143
|
+
end
|
|
144
|
+
@http_client.request(VERDICTS_PATH, payload)["verdicts"]
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Withholds a verdict on one assertion without recording a pass or a fail, with an optional reason.
|
|
148
|
+
def skip(assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil,
|
|
149
|
+
test_run_id: nil, annotation: nil)
|
|
150
|
+
experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
|
|
151
|
+
update = verdict_target(trace_id:, original_trace_id:, attempt:, assertion_id:).merge("skip" => true)
|
|
152
|
+
update["annotation"] = annotation unless annotation.nil?
|
|
153
|
+
save_all([update], experiment_id:).first
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# Records that the evidence does not show the part of the run this assertion checks, so there was
|
|
157
|
+
# nothing to judge. Counted as a fail in every pass rate, but stored and read back as an abstention,
|
|
158
|
+
# apart from a fail. +annotation:+ says what the evidence would have had to show. Shorthand for
|
|
159
|
+
# +save(verdict: "abstained")+.
|
|
160
|
+
def abstain(assertion_id:, trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil,
|
|
161
|
+
test_run_id: nil, annotation: nil)
|
|
162
|
+
experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
|
|
163
|
+
update = verdict_target(trace_id:, original_trace_id:, attempt:, assertion_id:).merge("verdict" => "abstained")
|
|
164
|
+
update["annotation"] = annotation unless annotation.nil?
|
|
165
|
+
save_all([update], experiment_id:).first
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# Archives the active verdict on one assertion, or an old whole-trace verdict when +assertion_id:+ is omitted.
|
|
169
|
+
def archive(trace_id: nil, original_trace_id: nil, attempt: nil, experiment_id: nil, assertion_id: nil, test_run_id: nil)
|
|
170
|
+
experiment_id = Bitfab.resolve_experiment_id(experiment_id, test_run_id)
|
|
171
|
+
update = verdict_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
|
|
172
|
+
save_all([update.merge("archive" => true)], experiment_id:).first
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Saves a person's verdict on one assertion of a trace. +verdict:+ is "pass" or "fail", since a person
|
|
176
|
+
# never abstains, and +annotation:+ is the reason, which is required. The deprecated +label:+ (true or
|
|
177
|
+
# false) is still accepted in place of +verdict:+.
|
|
178
|
+
def save_human(trace_id:, assertion_id:, annotation: nil, verdict: nil, confidence: nil, evidence: EVIDENCE_UNSET,
|
|
179
|
+
label: LABEL_UNSET)
|
|
180
|
+
verdict = VerdictInput.resolve_verdict(verdict, label, HUMAN_VERDICTS, LABEL_UNSET)
|
|
181
|
+
VerdictInput.require_annotation(verdict, annotation)
|
|
182
|
+
update = {"traceId" => trace_id, "assertionId" => assertion_id, "verdict" => verdict, "annotation" => annotation}
|
|
183
|
+
update["confidence"] = confidence unless confidence.nil?
|
|
184
|
+
update["evidence"] = evidence unless evidence.equal?(EVIDENCE_UNSET)
|
|
185
|
+
save_human_all([update]).first
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# Saves several of a person's verdict entries in one request.
|
|
189
|
+
def save_human_all(verdicts)
|
|
190
|
+
verdicts = verdicts.map { |entry| VerdictInput.normalize_entry(entry, HUMAN_VERDICTS) }
|
|
191
|
+
require_human_assertion_ids(verdicts)
|
|
192
|
+
@http_client.request(HUMAN_VERDICTS_PATH, {"verdicts" => verdicts})["verdicts"].map do |row|
|
|
193
|
+
row.key?("label") ? row : row.merge("label" => row["verdict"] == "pass")
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# Reads one trace's verdicts, or nil when the trace has none.
|
|
198
|
+
def get(trace_id)
|
|
199
|
+
get_all([trace_id]).first
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# Reads the verdicts of several traces.
|
|
203
|
+
def get_all(trace_ids)
|
|
204
|
+
return [] if trace_ids.empty?
|
|
205
|
+
|
|
206
|
+
query = URI.encode_www_form("traceIds" => trace_ids.join(","))
|
|
207
|
+
@http_client.get("#{VERDICTS_PATH}?#{query}")["verdicts"]
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Generates span-by-span evidence for one assertion on a trace, to send with its verdict.
|
|
211
|
+
def generate_evidence(trace_id, assertion_id)
|
|
212
|
+
encoded_trace_id = URI.encode_www_form_component(trace_id).gsub("+", "%20")
|
|
213
|
+
encoded_assertion_id = URI.encode_www_form_component(assertion_id).gsub("+", "%20")
|
|
214
|
+
@http_client.get(
|
|
215
|
+
"/api/sdk/traces/#{encoded_trace_id}/assertions/#{encoded_assertion_id}/evidence"
|
|
216
|
+
)["evidence"]
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# @deprecated Use {#generate_evidence}.
|
|
220
|
+
def generate_label_evidence(trace_id, assertion_id)
|
|
221
|
+
Bitfab.warn_once("generate-label-evidence-deprecated",
|
|
222
|
+
"generate_label_evidence is deprecated; use verdicts.generate_evidence instead.")
|
|
223
|
+
generate_evidence(trace_id, assertion_id)
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
private
|
|
227
|
+
|
|
228
|
+
def require_assertion_ids(verdicts)
|
|
229
|
+
unscoped = verdicts.reject do |entry|
|
|
230
|
+
entry_value(entry, "archive") == true || !blank?(entry_value(entry, "assertionId"))
|
|
231
|
+
end
|
|
232
|
+
return if unscoped.empty?
|
|
233
|
+
|
|
234
|
+
targets = unscoped.map { |entry| entry_value(entry, "traceId") || entry_value(entry, "originalTraceId") }
|
|
235
|
+
raise ArgumentError, "Agent verdicts must name an assertion: #{targets.join(", ")}. " \
|
|
236
|
+
"Pass the assertion_id of the assertion each verdict scores to verdicts.save, save_all, skip, and abstain. " \
|
|
237
|
+
"Only archive may omit it, to clear an old whole-trace verdict."
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def require_human_assertion_ids(verdicts)
|
|
241
|
+
unscoped = verdicts.select { |entry| blank?(entry_value(entry, "assertionId")) }
|
|
242
|
+
return if unscoped.empty?
|
|
243
|
+
|
|
244
|
+
targets = unscoped.map { |entry| entry_value(entry, "traceId") }
|
|
245
|
+
raise ArgumentError, "Human verdicts must name an assertion: #{targets.join(", ")}. " \
|
|
246
|
+
"Pass the assertion_id of the assertion each verdict scores to verdicts.save_human and save_human_all."
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
def entry_value(entry, key)
|
|
250
|
+
entry.key?(key) ? entry[key] : entry[key.to_sym]
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
def blank?(value)
|
|
254
|
+
value.nil? || value.to_s.empty?
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def verdict_target(trace_id:, original_trace_id:, attempt:, assertion_id:)
|
|
258
|
+
if trace_id.nil? == original_trace_id.nil?
|
|
259
|
+
raise ArgumentError, "Pass exactly one of trace_id or original_trace_id. " \
|
|
260
|
+
"Use original_trace_id for a replay verdict, with the experiment_id it ran under."
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
update = {}
|
|
264
|
+
update["assertionId"] = assertion_id unless assertion_id.nil?
|
|
265
|
+
if trace_id.nil?
|
|
266
|
+
update["originalTraceId"] = original_trace_id
|
|
267
|
+
update["attempt"] = attempt unless attempt.nil?
|
|
268
|
+
else
|
|
269
|
+
update["traceId"] = trace_id
|
|
270
|
+
end
|
|
271
|
+
update
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
Labels = Verdicts
|
|
276
|
+
end
|
data/lib/bitfab/version.rb
CHANGED
data/lib/bitfab.rb
CHANGED
|
@@ -21,7 +21,7 @@ require_relative "bitfab/traces"
|
|
|
21
21
|
require_relative "bitfab/spans"
|
|
22
22
|
require_relative "bitfab/assertions"
|
|
23
23
|
require_relative "bitfab/scenarios"
|
|
24
|
-
require_relative "bitfab/
|
|
24
|
+
require_relative "bitfab/verdicts"
|
|
25
25
|
require_relative "bitfab/graders"
|
|
26
26
|
require_relative "bitfab/experiments"
|
|
27
27
|
require_relative "bitfab/mock_override"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: bitfab
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.66.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Harvest Team
|
|
@@ -205,6 +205,7 @@ files:
|
|
|
205
205
|
- lib/bitfab/traceable.rb
|
|
206
206
|
- lib/bitfab/traces.rb
|
|
207
207
|
- lib/bitfab/transport.rb
|
|
208
|
+
- lib/bitfab/verdicts.rb
|
|
208
209
|
- lib/bitfab/version.rb
|
|
209
210
|
- lib/bitfab/warn_once.rb
|
|
210
211
|
homepage: https://bitfab.ai
|