bitfab 0.75.5 → 0.76.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/bitfab/assertions.rb +6 -0
- data/lib/bitfab/verdicts.rb +28 -0
- data/lib/bitfab/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 944b6f0cbd7154e1273b43aba22b70e5cf7b55827f7e046fafdac7ee10d06e7c
|
|
4
|
+
data.tar.gz: 94a69276be1db8d70a557e67bb0db363ca1dd46991020938537feeb83d5613ed
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9d12ad3c98b90a58b424c63bb45d1dfb964860758ff47bec3292dbc39be7b9e25a285558f0a6b9b9b1cc3765eef8361ea5274043219cdcc18979a0c773c4522a
|
|
7
|
+
data.tar.gz: b30aa0b927d59ae6387cd238da1ade630ca4191494cb676bb442d9495f94989b60c32a9408711ba978fd85d150a7e2a5b8b014549d01c12c32775070f79f85b1
|
data/lib/bitfab/assertions.rb
CHANGED
|
@@ -54,6 +54,12 @@ module Bitfab
|
|
|
54
54
|
# is off, and new code leaves the setting as it is. Removing the code turns
|
|
55
55
|
# it back on. Omit "codeOff" to keep the current setting.
|
|
56
56
|
#
|
|
57
|
+
# "kind" => "external" creates an assertion Bitfab never judges: your own
|
|
58
|
+
# system posts each verdict with VerdictsClient#save_external, and it takes
|
|
59
|
+
# no "code". Send it only on the save that creates the assertion; a
|
|
60
|
+
# different kind on an edit is refused, so omit it there. Returned
|
|
61
|
+
# assertions carry "kind": "bitfab" (Bitfab decides it) or "external".
|
|
62
|
+
#
|
|
57
63
|
# By default, after the save, Bitfab checks each assertion's text against its
|
|
58
64
|
# trace. When the text bundles two claims, adds a condition, points at a
|
|
59
65
|
# value instead of stating it, names an id from the recorded run, or cannot
|
data/lib/bitfab/verdicts.rb
CHANGED
|
@@ -97,6 +97,7 @@ module Bitfab
|
|
|
97
97
|
class Verdicts
|
|
98
98
|
VERDICTS_PATH = "/api/sdk/traces/verdicts"
|
|
99
99
|
HUMAN_VERDICTS_PATH = "#{VERDICTS_PATH}/human"
|
|
100
|
+
EXTERNAL_VERDICTS_PATH = "#{VERDICTS_PATH}/external"
|
|
100
101
|
# @deprecated Use {VERDICTS_PATH}.
|
|
101
102
|
LABELS_PATH = VERDICTS_PATH
|
|
102
103
|
# @deprecated Use {HUMAN_VERDICTS_PATH}.
|
|
@@ -194,6 +195,29 @@ module Bitfab
|
|
|
194
195
|
end
|
|
195
196
|
end
|
|
196
197
|
|
|
198
|
+
# Saves your own system's verdict on an external assertion: one created with "kind" => "external",
|
|
199
|
+
# which Bitfab never judges. It names only the assertion; Bitfab finds the trace the assertion is on.
|
|
200
|
+
# +verdict:+ is "pass" or "fail". It counts in every pass rate as soon as it is saved, and a person's
|
|
201
|
+
# or an agent's verdict on the same assertion still counts over it. Pass +experiment_id:+ (and
|
|
202
|
+
# +attempt:+ when the experiment ran each trace more than once) to save it on that experiment's
|
|
203
|
+
# replay of the assertion's trace instead.
|
|
204
|
+
def save_external(assertion_id:, verdict:, annotation: nil, confidence: nil, evidence: EVIDENCE_UNSET,
|
|
205
|
+
experiment_id: nil, attempt: nil)
|
|
206
|
+
update = {"assertionId" => assertion_id, "verdict" => verdict}
|
|
207
|
+
update["attempt"] = attempt unless attempt.nil?
|
|
208
|
+
update["annotation"] = annotation unless annotation.nil?
|
|
209
|
+
update["confidence"] = confidence unless confidence.nil?
|
|
210
|
+
update["evidence"] = evidence unless evidence.equal?(EVIDENCE_UNSET)
|
|
211
|
+
save_external_all([update], experiment_id:).first
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# Saves several external assertion verdicts in one all-or-nothing request.
|
|
215
|
+
def save_external_all(verdicts, experiment_id: nil)
|
|
216
|
+
body = {"verdicts" => verdicts}
|
|
217
|
+
body["experimentId"] = experiment_id unless experiment_id.nil?
|
|
218
|
+
@http_client.request(EXTERNAL_VERDICTS_PATH, body)["verdicts"]
|
|
219
|
+
end
|
|
220
|
+
|
|
197
221
|
# Reads one trace's verdicts, or nil when the trace has none. See {#get_all} for the shape of each
|
|
198
222
|
# "evidence" item.
|
|
199
223
|
def get(trace_id)
|
|
@@ -203,6 +227,10 @@ module Bitfab
|
|
|
203
227
|
# Reads the verdicts of several traces. Each trace and each entry of its "assertions" carries
|
|
204
228
|
# "evidence": nil, or an array of hashes, one per span cited, each with "spanId" and "text".
|
|
205
229
|
#
|
|
230
|
+
# An assertion can carry one entry per source. "counted" is true on the one that counts toward the
|
|
231
|
+
# trace's verdict: a person's counts over a coding agent's, and a coding agent's over Bitfab's
|
|
232
|
+
# judge's, which then reads "counted" false.
|
|
233
|
+
#
|
|
206
234
|
# Evidence written by Bitfab's assertion judge also carries "field", "ranges", and "spanChanged",
|
|
207
235
|
# which point at the exact passages the judge read. "field" is the span field the judge read
|
|
208
236
|
# ("summary", "input", "output", "reasoning", "content", "metadata", "errors", "contexts", or
|
data/lib/bitfab/version.rb
CHANGED