bitfab 0.75.5 → 0.76.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6e0606a717a81feb1acd6c5de56a63fc99c8a97d7cd29024041907085f170678
4
- data.tar.gz: 0a5d83f2efab239b2fa0a2d283e99f73261dd56d182821694b8001aa78c8b251
3
+ metadata.gz: 944b6f0cbd7154e1273b43aba22b70e5cf7b55827f7e046fafdac7ee10d06e7c
4
+ data.tar.gz: 94a69276be1db8d70a557e67bb0db363ca1dd46991020938537feeb83d5613ed
5
5
  SHA512:
6
- metadata.gz: 9dfa544f33218e474a66f4de66f4a5858c387ed4f332346edb7be6de8ed23a101fffb9ff8af91dbfe97cb1d51381fe2b6e724b5542ef8aedd3da7b76354b7e2b
7
- data.tar.gz: 4419df18d8738ef948d24c9d033ff6ec8a236fc26740f045ca8aebfa1b6029456980a54f1e0c3596987d80e7860bb339ef9e0d302daa28b6d7e26638a7680c66
6
+ metadata.gz: 9d12ad3c98b90a58b424c63bb45d1dfb964860758ff47bec3292dbc39be7b9e25a285558f0a6b9b9b1cc3765eef8361ea5274043219cdcc18979a0c773c4522a
7
+ data.tar.gz: b30aa0b927d59ae6387cd238da1ade630ca4191494cb676bb442d9495f94989b60c32a9408711ba978fd85d150a7e2a5b8b014549d01c12c32775070f79f85b1
@@ -54,6 +54,12 @@ module Bitfab
54
54
  # is off, and new code leaves the setting as it is. Removing the code turns
55
55
  # it back on. Omit "codeOff" to keep the current setting.
56
56
  #
57
+ # "kind" => "external" creates an assertion Bitfab never judges: your own
58
+ # system posts each verdict with VerdictsClient#save_external, and it takes
59
+ # no "code". Send it only on the save that creates the assertion; a
60
+ # different kind on an edit is refused, so omit it there. Returned
61
+ # assertions carry "kind": "bitfab" (Bitfab decides it) or "external".
62
+ #
57
63
  # By default, after the save, Bitfab checks each assertion's text against its
58
64
  # trace. When the text bundles two claims, adds a condition, points at a
59
65
  # value instead of stating it, names an id from the recorded run, or cannot
@@ -97,6 +97,7 @@ module Bitfab
97
97
  class Verdicts
98
98
  VERDICTS_PATH = "/api/sdk/traces/verdicts"
99
99
  HUMAN_VERDICTS_PATH = "#{VERDICTS_PATH}/human"
100
+ EXTERNAL_VERDICTS_PATH = "#{VERDICTS_PATH}/external"
100
101
  # @deprecated Use {VERDICTS_PATH}.
101
102
  LABELS_PATH = VERDICTS_PATH
102
103
  # @deprecated Use {HUMAN_VERDICTS_PATH}.
@@ -194,6 +195,29 @@ module Bitfab
194
195
  end
195
196
  end
196
197
 
198
+ # Saves your own system's verdict on an external assertion: one created with "kind" => "external",
199
+ # which Bitfab never judges. It names only the assertion; Bitfab finds the trace the assertion is on.
200
+ # +verdict:+ is "pass" or "fail". It counts in every pass rate as soon as it is saved, and a person's
201
+ # or an agent's verdict on the same assertion still counts over it. Pass +experiment_id:+ (and
202
+ # +attempt:+ when the experiment ran each trace more than once) to save it on that experiment's
203
+ # replay of the assertion's trace instead.
204
+ def save_external(assertion_id:, verdict:, annotation: nil, confidence: nil, evidence: EVIDENCE_UNSET,
205
+ experiment_id: nil, attempt: nil)
206
+ update = {"assertionId" => assertion_id, "verdict" => verdict}
207
+ update["attempt"] = attempt unless attempt.nil?
208
+ update["annotation"] = annotation unless annotation.nil?
209
+ update["confidence"] = confidence unless confidence.nil?
210
+ update["evidence"] = evidence unless evidence.equal?(EVIDENCE_UNSET)
211
+ save_external_all([update], experiment_id:).first
212
+ end
213
+
214
+ # Saves several external assertion verdicts in one all-or-nothing request.
215
+ def save_external_all(verdicts, experiment_id: nil)
216
+ body = {"verdicts" => verdicts}
217
+ body["experimentId"] = experiment_id unless experiment_id.nil?
218
+ @http_client.request(EXTERNAL_VERDICTS_PATH, body)["verdicts"]
219
+ end
220
+
197
221
  # Reads one trace's verdicts, or nil when the trace has none. See {#get_all} for the shape of each
198
222
  # "evidence" item.
199
223
  def get(trace_id)
@@ -203,6 +227,10 @@ module Bitfab
203
227
  # Reads the verdicts of several traces. Each trace and each entry of its "assertions" carries
204
228
  # "evidence": nil, or an array of hashes, one per span cited, each with "spanId" and "text".
205
229
  #
230
+ # An assertion can carry one entry per source. "counted" is true on the one that counts toward the
231
+ # trace's verdict: a person's counts over a coding agent's, and a coding agent's over Bitfab's
232
+ # judge's, which then reads "counted" false.
233
+ #
206
234
  # Evidence written by Bitfab's assertion judge also carries "field", "ranges", and "spanChanged",
207
235
  # which point at the exact passages the judge read. "field" is the span field the judge read
208
236
  # ("summary", "input", "output", "reasoning", "content", "metadata", "errors", "contexts", or
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Bitfab
4
- VERSION = "0.75.5"
4
+ VERSION = "0.76.0"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bitfab
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.75.5
4
+ version: 0.76.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Harvest Team