bitfab 0.23.0 → 0.23.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/lib/bitfab/replay.rb +16 -5
- data/lib/bitfab/traceable.rb +1 -1
- data/lib/bitfab/version.rb +1 -1
- data/lib/bitfab.rb +3 -4
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8a66c02d783c0a8acbe6e22b82394fcf7805df0bbec4ebf35eff5246a9c086cc
|
|
4
|
+
data.tar.gz: cd9f879e95a0b77ede2176db482ab925bcafb3f28d0a31388af3db0e64e99510
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a8cda8f967f41bdd4d11367a961ea6c2bf567d9fad4b5f963d6894f0274e30f358c324b00eff8a34efee77e07550a1330d5f632095c29ea378d6ce4b95970623
|
|
7
|
+
data.tar.gz: d82f7d930e2b99b3201beadfa01c9919584b96439096d3d2825be48ef5685b25286f9ca97e18ddfd373c039e85134fc49a0e4ed01271334ed0162a91effc1f8a
|
data/README.md
CHANGED
|
@@ -291,7 +291,7 @@ fn.wrap(OpenAI::Client, :embeddings, name: "Embed", type: "llm")
|
|
|
291
291
|
|
|
292
292
|
### Replay with Mock Strategies
|
|
293
293
|
|
|
294
|
-
Replay reruns historical traces through your code so you can compare outputs after an iteration. By default every child span runs real code
|
|
294
|
+
Replay reruns historical traces through your code so you can compare outputs after an iteration. By default every child span runs real code - fine for offline traces, but expensive when children make paid LLM/API calls. Three strategies control whether child spans return their historical output instead of executing:
|
|
295
295
|
|
|
296
296
|
```ruby
|
|
297
297
|
# "none" (default): everything runs real code
|
|
@@ -314,12 +314,12 @@ class Pipeline
|
|
|
314
314
|
# mock_on_replay: true → returns historical output under mock: "marked"
|
|
315
315
|
bitfab_span :call_llm, type: "llm", mock_on_replay: true
|
|
316
316
|
def call_llm(prompt)
|
|
317
|
-
# paid OpenAI call
|
|
317
|
+
# paid OpenAI call - skip during replay
|
|
318
318
|
end
|
|
319
319
|
|
|
320
320
|
bitfab_span :transform, type: "function"
|
|
321
321
|
def transform(text)
|
|
322
|
-
# cheap, deterministic
|
|
322
|
+
# cheap, deterministic - keep running real
|
|
323
323
|
end
|
|
324
324
|
|
|
325
325
|
bitfab_span :process, type: "agent"
|
data/lib/bitfab/replay.rb
CHANGED
|
@@ -266,7 +266,7 @@ module Bitfab
|
|
|
266
266
|
# SOURCE (historical) trace that was replayed, taken from the server
|
|
267
267
|
# item: the result's own :trace_id at this stage is the new replay
|
|
268
268
|
# trace id (assigned in run() after complete_replay), not the source.
|
|
269
|
-
report = lambda do |result, source_trace_id|
|
|
269
|
+
report = lambda do |result, source_trace_id, test_run_id|
|
|
270
270
|
return unless on_progress
|
|
271
271
|
|
|
272
272
|
progress_mutex.synchronize do
|
|
@@ -275,8 +275,19 @@ module Bitfab
|
|
|
275
275
|
error.nil? ? (succeeded += 1) : (errored += 1)
|
|
276
276
|
begin
|
|
277
277
|
on_progress.call({
|
|
278
|
-
completed:, total:, succeeded:, errored:,
|
|
279
|
-
item: {
|
|
278
|
+
test_run_id:, completed:, total:, succeeded:, errored:,
|
|
279
|
+
item: {
|
|
280
|
+
trace_id: source_trace_id,
|
|
281
|
+
replay_trace_id: result[:trace_id],
|
|
282
|
+
input: result[:input],
|
|
283
|
+
result: result[:result],
|
|
284
|
+
original_output: result[:original_output],
|
|
285
|
+
error:,
|
|
286
|
+
duration_ms: result[:duration_ms],
|
|
287
|
+
tokens: result[:tokens],
|
|
288
|
+
model: result[:model],
|
|
289
|
+
db_snapshot_ref: result[:db_snapshot_ref]
|
|
290
|
+
}
|
|
280
291
|
})
|
|
281
292
|
rescue => e
|
|
282
293
|
warn "Bitfab: replay on_progress callback raised: #{e.message}"
|
|
@@ -288,7 +299,7 @@ module Bitfab
|
|
|
288
299
|
server_items.map do |item|
|
|
289
300
|
result = process_single_item(http_client, item, receiver, method_name, test_run_id, mock_strategy,
|
|
290
301
|
adapt_inputs, include_db_branch_lease)
|
|
291
|
-
report.call(result, item["traceId"])
|
|
302
|
+
report.call(result, item["traceId"], test_run_id)
|
|
292
303
|
result
|
|
293
304
|
end
|
|
294
305
|
else
|
|
@@ -306,7 +317,7 @@ module Bitfab
|
|
|
306
317
|
result = process_single_item(http_client, item, receiver, method_name, test_run_id, mock_strategy,
|
|
307
318
|
adapt_inputs, include_db_branch_lease)
|
|
308
319
|
results_mutex.synchronize { results[idx] = result }
|
|
309
|
-
report.call(result, item["traceId"])
|
|
320
|
+
report.call(result, item["traceId"], test_run_id)
|
|
310
321
|
end
|
|
311
322
|
end
|
|
312
323
|
end
|
data/lib/bitfab/traceable.rb
CHANGED
|
@@ -146,7 +146,7 @@ module Bitfab
|
|
|
146
146
|
if method_defined?(method_name) || private_method_defined?(method_name)
|
|
147
147
|
_bitfab_wrap_method(method_name, trace_function_key:, name:, type:, mock_on_replay:)
|
|
148
148
|
else
|
|
149
|
-
# Method doesn't exist yet (before-method style)
|
|
149
|
+
# Method doesn't exist yet (before-method style) - register for method_added hook
|
|
150
150
|
@_bitfab_pending_spans ||= {}
|
|
151
151
|
@_bitfab_pending_spans[method_name] = {
|
|
152
152
|
trace_function_key:,
|
data/lib/bitfab/version.rb
CHANGED
data/lib/bitfab.rb
CHANGED
|
@@ -110,10 +110,9 @@ module Bitfab
|
|
|
110
110
|
# line per trace to stderr, which the Bitfab plugin polls to report live
|
|
111
111
|
# progress while the replay runs in the background, so replay scripts never
|
|
112
112
|
# hand-format the protocol. stdout stays the ReplayResult JSON. Never raises
|
|
113
|
-
# (progress must not crash a run). The progress hash's item
|
|
114
|
-
#
|
|
115
|
-
#
|
|
116
|
-
# settled.
|
|
113
|
+
# (progress must not crash a run). The progress hash's item is forwarded
|
|
114
|
+
# verbatim through to_json, so the plugin sees the source trace id, local
|
|
115
|
+
# replay trace id, outputs, error, and duration of the item that just settled.
|
|
117
116
|
#
|
|
118
117
|
# @example
|
|
119
118
|
# Bitfab.client.replay(..., on_progress: Bitfab.method(:report_replay_progress))
|