bitfab 0.76.9 → 0.77.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +9 -5
- data/lib/bitfab/agent_session.rb +38 -0
- data/lib/bitfab/client.rb +21 -8
- data/lib/bitfab/experiments.rb +2 -1
- data/lib/bitfab/http_client.rb +13 -5
- data/lib/bitfab/replay.rb +34 -8
- data/lib/bitfab/replay_cli.rb +4 -1
- data/lib/bitfab/replay_registry.rb +31 -5
- data/lib/bitfab/version.rb +1 -1
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 70548c4852d1ccf420ff549ab12baf2fc60cc05f58550c5553ee67cecefb6d86
|
|
4
|
+
data.tar.gz: 8842182c4ae2c227b1532c6f398e0d66fd0d0e8b7464e19653ab8d5986a8f4db
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3cab10abd13b21e2d29202714f81be75ec97227928dbca2b378dec538d220f29af0b3c0ae4c18f1ec6cf10cd23723b975c9f82bc5f9492142413747762b5c6c9
|
|
7
|
+
data.tar.gz: 52e818a16d56887ea6cee3cbfad1911780924d27c9607f48fc54a470a04748e2990732bd231995f41483be7f480505d0c42bf410679a1556825bc74bdb6848f1
|
data/README.md
CHANGED
|
@@ -336,18 +336,22 @@ fn.wrap(OpenAI::Client, :embeddings, name: "Embed", type: "llm")
|
|
|
336
336
|
|
|
337
337
|
### Replay with Mock Strategies
|
|
338
338
|
|
|
339
|
-
Replay reruns historical traces through your code so you can compare outputs after an iteration. By default, spans tagged with `mock_on_replay: true` return their historical output and other child spans run real code. Three strategies control whether child spans return their historical output instead of executing
|
|
339
|
+
Replay reruns historical traces through your code so you can compare outputs after an iteration. By default, spans tagged with `mock_on_replay: true` return their historical output and other child spans run real code. Three strategies control whether child spans return their historical output instead of executing.
|
|
340
|
+
|
|
341
|
+
Every replay that is not a dry run needs an experiment group: one series of experiments testing one thing, named after what is being tested. Pass the id of an existing group when the run tests the same thing as that group, or a new UUID with `experiment_group_name:` to start one.
|
|
340
342
|
|
|
341
343
|
```ruby
|
|
344
|
+
group = {experiment_group_id: SecureRandom.uuid, experiment_group_name: "Shorter summary prompt"}
|
|
345
|
+
|
|
342
346
|
# "marked" (default): only spans tagged with `mock_on_replay: true` return historical output
|
|
343
|
-
client.replay(pipeline, :process, trace_function_key: "my-fn")
|
|
344
|
-
client.replay(pipeline, :process, trace_function_key: "my-fn", mock: "marked")
|
|
347
|
+
client.replay(pipeline, :process, trace_function_key: "my-fn", **group)
|
|
348
|
+
client.replay(pipeline, :process, trace_function_key: "my-fn", mock: "marked", **group)
|
|
345
349
|
|
|
346
350
|
# "none": everything runs real code
|
|
347
|
-
client.replay(pipeline, :process, trace_function_key: "my-fn", mock: "none")
|
|
351
|
+
client.replay(pipeline, :process, trace_function_key: "my-fn", mock: "none", **group)
|
|
348
352
|
|
|
349
353
|
# "all": every matched recorded child returns history; missing occurrences fail closed
|
|
350
|
-
client.replay(pipeline, :process, trace_function_key: "my-fn", mock: "all")
|
|
354
|
+
client.replay(pipeline, :process, trace_function_key: "my-fn", mock: "all", **group)
|
|
351
355
|
```
|
|
352
356
|
|
|
353
357
|
Tag the spans you want mocked at definition time:
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bitfab
|
|
4
|
+
# The id of the coding agent session that launched a replay, recorded on the
|
|
5
|
+
# experiment so a run can be traced back to that session.
|
|
6
|
+
module AgentSession
|
|
7
|
+
# Environment variables read, in order, when no session id is passed.
|
|
8
|
+
ENV_VARS = %w[CLAUDE_CODE_SESSION_ID CODEX_THREAD_ID AGENT_SESSION_ID].freeze
|
|
9
|
+
# The longest session id the server accepts.
|
|
10
|
+
MAX_LENGTH = 255
|
|
11
|
+
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
# Returns the trimmed explicit id when one is given, otherwise the first
|
|
15
|
+
# non-empty value of ENV_VARS, otherwise nil. Raises ArgumentError when an
|
|
16
|
+
# explicit id is longer than MAX_LENGTH.
|
|
17
|
+
def resolve(explicit = nil, env: ENV)
|
|
18
|
+
given = text(explicit)
|
|
19
|
+
if given
|
|
20
|
+
if given.length > MAX_LENGTH
|
|
21
|
+
raise ArgumentError, "agent_session_id must be at most #{MAX_LENGTH} characters (got #{given.length})."
|
|
22
|
+
end
|
|
23
|
+
return given
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
ENV_VARS.each do |name|
|
|
27
|
+
value = text(env[name])
|
|
28
|
+
return value if value && value.length <= MAX_LENGTH
|
|
29
|
+
end
|
|
30
|
+
nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def text(value)
|
|
34
|
+
trimmed = value&.to_s&.strip
|
|
35
|
+
(trimmed.nil? || trimmed.empty?) ? nil : trimmed
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
data/lib/bitfab/client.rb
CHANGED
|
@@ -132,7 +132,7 @@ module Bitfab
|
|
|
132
132
|
# Passed alongside dataset_id or dataset_ids it pins which members of that
|
|
133
133
|
# selection replay, and the server rejects any ID none of those datasets
|
|
134
134
|
# contains
|
|
135
|
-
# @param name [String, nil] What this run
|
|
135
|
+
# @param name [String, nil] What this run changes, in a few words, such
|
|
136
136
|
# as 'baseline' or 'shorter system prompt'. Bitfab records the commit,
|
|
137
137
|
# branch, tree state, datasets, and who ran it with every experiment, so
|
|
138
138
|
# do not repeat them here.
|
|
@@ -163,18 +163,29 @@ module Bitfab
|
|
|
163
163
|
# against when code_change is "ref", and only then, such as "dev",
|
|
164
164
|
# "release/2.3", or a commit SHA. A branch that exists only on the remote
|
|
165
165
|
# is read as origin/<name>. A name git cannot resolve raises ArgumentError.
|
|
166
|
-
# @param experiment_group_id [String, nil]
|
|
167
|
-
#
|
|
168
|
-
#
|
|
166
|
+
# @param experiment_group_id [String, nil] UUID of the experiment group
|
|
167
|
+
# this run belongs to: one series of experiments testing one thing.
|
|
168
|
+
# Required except for a dry run or skip_experiment_group; leaving it out raises ArgumentError
|
|
169
|
+
# before any request. The run joins that group, and an id no group has
|
|
170
|
+
# yet creates it
|
|
169
171
|
# @param experiment_group_name [String, nil] optional display name for the
|
|
170
|
-
# experiment group,
|
|
171
|
-
# has no name yet. A group that
|
|
172
|
-
# experiment_group_id; passing it
|
|
172
|
+
# experiment group, named after what the run tests, applied when this
|
|
173
|
+
# replay creates the group or the group has no name yet. A group that
|
|
174
|
+
# already has a name keeps it. Requires experiment_group_id; passing it
|
|
175
|
+
# alone raises ArgumentError
|
|
173
176
|
# @param baseline_experiment_id [String, nil] optional UUID of an earlier
|
|
174
177
|
# experiment this run is compared against. It takes precedence over the
|
|
175
178
|
# first finished experiment in the group and over the original traces.
|
|
176
179
|
# It must belong to the caller's organization or the server rejects the
|
|
177
180
|
# replay
|
|
181
|
+
# @param skip_experiment_group [Boolean] runs without an experiment group,
|
|
182
|
+
# for scripts and schedules whose runs should not be grouped. Cannot be
|
|
183
|
+
# combined with experiment_group_id or experiment_group_name; doing so
|
|
184
|
+
# raises ArgumentError before any request
|
|
185
|
+
# @param agent_session_id [String, nil] the coding agent session that
|
|
186
|
+
# launched this run, stored on the experiment. Defaults to the first
|
|
187
|
+
# non-empty of CLAUDE_CODE_SESSION_ID, CODEX_THREAD_ID, and
|
|
188
|
+
# AGENT_SESSION_ID
|
|
178
189
|
# @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
|
|
179
190
|
# replay runs against, for benchmarking one method against several corpora
|
|
180
191
|
# in a single run; mutually exclusive with dataset_id. Passing trace_ids
|
|
@@ -239,7 +250,7 @@ module Bitfab
|
|
|
239
250
|
# @return [Hash] with :items, :experiment_id, :experiment_url, and the
|
|
240
251
|
# deprecated aliases :test_run_id and :test_run_url carrying the same values
|
|
241
252
|
def replay(receiver, method_name = nil, trace_function_key:, limit: nil, trace_ids: nil, concurrency: nil, max_concurrency: ReplayConcurrency::UNSET,
|
|
242
|
-
name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
|
|
253
|
+
name: nil, notes: nil, metadata: nil, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, skip_experiment_group: false, agent_session_id: nil, dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
|
|
243
254
|
adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
|
|
244
255
|
on_experiment_start: nil, experimental_selective_replay: nil)
|
|
245
256
|
Replay.run(
|
|
@@ -267,6 +278,8 @@ module Bitfab
|
|
|
267
278
|
experiment_group_id:,
|
|
268
279
|
experiment_group_name:,
|
|
269
280
|
baseline_experiment_id:,
|
|
281
|
+
skip_experiment_group:,
|
|
282
|
+
agent_session_id:,
|
|
270
283
|
dataset_id:,
|
|
271
284
|
dataset_ids:,
|
|
272
285
|
grader_ids:,
|
data/lib/bitfab/experiments.rb
CHANGED
|
@@ -14,13 +14,14 @@ module Bitfab
|
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
def list(dataset_id: nil, trace_function_key: nil, git_branch: nil, experiment_group_id: nil, status: nil,
|
|
17
|
-
metadata: nil, created_after: nil, created_before: nil, cursor: nil, limit: nil)
|
|
17
|
+
metadata: nil, agent_session_id: nil, created_after: nil, created_before: nil, cursor: nil, limit: nil)
|
|
18
18
|
params = {}
|
|
19
19
|
params["datasetId"] = dataset_id unless dataset_id.nil?
|
|
20
20
|
params["traceFunctionKey"] = trace_function_key unless trace_function_key.nil?
|
|
21
21
|
params["gitBranch"] = git_branch unless git_branch.nil?
|
|
22
22
|
params["experimentGroupId"] = experiment_group_id unless experiment_group_id.nil?
|
|
23
23
|
params["status"] = status unless status.nil?
|
|
24
|
+
params["agentSessionId"] = agent_session_id unless agent_session_id.nil?
|
|
24
25
|
params["metadata"] = JSON.generate(metadata) unless metadata.nil?
|
|
25
26
|
params["createdAfter"] = iso8601(created_after) unless created_after.nil?
|
|
26
27
|
params["createdBefore"] = iso8601(created_before) unless created_before.nil?
|
data/lib/bitfab/http_client.rb
CHANGED
|
@@ -5,6 +5,7 @@ require "json"
|
|
|
5
5
|
require "uri"
|
|
6
6
|
|
|
7
7
|
require_relative "git_state"
|
|
8
|
+
require_relative "agent_session"
|
|
8
9
|
require_relative "replay_invocation"
|
|
9
10
|
require_relative "compress"
|
|
10
11
|
require_relative "connection_pool"
|
|
@@ -311,6 +312,12 @@ module Bitfab
|
|
|
311
312
|
# has no name yet; only sent together with experiment_group_id
|
|
312
313
|
# @param baseline_experiment_id [String, nil] optional UUID of an earlier
|
|
313
314
|
# experiment this run is compared against, sent as baselineExperimentId
|
|
315
|
+
# @param require_experiment_group [Boolean] asks the server to refuse a
|
|
316
|
+
# replay with no experiment_group_id; false leaves the field out
|
|
317
|
+
# @param agent_session_id [String, nil] the coding agent session that
|
|
318
|
+
# launched this replay. Defaults to the first non-empty of
|
|
319
|
+
# CLAUDE_CODE_SESSION_ID, CODEX_THREAD_ID, and AGENT_SESSION_ID, and is
|
|
320
|
+
# left out when none is set
|
|
314
321
|
# @param name [String, nil] optional display name for the resulting
|
|
315
322
|
# experiment
|
|
316
323
|
# @param dataset_ids [Array<String>, nil] optional UUIDs of the datasets this
|
|
@@ -329,11 +336,10 @@ module Bitfab
|
|
|
329
336
|
# ArgumentError when empty or passed without a trace ID or dataset
|
|
330
337
|
# selection.
|
|
331
338
|
def start_replay(trace_function_key, limit, trace_ids: nil, code_change_type: nil, code_change_description: nil,
|
|
332
|
-
code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
|
|
333
|
-
grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil)
|
|
334
|
-
payload = {
|
|
335
|
-
|
|
336
|
-
}
|
|
339
|
+
code_change_files: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, require_experiment_group: true, name: nil, notes: nil, metadata: nil, include_db_branch_lease: false, dataset_ids: nil,
|
|
340
|
+
grader_ids: nil, db_branch_settings: nil, attempts: 1, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, include_original_metadata: false, experimental_selective_replay: nil, dry_run: false, invocation: nil, agent_session_id: nil)
|
|
341
|
+
payload = {"traceFunctionKey" => trace_function_key}
|
|
342
|
+
payload["requireExperimentGroup"] = true if require_experiment_group
|
|
337
343
|
# limit is only meaningful without trace_ids (an explicit ID list
|
|
338
344
|
# already determines the count), so it's omitted when nil.
|
|
339
345
|
payload["limit"] = limit unless limit.nil?
|
|
@@ -347,6 +353,8 @@ module Bitfab
|
|
|
347
353
|
payload["experimentGroupId"] = experiment_group_id unless experiment_group_id.nil?
|
|
348
354
|
payload["experimentGroupName"] = experiment_group_name unless experiment_group_name.nil?
|
|
349
355
|
payload["baselineExperimentId"] = baseline_experiment_id unless baseline_experiment_id.nil?
|
|
356
|
+
resolved_agent_session_id = AgentSession.resolve(agent_session_id)
|
|
357
|
+
payload["agentSessionId"] = resolved_agent_session_id if resolved_agent_session_id
|
|
350
358
|
payload["includeDbBranchLease"] = true if include_db_branch_lease
|
|
351
359
|
payload["lazyDbBranchLease"] = true if include_db_branch_lease
|
|
352
360
|
unless dataset_ids.nil?
|
data/lib/bitfab/replay.rb
CHANGED
|
@@ -148,6 +148,13 @@ module Bitfab
|
|
|
148
148
|
CODE_CHANGE_CHOICES = '"working-tree" (uncommitted changes vs HEAD), "last-branch" (the diff against the ' \
|
|
149
149
|
'branch this one was cut from), "default-branch" (the diff against the default branch), "ref" (the diff ' \
|
|
150
150
|
'against a branch or commit you name), "skip" (no code change), or "manual" (pass the files yourself)'
|
|
151
|
+
# The error a replay raises when it names no experiment group.
|
|
152
|
+
EXPERIMENT_GROUP_REQUIRED = "experiment_group_id is required. An experiment group is one series of experiments " \
|
|
153
|
+
"testing one thing. Reuse the id of the group whose runs test the same thing, or pass a new UUID with " \
|
|
154
|
+
"experiment_group_name, named after what this run tests, to start a group. To run without a group, " \
|
|
155
|
+
"pass skip_experiment_group: true."
|
|
156
|
+
SKIP_EXPERIMENT_GROUP_CONFLICT = "skip_experiment_group runs without an experiment group and cannot be combined " \
|
|
157
|
+
"with experiment_group_id or experiment_group_name. Pass one or the other."
|
|
151
158
|
UNSET_DEFAULT = Object.new.freeze
|
|
152
159
|
OPTION_DEFAULTS = {mock: "marked"}.freeze
|
|
153
160
|
PERSISTENCE_TIMEOUT_SECONDS = 30.0
|
|
@@ -219,7 +226,7 @@ module Bitfab
|
|
|
219
226
|
# already determines how many traces replay. Supplying trace_ids also
|
|
220
227
|
# emits a warning.
|
|
221
228
|
# @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100)
|
|
222
|
-
# @param name [String, nil] What this run
|
|
229
|
+
# @param name [String, nil] What this run changes, in a few words, such
|
|
223
230
|
# as 'baseline' or 'shorter system prompt'. Bitfab records the commit,
|
|
224
231
|
# branch, tree state, datasets, and who ran it with every experiment, so
|
|
225
232
|
# do not repeat them here.
|
|
@@ -250,18 +257,29 @@ module Bitfab
|
|
|
250
257
|
# against when code_change is "ref", and only then, such as "dev",
|
|
251
258
|
# "release/2.3", or a commit SHA. A branch that exists only on the remote
|
|
252
259
|
# is read as origin/<name>. A name git cannot resolve raises ArgumentError.
|
|
253
|
-
# @param experiment_group_id [String, nil]
|
|
254
|
-
#
|
|
255
|
-
#
|
|
260
|
+
# @param experiment_group_id [String, nil] UUID of the experiment group
|
|
261
|
+
# this run belongs to: one series of experiments testing one thing.
|
|
262
|
+
# Required except for a dry run or skip_experiment_group; leaving it out raises ArgumentError
|
|
263
|
+
# before any request. The run joins that group, and an id no group has
|
|
264
|
+
# yet creates it
|
|
256
265
|
# @param experiment_group_name [String, nil] optional display name for the
|
|
257
|
-
# experiment group,
|
|
258
|
-
# has no name yet. A group that
|
|
259
|
-
# experiment_group_id; passing it
|
|
266
|
+
# experiment group, named after what the run tests, applied when this
|
|
267
|
+
# replay creates the group or the group has no name yet. A group that
|
|
268
|
+
# already has a name keeps it. Requires experiment_group_id; passing it
|
|
269
|
+
# alone raises ArgumentError
|
|
260
270
|
# @param baseline_experiment_id [String, nil] optional UUID of an earlier
|
|
261
271
|
# experiment this run is compared against. It takes precedence over the
|
|
262
272
|
# first finished experiment in the group and over the original traces.
|
|
263
273
|
# It must belong to the caller's organization or the server rejects the
|
|
264
274
|
# replay
|
|
275
|
+
# @param skip_experiment_group [Boolean] runs without an experiment group,
|
|
276
|
+
# for scripts and schedules whose runs should not be grouped. Cannot be
|
|
277
|
+
# combined with experiment_group_id or experiment_group_name; doing so
|
|
278
|
+
# raises ArgumentError before any request
|
|
279
|
+
# @param agent_session_id [String, nil] the coding agent session that
|
|
280
|
+
# launched this run, stored on the experiment. Defaults to the first
|
|
281
|
+
# non-empty of CLAUDE_CODE_SESSION_ID, CODEX_THREAD_ID, and
|
|
282
|
+
# AGENT_SESSION_ID
|
|
265
283
|
# @param dataset_id [String, nil] optional UUID of the dataset this replay
|
|
266
284
|
# runs against, stored on the resulting experiment for durable attribution;
|
|
267
285
|
# mutually exclusive with dataset_ids. Alone it replays the dataset's full
|
|
@@ -327,7 +345,7 @@ module Bitfab
|
|
|
327
345
|
# The experiment fields are nil for a dry run, which creates no experiment,
|
|
328
346
|
# and progress events then carry a nil :experiment_id too.
|
|
329
347
|
def run(client, receiver, method_name, trace_function_key:, limit: nil, trace_ids: nil, name: nil, notes: nil, metadata: nil,
|
|
330
|
-
concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil,
|
|
348
|
+
concurrency: nil, max_concurrency: ReplayConcurrency::UNSET, code_change: nil, code_change_description: nil, code_change_files: nil, code_change_base: nil, experiment_group_id: nil, experiment_group_name: nil, baseline_experiment_id: nil, skip_experiment_group: false, agent_session_id: nil,
|
|
331
349
|
dataset_id: nil, dataset_ids: nil, grader_ids: nil, mock: "marked", attempts: ReplayConcurrency::UNSET, only_with_assertions: false, category_ids: nil, skip_assertion_judging: false, judge_assertions: nil, dry_run: false,
|
|
332
350
|
adapt_inputs: nil, mock_override: nil, db_branch: nil, on_item_start: nil, on_item_finish: nil, on_progress: nil,
|
|
333
351
|
on_experiment_start: nil, experimental_selective_replay: nil)
|
|
@@ -352,9 +370,15 @@ module Bitfab
|
|
|
352
370
|
unless attempts.is_a?(Integer) && attempts.between?(1, 100)
|
|
353
371
|
raise ArgumentError, "attempts must be an integer from 1 to 100."
|
|
354
372
|
end
|
|
373
|
+
if skip_experiment_group && !(experiment_group_id.nil? && experiment_group_name.nil?)
|
|
374
|
+
raise ArgumentError, SKIP_EXPERIMENT_GROUP_CONFLICT
|
|
375
|
+
end
|
|
355
376
|
if experiment_group_name && experiment_group_id.nil?
|
|
356
377
|
raise ArgumentError, "experiment_group_name only applies with experiment_group_id. Pass it together with experiment_group_id."
|
|
357
378
|
end
|
|
379
|
+
if experiment_group_id.to_s.strip.empty? && !dry_run && !skip_experiment_group
|
|
380
|
+
raise ArgumentError, EXPERIMENT_GROUP_REQUIRED
|
|
381
|
+
end
|
|
358
382
|
item_finish_callback = on_item_finish || on_progress
|
|
359
383
|
if trace_ids
|
|
360
384
|
raise ArgumentError, "trace_ids must contain at least one trace ID." if trace_ids.empty?
|
|
@@ -427,6 +451,8 @@ module Bitfab
|
|
|
427
451
|
experiment_group_id:,
|
|
428
452
|
experiment_group_name:,
|
|
429
453
|
baseline_experiment_id:,
|
|
454
|
+
require_experiment_group: !skip_experiment_group,
|
|
455
|
+
agent_session_id:,
|
|
430
456
|
include_db_branch_lease:,
|
|
431
457
|
dataset_ids: resolved_dataset_ids,
|
|
432
458
|
grader_ids:,
|
data/lib/bitfab/replay_cli.rb
CHANGED
|
@@ -48,7 +48,10 @@ module Bitfab
|
|
|
48
48
|
def check_arguments(argv)
|
|
49
49
|
_, replay_argv = parse(argv)
|
|
50
50
|
args = ReplayCli.parse(nil, replay_argv)
|
|
51
|
-
|
|
51
|
+
unless args[:seed] || args[:from_trace]
|
|
52
|
+
ReplayCli.require_code_change(args)
|
|
53
|
+
ReplayCli.require_experiment_group(args)
|
|
54
|
+
end
|
|
52
55
|
0
|
|
53
56
|
end
|
|
54
57
|
|
|
@@ -19,7 +19,7 @@ module Bitfab
|
|
|
19
19
|
# Project-owned registry of production replay roots.
|
|
20
20
|
class ReplayRegistry
|
|
21
21
|
OPTION_NAMES = %i[
|
|
22
|
-
limit trace_ids name notes metadata concurrency max_concurrency experiment_group_id experiment_group_name baseline_experiment_id dataset_id dataset_ids grader_ids mock
|
|
22
|
+
limit trace_ids name notes metadata concurrency max_concurrency experiment_group_id experiment_group_name baseline_experiment_id skip_experiment_group agent_session_id dataset_id dataset_ids grader_ids mock
|
|
23
23
|
mock_override adapt_inputs db_branch attempts only_with_assertions category_ids skip_assertion_judging judge_assertions dry_run on_item_finish
|
|
24
24
|
].freeze
|
|
25
25
|
|
|
@@ -88,6 +88,12 @@ module Bitfab
|
|
|
88
88
|
module ReplayCli
|
|
89
89
|
MAX_PINNED_TRACE_IDS = 100
|
|
90
90
|
CLI_INVOCATION_SKIPPED = %i[pipeline seed run from_trace].freeze
|
|
91
|
+
EXPERIMENT_GROUP_REQUIRED = "--experiment-group-id is required. An experiment group is one series of experiments " \
|
|
92
|
+
"testing one thing. Reuse the id of the group whose runs test the same thing, or pass a new UUID with " \
|
|
93
|
+
"--experiment-group-name, named after what this run tests, to start a group. To run without a group, " \
|
|
94
|
+
"pass --skip-experiment-group."
|
|
95
|
+
SKIP_EXPERIMENT_GROUP_CONFLICT = "--skip-experiment-group runs without an experiment group and cannot be combined " \
|
|
96
|
+
"with --experiment-group-id or --experiment-group-name. Pass one or the other."
|
|
91
97
|
|
|
92
98
|
class ItemsErroredError < StandardError; end
|
|
93
99
|
|
|
@@ -149,6 +155,7 @@ module Bitfab
|
|
|
149
155
|
args = parse(registry, argv.dup)
|
|
150
156
|
return SeedCli.run(registry, argv:, stdout:, stderr:) if args[:seed] || args[:from_trace]
|
|
151
157
|
require_code_change(args)
|
|
158
|
+
require_experiment_group(args)
|
|
152
159
|
stderr.puts "Bitfab: #{Replay::JUDGE_ASSERTIONS_DEPRECATION}" if args[:judge_assertions]
|
|
153
160
|
|
|
154
161
|
registration = registry.fetch(args.fetch(:pipeline))
|
|
@@ -206,7 +213,7 @@ module Bitfab
|
|
|
206
213
|
options[:limit] = bound || 10
|
|
207
214
|
end
|
|
208
215
|
|
|
209
|
-
%i[name notes metadata max_concurrency experiment_group_id experiment_group_name baseline_experiment_id dataset_ids grader_ids mock attempts only_with_assertions category_ids skip_assertion_judging dry_run].each do |key|
|
|
216
|
+
%i[name notes metadata max_concurrency experiment_group_id experiment_group_name baseline_experiment_id skip_experiment_group agent_session_id dataset_ids grader_ids mock attempts only_with_assertions category_ids skip_assertion_judging dry_run].each do |key|
|
|
210
217
|
options[key] = args[key] unless args[key].nil?
|
|
211
218
|
end
|
|
212
219
|
|
|
@@ -333,7 +340,7 @@ module Bitfab
|
|
|
333
340
|
options.on("--run") { args[:run] = true }
|
|
334
341
|
options.on("--from-trace IDS") { |value| args[:from_trace] = comma_separated("--from-trace", value) }
|
|
335
342
|
options.on("--name NAME",
|
|
336
|
-
"What this run
|
|
343
|
+
"What this run changes, in a few words, such as 'baseline' or 'shorter system prompt'. " \
|
|
337
344
|
"Bitfab records the commit, branch, tree state, datasets, and who ran it with every experiment, " \
|
|
338
345
|
"so do not repeat them here.") { |value| args[:name] = value }
|
|
339
346
|
options.on("--notes TEXT",
|
|
@@ -371,13 +378,21 @@ module Bitfab
|
|
|
371
378
|
options.on("--code-change-base BRANCH|COMMIT",
|
|
372
379
|
"With --code-change ref: the branch or commit to diff against.") { |value| args[:code_change_base] = value }
|
|
373
380
|
options.on("--experiment-group-id UUID",
|
|
374
|
-
"
|
|
381
|
+
"Required unless --dry-run or --skip-experiment-group: the experiment group this run belongs to, one series of experiments " \
|
|
382
|
+
"testing one thing. Reuse the id of the group whose runs test the same thing, or pass a new UUID " \
|
|
383
|
+
"with --experiment-group-name, named after what this run tests, to start a group.") { |value| args[:experiment_group_id] = value }
|
|
375
384
|
options.on("--experiment-group-name NAME",
|
|
376
|
-
"Name the experiment group when this run creates it or it has no name yet. " \
|
|
385
|
+
"Name the experiment group after what this run tests, when this run creates it or it has no name yet. " \
|
|
377
386
|
"A group that already has a name keeps it. Requires --experiment-group-id.") { |value| args[:experiment_group_name] = value }
|
|
378
387
|
options.on("--baseline-experiment-id UUID",
|
|
379
388
|
"Compare this run against an earlier experiment instead of the group's first finished run " \
|
|
380
389
|
"or the original traces.") { |value| args[:baseline_experiment_id] = value }
|
|
390
|
+
options.on("--skip-experiment-group",
|
|
391
|
+
"Run without an experiment group, for scripts and schedules whose runs should not be grouped. " \
|
|
392
|
+
"Cannot be combined with --experiment-group-id or --experiment-group-name.") { args[:skip_experiment_group] = true }
|
|
393
|
+
options.on("--agent-session-id ID",
|
|
394
|
+
"The coding agent session that launched this run, stored on the experiment. Defaults to " \
|
|
395
|
+
"CLAUDE_CODE_SESSION_ID, CODEX_THREAD_ID, or AGENT_SESSION_ID, the first one set.") { |value| args[:agent_session_id] = value }
|
|
381
396
|
options.on("--dataset-ids IDS", "--dataset-id IDS") { |value| args[:dataset_ids] = comma_separated("--dataset-ids", value) }
|
|
382
397
|
options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
|
|
383
398
|
options.on("--attempts N", Integer) { |value| args[:attempts] = positive_integer("--attempts", value) }
|
|
@@ -431,6 +446,17 @@ module Bitfab
|
|
|
431
446
|
"--code-change-base goes with --code-change ref, and --code-change ref needs --code-change-base BRANCH|COMMIT."
|
|
432
447
|
end
|
|
433
448
|
|
|
449
|
+
def require_experiment_group(args)
|
|
450
|
+
if args[:skip_experiment_group]
|
|
451
|
+
raise ArgumentError, SKIP_EXPERIMENT_GROUP_CONFLICT unless args[:experiment_group_id].nil? && args[:experiment_group_name].nil?
|
|
452
|
+
|
|
453
|
+
return args
|
|
454
|
+
end
|
|
455
|
+
return args if args[:dry_run] || !args[:experiment_group_id].to_s.strip.empty?
|
|
456
|
+
|
|
457
|
+
raise ArgumentError, EXPERIMENT_GROUP_REQUIRED
|
|
458
|
+
end
|
|
459
|
+
|
|
434
460
|
def registry_dataset_ids(options)
|
|
435
461
|
dataset_id = options[:dataset_id]
|
|
436
462
|
dataset_ids = options[:dataset_ids]
|
data/lib/bitfab/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: bitfab
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.77.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Harvest Team
|
|
@@ -169,6 +169,7 @@ files:
|
|
|
169
169
|
- exe/bitfab-replay
|
|
170
170
|
- exe/bitfab-seed
|
|
171
171
|
- lib/bitfab.rb
|
|
172
|
+
- lib/bitfab/agent_session.rb
|
|
172
173
|
- lib/bitfab/assertions.rb
|
|
173
174
|
- lib/bitfab/baml.rb
|
|
174
175
|
- lib/bitfab/categories.rb
|