bitfab 0.38.0 → 0.38.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/bitfab-replay +6 -0
- data/lib/bitfab/client.rb +47 -16
- data/lib/bitfab/mock_override.rb +14 -6
- data/lib/bitfab/replay.rb +4 -1
- data/lib/bitfab/replay_cli.rb +58 -0
- data/lib/bitfab/replay_registry.rb +290 -0
- data/lib/bitfab/version.rb +1 -1
- data/lib/bitfab.rb +2 -0
- metadata +7 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 29901e7611834921e9378870830af40c08e5af02d1ca0708585a02b169b12e6e
|
|
4
|
+
data.tar.gz: b42984a3710e9399159fd44af3aaac7b76e3155bd3121eaa5482aa27570fbb2e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 576a44cbe7e2fdf7270f14af3d8a98541bb2e15cd2b02289c18d58f033688d6797b2b3a0cfc8153f1ba8adbf0d72f9b9b9241e1bfd5fbd3ffcc687a4cb9adb44
|
|
7
|
+
data.tar.gz: ce3586fbdc541ffad387d87ca4299ee34df5e0c61a5c01a6b5112ee11084cb7900d6616f9e3db45ec0386fc019bc1712f267ea3c5799809880ae3611d3d30cb9
|
data/exe/bitfab-replay
ADDED
data/lib/bitfab/client.rb
CHANGED
|
@@ -91,7 +91,8 @@ module Bitfab
|
|
|
91
91
|
# maximum: 5,000). Ignored when trace_ids or dataset_id is passed because
|
|
92
92
|
# either source already determines how many traces replay. Supplying
|
|
93
93
|
# trace_ids also emits a warning.
|
|
94
|
-
# @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100)
|
|
94
|
+
# @param trace_ids [Array<String>, nil] optional list of trace IDs to replay (max 100),
|
|
95
|
+
# mutually exclusive with dataset_id
|
|
95
96
|
# @param max_concurrency [Integer, nil] max threads for parallel replay (default: 10)
|
|
96
97
|
# @param code_change_description [String, nil] optional rationale for the
|
|
97
98
|
# code change being tested in this replay (stored on the experiment).
|
|
@@ -105,7 +106,8 @@ module Bitfab
|
|
|
105
106
|
# @param experiment_group_id [String, nil] optional UUID grouping multiple
|
|
106
107
|
# replay runs into a single experiment batch
|
|
107
108
|
# @param dataset_id [String, nil] optional UUID of the dataset this replay
|
|
108
|
-
# runs against,
|
|
109
|
+
# runs against, mutually exclusive with trace_ids and stored on the
|
|
110
|
+
# resulting experiment for durable attribution
|
|
109
111
|
# @param grader_ids [Array<String>, nil] optional UUIDs of graders attached
|
|
110
112
|
# directly to this experiment, graded as the union with the dataset's
|
|
111
113
|
# runnable graders at completion; each must be an active/live grader in the
|
|
@@ -120,14 +122,15 @@ module Bitfab
|
|
|
120
122
|
# traces were captured. Receives (args, kwargs, ctx) where ctx is
|
|
121
123
|
# { original_trace_id:, original_span_id: } (with deprecated
|
|
122
124
|
# source_trace_id/source_span_id aliases), and returns [new_args, new_kwargs].
|
|
123
|
-
# @param mock_override [Hash, Array<Hash>, nil] optional selective
|
|
124
|
-
# override(s),
|
|
125
|
+
# @param mock_override [Hash, Proc, Array<Hash, Proc>, nil] optional selective
|
|
126
|
+
# mock override(s), as { match:, value: } hashes or global resolvers. match is a callable:
|
|
125
127
|
# match.call(node) selects spans to substitute (node is
|
|
126
128
|
# { trace_function_key:, span_name:, type:, original_span_id: }). value is
|
|
127
129
|
# EITHER a flat value injected directly OR a callable invoked with
|
|
128
130
|
# { node:, inputs:, kwargs:, get_original_output: }; its result becomes
|
|
129
131
|
# the span's output (full replacement), so downstream real code runs against it. The
|
|
130
|
-
# first matching override
|
|
132
|
+
# first matching override that does not return NO_MOCK_OVERRIDE wins.
|
|
133
|
+
# Per-call overrides take precedence over
|
|
131
134
|
# those registered via register_mock_override, and both take precedence
|
|
132
135
|
# over the base mock strategy. The root span is never overridden.
|
|
133
136
|
# @param db_branch [Boolean, Hash, nil] +true+ or a Hash enables database
|
|
@@ -167,10 +170,9 @@ module Bitfab
|
|
|
167
170
|
# clear_mock_overrides to reset. Per-call replay(mock_override:) overrides
|
|
168
171
|
# take precedence, and both take precedence over the base mock strategy.
|
|
169
172
|
#
|
|
170
|
-
# Accepts
|
|
171
|
-
#
|
|
172
|
-
#
|
|
173
|
-
# with the ctx hash.
|
|
173
|
+
# Accepts the (match, value) positional form, the keyword/hash form, one
|
|
174
|
+
# global resolver callable, or (trace_function_key, override_or_resolver).
|
|
175
|
+
# A resolver can return Bitfab::NO_MOCK_OVERRIDE to decline the span.
|
|
174
176
|
#
|
|
175
177
|
# @example keyword form (primary), flat value
|
|
176
178
|
# client.register_mock_override(
|
|
@@ -184,19 +186,43 @@ module Bitfab
|
|
|
184
186
|
# ->(ctx) { {label: "refund", inputs: ctx[:inputs]} }
|
|
185
187
|
# )
|
|
186
188
|
#
|
|
187
|
-
# @param positional [Array]
|
|
188
|
-
#
|
|
189
|
+
# @param positional [Array] a (match, value) pair, one { match:, value: }
|
|
190
|
+
# hash, one global resolver, or a trace function key followed by a
|
|
191
|
+
# resolver or override hash
|
|
189
192
|
# @param match [#call, nil] keyword form matcher
|
|
190
193
|
# @param value [Object, #call, nil] keyword form injected value or producer
|
|
191
194
|
# @return [void]
|
|
192
195
|
def register_mock_override(*positional, match: nil, value: VALUE_UNSET)
|
|
193
196
|
if match.nil? && value.equal?(VALUE_UNSET)
|
|
194
|
-
if positional.length ==
|
|
197
|
+
if positional.length == 2 && positional[0].is_a?(String)
|
|
198
|
+
trace_function_key, keyed_override = positional
|
|
199
|
+
if keyed_override.is_a?(Hash) && keyed_override[:match].respond_to?(:call) && keyed_override.key?(:value)
|
|
200
|
+
keyed_match = keyed_override[:match]
|
|
201
|
+
@mock_overrides << {
|
|
202
|
+
match: ->(node) { node[:trace_function_key] == trace_function_key && keyed_match.call(node) },
|
|
203
|
+
value: keyed_override[:value]
|
|
204
|
+
}
|
|
205
|
+
return nil
|
|
206
|
+
end
|
|
207
|
+
if keyed_override.respond_to?(:call)
|
|
208
|
+
@mock_overrides << {
|
|
209
|
+
match: ->(node) { node[:trace_function_key] == trace_function_key },
|
|
210
|
+
value: keyed_override
|
|
211
|
+
}
|
|
212
|
+
return nil
|
|
213
|
+
end
|
|
214
|
+
raise ArgumentError,
|
|
215
|
+
"register_mock_override(trace_function_key, override) requires " \
|
|
216
|
+
"a { match:, value: } hash or callable resolver."
|
|
217
|
+
elsif positional.length == 1 && positional[0].is_a?(Hash)
|
|
195
218
|
override = positional[0]
|
|
196
219
|
match = override[:match]
|
|
197
220
|
# Distinguish an omitted :value from an explicit `value: nil`: the
|
|
198
221
|
# key's presence, not a nil read, is what marks it provided.
|
|
199
222
|
value = override.key?(:value) ? override[:value] : VALUE_UNSET
|
|
223
|
+
elsif positional.length == 1 && positional[0].respond_to?(:call)
|
|
224
|
+
@mock_overrides << {match: ->(_) { true }, value: positional[0]}
|
|
225
|
+
return nil
|
|
200
226
|
elsif positional.length == 2
|
|
201
227
|
match, value = positional
|
|
202
228
|
end
|
|
@@ -780,8 +806,9 @@ module Bitfab
|
|
|
780
806
|
type: span_type,
|
|
781
807
|
original_span_id: mock_entry && mock_entry[:source_span_id]
|
|
782
808
|
}
|
|
783
|
-
|
|
784
|
-
|
|
809
|
+
overrides.each do |override|
|
|
810
|
+
next unless override[:match].call(node)
|
|
811
|
+
|
|
785
812
|
value = override[:value]
|
|
786
813
|
# value is EITHER a flat value (injected directly) OR a callable
|
|
787
814
|
# invoked with the ctx hash. The result IS the span's output (full
|
|
@@ -789,7 +816,10 @@ module Bitfab
|
|
|
789
816
|
# override always injects, execute_span short-circuits even when this
|
|
790
817
|
# is nil (nil != MOCK_REPLAY_MISS). To inject a proc as the literal
|
|
791
818
|
# output, wrap it in a callable value.
|
|
792
|
-
|
|
819
|
+
unless value.respond_to?(:call)
|
|
820
|
+
return [value, "override"] unless value.equal?(NO_MOCK_OVERRIDE)
|
|
821
|
+
next
|
|
822
|
+
end
|
|
793
823
|
|
|
794
824
|
get_original_output = lambda do
|
|
795
825
|
unless mock_entry
|
|
@@ -797,7 +827,8 @@ module Bitfab
|
|
|
797
827
|
end
|
|
798
828
|
resolve_recorded_output(mock_entry, replay_ctx)
|
|
799
829
|
end
|
|
800
|
-
|
|
830
|
+
resolved = value.call({node:, inputs: args, kwargs:, get_original_output:})
|
|
831
|
+
return [resolved, "override"] unless resolved.equal?(NO_MOCK_OVERRIDE)
|
|
801
832
|
end
|
|
802
833
|
end
|
|
803
834
|
|
data/lib/bitfab/mock_override.rb
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Bitfab
|
|
4
|
+
# Return from an override resolver to continue to the next override and then
|
|
5
|
+
# the replay's base mock strategy.
|
|
6
|
+
NO_MOCK_OVERRIDE = Object.new.freeze
|
|
7
|
+
|
|
4
8
|
# Selective mock overrides for replay. Mirrors the TypeScript SDK's
|
|
5
9
|
# mockOverride feature.
|
|
6
10
|
#
|
|
@@ -10,8 +14,10 @@ module Bitfab
|
|
|
10
14
|
# substituted output. This is a third mock mode alongside "run real code" and
|
|
11
15
|
# "replay recorded output" (see MOCK_STRATEGIES in replay.rb).
|
|
12
16
|
#
|
|
13
|
-
# An override is a (match, value) pair
|
|
14
|
-
#
|
|
17
|
+
# An override is a (match, value) pair represented as { match:, value: }, or
|
|
18
|
+
# one global resolver invoked for every child span. A resolver can route on
|
|
19
|
+
# ctx[:node][:trace_function_key] and return Bitfab::NO_MOCK_OVERRIDE to
|
|
20
|
+
# decline the current span.
|
|
15
21
|
#
|
|
16
22
|
# - match is a callable (proc/lambda): match.call(node) -> boolean. node is a
|
|
17
23
|
# structural-metadata hash { trace_function_key:, span_name:, type:,
|
|
@@ -38,9 +44,8 @@ module Bitfab
|
|
|
38
44
|
module MockOverride
|
|
39
45
|
module_function
|
|
40
46
|
|
|
41
|
-
# Normalize the per-call `mock_override` option (
|
|
42
|
-
#
|
|
43
|
-
# order is preserved.
|
|
47
|
+
# Normalize the per-call `mock_override` option (an override hash, a global
|
|
48
|
+
# resolver callable, an array, or nil) into an ordered override array.
|
|
44
49
|
#
|
|
45
50
|
# Each override is validated: it must be a hash with a callable `:match` and
|
|
46
51
|
# must include a `:value` key. A forgotten `:value` raises rather than
|
|
@@ -49,12 +54,15 @@ module Bitfab
|
|
|
49
54
|
# `register_mock_override`, and the Python/TypeScript SDKs, where the per-call
|
|
50
55
|
# override object requires a value.
|
|
51
56
|
#
|
|
52
|
-
# @param mock_override [Hash, Array<Hash>, nil]
|
|
57
|
+
# @param mock_override [Hash, Proc, Array<Hash, Proc>, nil]
|
|
53
58
|
# @return [Array<Hash>]
|
|
54
59
|
def normalize(mock_override)
|
|
55
60
|
return [] if mock_override.nil?
|
|
56
61
|
|
|
57
62
|
overrides = mock_override.is_a?(Array) ? mock_override : [mock_override]
|
|
63
|
+
overrides = overrides.map do |override|
|
|
64
|
+
override.respond_to?(:call) ? {match: ->(_) { true }, value: override} : override
|
|
65
|
+
end
|
|
58
66
|
overrides.each do |override|
|
|
59
67
|
unless override.is_a?(Hash) && override[:match].respond_to?(:call)
|
|
60
68
|
raise ArgumentError,
|
data/lib/bitfab/replay.rb
CHANGED
|
@@ -260,11 +260,14 @@ module Bitfab
|
|
|
260
260
|
raise ArgumentError, "trace_ids supports at most 100 trace IDs per replay (got #{trace_ids.length})."
|
|
261
261
|
end
|
|
262
262
|
end
|
|
263
|
+
if trace_ids && dataset_id
|
|
264
|
+
raise ArgumentError,
|
|
265
|
+
"trace_ids and dataset_id select different replay sources and cannot be used together."
|
|
266
|
+
end
|
|
263
267
|
if limit && trace_ids
|
|
264
268
|
warn "Bitfab: limit is ignored when trace_ids is passed: the explicit trace ID list already " \
|
|
265
269
|
"determines how many traces replay."
|
|
266
270
|
end
|
|
267
|
-
|
|
268
271
|
# Reject a trace_function_key that contradicts the method's declared key.
|
|
269
272
|
# replay() fetches historical traces by trace_function_key but records the
|
|
270
273
|
# replayed spans under the method's own bitfab_span key (via send below),
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "optparse"
|
|
4
|
+
|
|
5
|
+
module Bitfab
|
|
6
|
+
# Installed command that loads a project-owned registry module.
|
|
7
|
+
module ReplayCommand
|
|
8
|
+
USAGE = "Usage: bitfab-replay --registry <path> <pipeline> [replay options]"
|
|
9
|
+
HELP = "#{USAGE}\nRun bitfab-replay --registry <path> --help to list replay options."
|
|
10
|
+
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
def run(argv: ARGV, stdout: $stdout, stderr: $stderr, registry_loader: nil)
|
|
14
|
+
registry_path, replay_argv = parse(argv)
|
|
15
|
+
loader = registry_loader || method(:load_registry)
|
|
16
|
+
registry = loader.call(registry_path)
|
|
17
|
+
ReplayCli.run(registry, argv: replay_argv, stdout:, stderr:)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def main(argv: ARGV, stdout: $stdout, stderr: $stderr)
|
|
21
|
+
if (argv.include?("--help") || argv.include?("-h")) && !argv.include?("--registry")
|
|
22
|
+
stdout.puts HELP
|
|
23
|
+
return 0
|
|
24
|
+
end
|
|
25
|
+
run(argv:, stdout:, stderr:)
|
|
26
|
+
0
|
|
27
|
+
rescue => error
|
|
28
|
+
stderr.puts error.message
|
|
29
|
+
1
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def parse(argv)
|
|
33
|
+
values = argv.dup
|
|
34
|
+
registry_index = values.index("--registry")
|
|
35
|
+
registry_path = values[registry_index + 1] if registry_index
|
|
36
|
+
unless registry_path && !registry_path.start_with?("--")
|
|
37
|
+
raise OptionParser::MissingArgument, USAGE
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
values.slice!(registry_index, 2)
|
|
41
|
+
raise OptionParser::MissingArgument, USAGE if values.empty?
|
|
42
|
+
|
|
43
|
+
[File.expand_path(registry_path), values]
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def load_registry(path)
|
|
47
|
+
container = Module.new
|
|
48
|
+
container.module_eval(File.read(path), path)
|
|
49
|
+
registry = container.const_get(:REGISTRY, false) if container.const_defined?(:REGISTRY, false)
|
|
50
|
+
unless registry.is_a?(ReplayRegistry)
|
|
51
|
+
raise ArgumentError,
|
|
52
|
+
"Replay registry '#{path}' must define REGISTRY as a Bitfab::ReplayRegistry."
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
registry
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "optparse"
|
|
5
|
+
|
|
6
|
+
module Bitfab
|
|
7
|
+
# One project function exposed through the SDK-owned replay command.
|
|
8
|
+
ReplayRegistration = Data.define(
|
|
9
|
+
:client,
|
|
10
|
+
:receiver,
|
|
11
|
+
:method_name,
|
|
12
|
+
:trace_function_key,
|
|
13
|
+
:options,
|
|
14
|
+
:options_factory
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
ReplayRegistryContext = Data.define(:params)
|
|
18
|
+
|
|
19
|
+
# Project-owned registry of production replay roots.
|
|
20
|
+
class ReplayRegistry
|
|
21
|
+
OPTION_NAMES = %i[
|
|
22
|
+
limit trace_ids name max_concurrency code_change_description
|
|
23
|
+
code_change_files experiment_group_id dataset_id grader_ids mock
|
|
24
|
+
mock_override adapt_inputs db_branch
|
|
25
|
+
].freeze
|
|
26
|
+
|
|
27
|
+
def initialize
|
|
28
|
+
@entries = {}
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def names
|
|
32
|
+
@entries.keys.freeze
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Register the exact traced receiver method production invokes.
|
|
36
|
+
#
|
|
37
|
+
# A +bitfab_span+ method carries its trace function key automatically.
|
|
38
|
+
# Handler-instrumented or otherwise plain methods must pass
|
|
39
|
+
# +trace_function_key:+ explicitly. Pass executable per-function replay
|
|
40
|
+
# behavior such as +mock_override:+ and +adapt_inputs:+ as keyword options;
|
|
41
|
+
# the installed command forwards them to +Bitfab::Client#replay+.
|
|
42
|
+
def register(name, receiver, method_name, client: nil, trace_function_key: nil, options_factory: nil, **options)
|
|
43
|
+
raise ArgumentError, "Replay registry names cannot be empty." if name.to_s.empty?
|
|
44
|
+
raise ArgumentError, "Replay registry already contains '#{name}'." if @entries.key?(name.to_s)
|
|
45
|
+
if options_factory && !options_factory.respond_to?(:call)
|
|
46
|
+
raise ArgumentError, "Replay registry options_factory must be callable."
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
key = trace_function_key || Traceable.trace_function_key_for(receiver, method_name)
|
|
50
|
+
unless key
|
|
51
|
+
raise ArgumentError,
|
|
52
|
+
"Replay registry entry uses a plain method. Set trace_function_key: " \
|
|
53
|
+
"to the key its production handler records."
|
|
54
|
+
end
|
|
55
|
+
validate_options(options)
|
|
56
|
+
|
|
57
|
+
@entries[name.to_s] = ReplayRegistration.new(
|
|
58
|
+
client: client || Bitfab.client,
|
|
59
|
+
receiver:,
|
|
60
|
+
method_name: method_name.to_sym,
|
|
61
|
+
trace_function_key: key,
|
|
62
|
+
options: options.freeze,
|
|
63
|
+
options_factory:
|
|
64
|
+
)
|
|
65
|
+
self
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def fetch(name)
|
|
69
|
+
@entries.fetch(name)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def validate_options(options)
|
|
73
|
+
unknown = options.keys - OPTION_NAMES
|
|
74
|
+
return if unknown.empty?
|
|
75
|
+
|
|
76
|
+
raise ArgumentError,
|
|
77
|
+
"Unknown replay registry option(s): #{unknown.sort.join(", ")}. " \
|
|
78
|
+
"Allowed options: #{OPTION_NAMES.sort.join(", ")}."
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Argument parsing and output for the SDK-owned replay command.
|
|
83
|
+
module ReplayCli
|
|
84
|
+
module_function
|
|
85
|
+
|
|
86
|
+
def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
|
|
87
|
+
args = parse(registry, argv.dup)
|
|
88
|
+
registration = registry.fetch(args.fetch(:pipeline))
|
|
89
|
+
options = registration.options.dup
|
|
90
|
+
params = load_parameters(args[:params], args.fetch(:param, []))
|
|
91
|
+
if registration.options_factory
|
|
92
|
+
dynamic_options = registration.options_factory.call(ReplayRegistryContext.new(params:))
|
|
93
|
+
unless dynamic_options.is_a?(Hash)
|
|
94
|
+
raise ArgumentError, "Replay registry options_factory must return a Hash."
|
|
95
|
+
end
|
|
96
|
+
registry.validate_options(dynamic_options)
|
|
97
|
+
options.merge!(dynamic_options)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
if options[:trace_ids] && options[:dataset_id]
|
|
101
|
+
raise ArgumentError,
|
|
102
|
+
"Replay registry options trace_ids and dataset_id select different sources and cannot be used together."
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
if args[:trace_ids]
|
|
106
|
+
options.delete(:limit)
|
|
107
|
+
options.delete(:dataset_id)
|
|
108
|
+
options[:trace_ids] = args[:trace_ids]
|
|
109
|
+
elsif args[:dataset_id]
|
|
110
|
+
options.delete(:trace_ids)
|
|
111
|
+
options.delete(:limit)
|
|
112
|
+
elsif args[:limit]
|
|
113
|
+
options.delete(:trace_ids)
|
|
114
|
+
options.delete(:dataset_id)
|
|
115
|
+
options[:limit] = args[:limit]
|
|
116
|
+
elsif options[:trace_ids] || options[:dataset_id]
|
|
117
|
+
options.delete(:limit)
|
|
118
|
+
else
|
|
119
|
+
options[:limit] = options.fetch(:limit, 10)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
%i[name max_concurrency experiment_group_id dataset_id grader_ids mock].each do |key|
|
|
123
|
+
options[key] = args[key] unless args[key].nil?
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
unless args[:db_branch].nil?
|
|
127
|
+
if args[:db_branch]
|
|
128
|
+
configured = options[:db_branch]
|
|
129
|
+
options[:db_branch] = (configured.nil? || configured == false) ? true : configured
|
|
130
|
+
else
|
|
131
|
+
options[:db_branch] = false
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
if args[:code_change]
|
|
136
|
+
code_change = load_code_change(args[:code_change])
|
|
137
|
+
options[:code_change_description] = code_change.fetch("description")
|
|
138
|
+
options[:code_change_files] = code_change.fetch("files")
|
|
139
|
+
elsif args[:no_code_change]
|
|
140
|
+
options[:code_change_description] = nil
|
|
141
|
+
options[:code_change_files] = nil
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
reporter = Bitfab.method(:report_replay_progress)
|
|
145
|
+
options[:on_item_start] = reporter
|
|
146
|
+
options[:on_item_finish] = reporter
|
|
147
|
+
|
|
148
|
+
count = options[:trace_ids]&.length || options[:limit] || "dataset"
|
|
149
|
+
stderr.puts "[replay] Replaying #{count} traces from \"#{registration.trace_function_key}\"..."
|
|
150
|
+
|
|
151
|
+
result = begin
|
|
152
|
+
registration.client.replay(
|
|
153
|
+
registration.receiver,
|
|
154
|
+
registration.method_name,
|
|
155
|
+
trace_function_key: registration.trace_function_key,
|
|
156
|
+
**options
|
|
157
|
+
)
|
|
158
|
+
rescue ReplayError => error
|
|
159
|
+
error.items.each do |item|
|
|
160
|
+
item_error = item[:trace_error] || item[:replay_error] || item[:error]
|
|
161
|
+
stderr.puts "#{item[:original_trace_id]}: #{item_error}"
|
|
162
|
+
end
|
|
163
|
+
raise
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
render_summary(args.fetch(:pipeline), result, stderr)
|
|
167
|
+
stdout.puts Bitfab.serialize_replay_result(result)
|
|
168
|
+
result
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def parse(registry, argv)
|
|
172
|
+
if argv.include?("--db-branch") && argv.include?("--no-db-branch")
|
|
173
|
+
raise OptionParser::InvalidArgument, "--db-branch and --no-db-branch cannot be used together"
|
|
174
|
+
end
|
|
175
|
+
args = {}
|
|
176
|
+
parser = OptionParser.new do |options|
|
|
177
|
+
options.banner = "Usage: bitfab-replay <#{registry.names.join("|")}> [options]"
|
|
178
|
+
options.on("--limit N", Integer) { |value| args[:limit] = positive_integer("--limit", value) }
|
|
179
|
+
options.on("--trace-ids IDS") { |value| args[:trace_ids] = comma_separated("--trace-ids", value) }
|
|
180
|
+
options.on("--name NAME") { |value| args[:name] = value }
|
|
181
|
+
options.on("--concurrency N", Integer) do |value|
|
|
182
|
+
args[:max_concurrency] = positive_integer("--concurrency", value)
|
|
183
|
+
end
|
|
184
|
+
options.on("--max-concurrency N", Integer) do |value|
|
|
185
|
+
args[:max_concurrency] = positive_integer("--max-concurrency", value)
|
|
186
|
+
end
|
|
187
|
+
options.on("--code-change PATH") { |value| args[:code_change] = value }
|
|
188
|
+
options.on("--experiment-group-id UUID") { |value| args[:experiment_group_id] = value }
|
|
189
|
+
options.on("--dataset-id UUID") { |value| args[:dataset_id] = value }
|
|
190
|
+
options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
|
|
191
|
+
options.on("--mock STRATEGY", %w[none all marked]) { |value| args[:mock] = value }
|
|
192
|
+
options.on("--db-branch") { args[:db_branch] = true }
|
|
193
|
+
options.on("--no-db-branch") { args[:db_branch] = false }
|
|
194
|
+
options.on("--no-code-change") { args[:no_code_change] = true }
|
|
195
|
+
options.on("--params PATH") { |value| args[:params] = value }
|
|
196
|
+
options.on("--param NAME=VALUE") { |value| (args[:param] ||= []) << value }
|
|
197
|
+
end
|
|
198
|
+
parser.parse!(argv)
|
|
199
|
+
if args[:trace_ids] && args[:dataset_id]
|
|
200
|
+
raise OptionParser::InvalidArgument,
|
|
201
|
+
"--trace-ids and --dataset-id select different replay sources and cannot be used together"
|
|
202
|
+
end
|
|
203
|
+
if args[:code_change] && args[:no_code_change]
|
|
204
|
+
raise OptionParser::InvalidArgument, "--code-change and --no-code-change cannot be used together"
|
|
205
|
+
end
|
|
206
|
+
pipeline = argv.shift
|
|
207
|
+
raise OptionParser::MissingArgument, parser.banner unless pipeline && registry.names.include?(pipeline)
|
|
208
|
+
raise OptionParser::InvalidArgument, "unexpected arguments: #{argv.join(" ")}" unless argv.empty?
|
|
209
|
+
|
|
210
|
+
args.merge(pipeline:)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def positive_integer(flag, value)
|
|
214
|
+
raise OptionParser::InvalidArgument, "#{flag} must be a positive integer" if value < 1
|
|
215
|
+
|
|
216
|
+
value
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def comma_separated(flag, value)
|
|
220
|
+
values = value.split(",").map(&:strip).reject(&:empty?)
|
|
221
|
+
raise OptionParser::InvalidArgument, "#{flag} must contain at least one value" if values.empty?
|
|
222
|
+
|
|
223
|
+
values
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def load_code_change(path)
|
|
227
|
+
value = JSON.parse(File.read(path))
|
|
228
|
+
unless value.is_a?(Hash) && value["description"].is_a?(String) && value["files"].is_a?(Array)
|
|
229
|
+
raise ArgumentError, "Invalid --code-change file '#{path}': expected { description, files }."
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
value
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def parse_parameter(raw)
|
|
236
|
+
key, separator, value = raw.partition("=")
|
|
237
|
+
key = key.strip
|
|
238
|
+
if separator.empty? || key.empty?
|
|
239
|
+
raise OptionParser::InvalidArgument,
|
|
240
|
+
"Invalid --param '#{raw}': expected a non-empty name=value pair"
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
parsed = begin
|
|
244
|
+
JSON.parse(value)
|
|
245
|
+
rescue JSON::ParserError
|
|
246
|
+
value
|
|
247
|
+
end
|
|
248
|
+
[key, parsed]
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
def load_parameters(path, raw_parameters)
|
|
252
|
+
params = {}
|
|
253
|
+
if path
|
|
254
|
+
value = JSON.parse(File.read(path))
|
|
255
|
+
unless value.is_a?(Hash)
|
|
256
|
+
raise ArgumentError, "Invalid --params file '#{path}': expected a JSON object."
|
|
257
|
+
end
|
|
258
|
+
params.merge!(value)
|
|
259
|
+
end
|
|
260
|
+
raw_parameters.each do |raw|
|
|
261
|
+
key, value = parse_parameter(raw)
|
|
262
|
+
params[key] = value
|
|
263
|
+
end
|
|
264
|
+
params.freeze
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def render_summary(pipeline, result, stderr)
|
|
268
|
+
same = 0
|
|
269
|
+
changed = 0
|
|
270
|
+
errors = 0
|
|
271
|
+
result[:items].each do |item|
|
|
272
|
+
if item[:error]
|
|
273
|
+
errors += 1
|
|
274
|
+
elsif item[:result] == item[:original_output]
|
|
275
|
+
same += 1
|
|
276
|
+
else
|
|
277
|
+
changed += 1
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
stderr.puts "\n─── Summary ───"
|
|
282
|
+
stderr.puts " Pipeline: #{pipeline}"
|
|
283
|
+
stderr.puts " Replayed: #{result[:items].length}"
|
|
284
|
+
stderr.puts " Same: #{same}"
|
|
285
|
+
stderr.puts " Changed: #{changed}"
|
|
286
|
+
stderr.puts " Errors: #{errors}" if errors > 0
|
|
287
|
+
stderr.puts "\n #{result[:test_run_url]}"
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
end
|
data/lib/bitfab/version.rb
CHANGED
data/lib/bitfab.rb
CHANGED
|
@@ -15,6 +15,8 @@ require_relative "bitfab/replay"
|
|
|
15
15
|
require_relative "bitfab/replay_branch"
|
|
16
16
|
require_relative "bitfab/client"
|
|
17
17
|
require_relative "bitfab/traceable"
|
|
18
|
+
require_relative "bitfab/replay_registry"
|
|
19
|
+
require_relative "bitfab/replay_cli"
|
|
18
20
|
|
|
19
21
|
module Bitfab
|
|
20
22
|
# No-op span handle returned when outside a span context.
|
metadata
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: bitfab
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.38.
|
|
4
|
+
version: 0.38.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Harvest Team
|
|
8
|
-
bindir:
|
|
8
|
+
bindir: exe
|
|
9
9
|
cert_chain: []
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
@@ -131,11 +131,13 @@ description: Client library for sending function execution spans to the Bitfab A
|
|
|
131
131
|
Provides automatic tracing with nested span support.
|
|
132
132
|
email:
|
|
133
133
|
- team@goharvest.ai
|
|
134
|
-
executables:
|
|
134
|
+
executables:
|
|
135
|
+
- bitfab-replay
|
|
135
136
|
extensions: []
|
|
136
137
|
extra_rdoc_files: []
|
|
137
138
|
files:
|
|
138
139
|
- README.md
|
|
140
|
+
- exe/bitfab-replay
|
|
139
141
|
- lib/bitfab.rb
|
|
140
142
|
- lib/bitfab/client.rb
|
|
141
143
|
- lib/bitfab/compress.rb
|
|
@@ -147,6 +149,8 @@ files:
|
|
|
147
149
|
- lib/bitfab/payload_budget.rb
|
|
148
150
|
- lib/bitfab/replay.rb
|
|
149
151
|
- lib/bitfab/replay_branch.rb
|
|
152
|
+
- lib/bitfab/replay_cli.rb
|
|
153
|
+
- lib/bitfab/replay_registry.rb
|
|
150
154
|
- lib/bitfab/serialize.rb
|
|
151
155
|
- lib/bitfab/span_context.rb
|
|
152
156
|
- lib/bitfab/traceable.rb
|