freddy 1.6.0 → 2.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +31 -0
- data/.rubocop.yml +9 -28
- data/.ruby-gemset +1 -1
- data/.ruby-version +1 -1
- data/Gemfile +3 -4
- data/README.md +49 -18
- data/freddy.gemspec +12 -18
- data/lib/freddy/adapters/bunny_adapter.rb +20 -3
- data/lib/freddy/adapters.rb +3 -28
- data/lib/freddy/consumers/respond_to_consumer.rb +3 -13
- data/lib/freddy/consumers/response_consumer.rb +2 -4
- data/lib/freddy/consumers/tap_into_consumer.rb +25 -34
- data/lib/freddy/consumers.rb +1 -1
- data/lib/freddy/delivery.rb +45 -13
- data/lib/freddy/encoding.rb +27 -0
- data/lib/freddy/payload.rb +4 -34
- data/lib/freddy/producers/reply_producer.rb +12 -5
- data/lib/freddy/producers/send_and_forget_producer.rb +9 -12
- data/lib/freddy/producers/send_and_wait_response_producer.rb +19 -33
- data/lib/freddy/producers.rb +1 -1
- data/lib/freddy/request_manager.rb +1 -9
- data/lib/freddy/tracing.rb +37 -0
- data/lib/freddy/version.rb +5 -0
- data/lib/freddy.rb +18 -20
- data/spec/.rubocop.yml +26 -0
- data/spec/freddy/error_response_spec.rb +6 -6
- data/spec/freddy/freddy_spec.rb +23 -0
- data/spec/freddy/payload_spec.rb +25 -16
- data/spec/integration/concurrency_spec.rb +8 -12
- data/spec/integration/tap_into_with_group_spec.rb +15 -0
- data/spec/integration/tracing_spec.rb +45 -32
- data/spec/spec_helper.rb +5 -13
- metadata +47 -20
- data/.npmignore +0 -8
- data/.travis.yml +0 -13
- data/lib/freddy/adapters/march_hare_adapter.rb +0 -64
- data/lib/freddy/trace_carrier.rb +0 -28
- data/spec/freddy/trace_carrier_spec.rb +0 -56
@@ -7,9 +7,9 @@ class Freddy
|
|
7
7
|
new(*attrs).consume(&block)
|
8
8
|
end
|
9
9
|
|
10
|
-
def initialize(thread_pool:,
|
10
|
+
def initialize(thread_pool:, patterns:, channel:, options:)
|
11
11
|
@consume_thread_pool = thread_pool
|
12
|
-
@
|
12
|
+
@patterns = patterns
|
13
13
|
@channel = channel
|
14
14
|
@options = options
|
15
15
|
|
@@ -31,46 +31,37 @@ class Freddy
|
|
31
31
|
def create_queue
|
32
32
|
topic_exchange = @channel.topic(Freddy::FREDDY_TOPIC_EXCHANGE_NAME)
|
33
33
|
|
34
|
-
|
35
|
-
|
36
|
-
.queue("groups.#{group}", durable: durable?)
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
34
|
+
queue =
|
35
|
+
if group
|
36
|
+
@channel.queue("groups.#{group}", durable: durable?)
|
37
|
+
else
|
38
|
+
@channel.queue('', exclusive: true)
|
39
|
+
end
|
40
|
+
|
41
|
+
@patterns.each do |pattern|
|
42
|
+
queue.bind(topic_exchange, routing_key: pattern)
|
42
43
|
end
|
44
|
+
|
45
|
+
queue
|
43
46
|
end
|
44
47
|
|
45
48
|
def process_message(_queue, delivery)
|
46
49
|
@consume_thread_pool.process do
|
47
|
-
|
48
|
-
scope = delivery.build_trace("freddy:observe:#{@pattern}",
|
49
|
-
tags: {
|
50
|
-
'message_bus.destination' => @pattern,
|
51
|
-
'message_bus.correlation_id' => delivery.correlation_id,
|
52
|
-
'component' => 'freddy',
|
53
|
-
'span.kind' => 'consumer' # Message Bus
|
54
|
-
},
|
55
|
-
force_follows_from: true)
|
56
|
-
|
50
|
+
delivery.in_span(force_follows_from: true) do
|
57
51
|
yield delivery.payload, delivery.routing_key
|
58
|
-
|
59
52
|
@channel.acknowledge(delivery.tag)
|
60
|
-
rescue StandardError
|
61
|
-
case on_exception
|
62
|
-
when :reject
|
63
|
-
@channel.reject(delivery.tag)
|
64
|
-
when :requeue
|
65
|
-
@channel.reject(delivery.tag, true)
|
66
|
-
else
|
67
|
-
@channel.acknowledge(delivery.tag)
|
68
|
-
end
|
69
|
-
|
70
|
-
raise
|
71
|
-
ensure
|
72
|
-
scope.close
|
73
53
|
end
|
54
|
+
rescue StandardError
|
55
|
+
case on_exception
|
56
|
+
when :reject
|
57
|
+
@channel.reject(delivery.tag)
|
58
|
+
when :requeue
|
59
|
+
@channel.reject(delivery.tag, true)
|
60
|
+
else
|
61
|
+
@channel.acknowledge(delivery.tag)
|
62
|
+
end
|
63
|
+
|
64
|
+
raise
|
74
65
|
end
|
75
66
|
end
|
76
67
|
|
data/lib/freddy/consumers.rb
CHANGED
data/lib/freddy/delivery.rb
CHANGED
@@ -4,11 +4,12 @@ class Freddy
|
|
4
4
|
class Delivery
|
5
5
|
attr_reader :routing_key, :payload, :tag
|
6
6
|
|
7
|
-
def initialize(payload, metadata, routing_key, tag)
|
7
|
+
def initialize(payload, metadata, routing_key, tag, exchange)
|
8
8
|
@payload = payload
|
9
9
|
@metadata = metadata
|
10
10
|
@routing_key = routing_key
|
11
11
|
@tag = tag
|
12
|
+
@exchange = exchange
|
12
13
|
end
|
13
14
|
|
14
15
|
def correlation_id
|
@@ -23,20 +24,51 @@ class Freddy
|
|
23
24
|
@metadata.reply_to
|
24
25
|
end
|
25
26
|
|
26
|
-
def
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
27
|
+
def in_span(force_follows_from: false, &block)
|
28
|
+
name = "#{@exchange}.#{@routing_key} receive"
|
29
|
+
kind = OpenTelemetry::Trace::SpanKind::CONSUMER
|
30
|
+
producer_context = OpenTelemetry.propagation.extract(@metadata[:headers] || {})
|
31
|
+
|
32
|
+
if force_follows_from
|
33
|
+
producer_span_context = OpenTelemetry::Trace.current_span(producer_context).context
|
34
|
+
|
35
|
+
links = []
|
36
|
+
links << OpenTelemetry::Trace::Link.new(producer_span_context) if producer_span_context.valid?
|
37
|
+
|
38
|
+
root_span = Freddy.tracer.start_root_span(name, attributes: span_attributes, links: links, kind: kind)
|
39
|
+
OpenTelemetry::Trace.with_span(root_span) do
|
40
|
+
Freddy.tracer.in_span(name, attributes: span_attributes, links: links, kind: kind, &block)
|
41
|
+
ensure
|
42
|
+
root_span.finish
|
37
43
|
end
|
44
|
+
else
|
45
|
+
OpenTelemetry::Context.with_current(producer_context) do
|
46
|
+
Freddy.tracer.in_span(name, attributes: span_attributes, kind: kind, &block)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
def span_attributes
|
54
|
+
destination_kind = @exchange == '' ? 'queue' : 'topic'
|
55
|
+
|
56
|
+
attributes = {
|
57
|
+
'payload.type' => (@payload[:type] || 'unknown').to_s,
|
58
|
+
OpenTelemetry::SemanticConventions::Trace::MESSAGING_SYSTEM => 'rabbitmq',
|
59
|
+
OpenTelemetry::SemanticConventions::Trace::MESSAGING_DESTINATION => @exchange,
|
60
|
+
OpenTelemetry::SemanticConventions::Trace::MESSAGING_DESTINATION_KIND => destination_kind,
|
61
|
+
OpenTelemetry::SemanticConventions::Trace::MESSAGING_RABBITMQ_ROUTING_KEY => @routing_key,
|
62
|
+
OpenTelemetry::SemanticConventions::Trace::MESSAGING_OPERATION => 'process'
|
63
|
+
}
|
64
|
+
|
65
|
+
# There's no correlation_id when a message was sent using
|
66
|
+
# `Freddy#deliver`.
|
67
|
+
if correlation_id
|
68
|
+
attributes[OpenTelemetry::SemanticConventions::Trace::MESSAGING_CONVERSATION_ID] = correlation_id
|
69
|
+
end
|
38
70
|
|
39
|
-
|
71
|
+
attributes
|
40
72
|
end
|
41
73
|
end
|
42
74
|
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'zlib'
|
4
|
+
|
5
|
+
class Freddy
|
6
|
+
class Encoding
|
7
|
+
ZLIB_CONTENT_ENCODING = 'zlib'
|
8
|
+
|
9
|
+
def self.compress(data, encoding)
|
10
|
+
case encoding
|
11
|
+
when ZLIB_CONTENT_ENCODING
|
12
|
+
::Zlib::Deflate.deflate(data)
|
13
|
+
else
|
14
|
+
data
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.uncompress(data, encoding)
|
19
|
+
case encoding
|
20
|
+
when ZLIB_CONTENT_ENCODING
|
21
|
+
::Zlib::Inflate.inflate(data)
|
22
|
+
else
|
23
|
+
data
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/lib/freddy/payload.rb
CHANGED
@@ -1,17 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
4
|
-
require 'oj'
|
5
|
-
rescue LoadError
|
6
|
-
require 'symbolizer'
|
7
|
-
require 'json'
|
8
|
-
end
|
3
|
+
require 'oj'
|
9
4
|
|
10
5
|
class Freddy
|
11
6
|
class Payload
|
12
|
-
def self.parse(payload)
|
7
|
+
def self.parse(payload, encoding)
|
13
8
|
return {} if payload == 'null'
|
14
9
|
|
10
|
+
payload = Freddy::Encoding.uncompress(payload, encoding)
|
15
11
|
json_handler.parse(payload)
|
16
12
|
end
|
17
13
|
|
@@ -20,7 +16,7 @@ class Freddy
|
|
20
16
|
end
|
21
17
|
|
22
18
|
def self.json_handler
|
23
|
-
@json_handler ||=
|
19
|
+
@json_handler ||= OjAdapter
|
24
20
|
end
|
25
21
|
|
26
22
|
class OjAdapter
|
@@ -35,31 +31,5 @@ class Freddy
|
|
35
31
|
Oj.dump(payload, DUMP_OPTIONS)
|
36
32
|
end
|
37
33
|
end
|
38
|
-
|
39
|
-
class JsonAdapter
|
40
|
-
def self.parse(payload)
|
41
|
-
# MRI has :symbolize_keys, but JRuby does not. Not adding it at the
|
42
|
-
# moment.
|
43
|
-
Symbolizer.symbolize(JSON.parse(payload))
|
44
|
-
end
|
45
|
-
|
46
|
-
def self.dump(payload)
|
47
|
-
JSON.dump(serialize_time_objects(payload))
|
48
|
-
end
|
49
|
-
|
50
|
-
def self.serialize_time_objects(object)
|
51
|
-
if object.is_a?(Hash)
|
52
|
-
object.reduce({}) do |hash, (key, value)|
|
53
|
-
hash.merge(key => serialize_time_objects(value))
|
54
|
-
end
|
55
|
-
elsif object.is_a?(Array)
|
56
|
-
object.map(&method(:serialize_time_objects))
|
57
|
-
elsif object.is_a?(Time) || object.is_a?(Date)
|
58
|
-
object.iso8601
|
59
|
-
else
|
60
|
-
object
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|
64
34
|
end
|
65
35
|
end
|
@@ -10,17 +10,24 @@ class Freddy
|
|
10
10
|
@exchange = channel.default_exchange
|
11
11
|
end
|
12
12
|
|
13
|
-
def produce(
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
def produce(routing_key, payload, properties)
|
14
|
+
span = Tracing.span_for_produce(
|
15
|
+
@exchange,
|
16
|
+
routing_key,
|
17
|
+
payload,
|
18
|
+
correlation_id: properties[:correlation_id]
|
19
|
+
)
|
17
20
|
|
18
21
|
properties = properties.merge(
|
19
|
-
routing_key:
|
22
|
+
routing_key: routing_key,
|
20
23
|
content_type: CONTENT_TYPE
|
21
24
|
)
|
25
|
+
Tracing.inject_tracing_information_to_properties!(properties)
|
22
26
|
|
23
27
|
@exchange.publish Payload.dump(payload), properties
|
28
|
+
ensure
|
29
|
+
# We won't wait for a reply. Just finish the span immediately.
|
30
|
+
span.finish
|
24
31
|
end
|
25
32
|
end
|
26
33
|
end
|
@@ -11,20 +11,19 @@ class Freddy
|
|
11
11
|
@topic_exchange = channel.topic Freddy::FREDDY_TOPIC_EXCHANGE_NAME
|
12
12
|
end
|
13
13
|
|
14
|
-
def produce(
|
15
|
-
span =
|
16
|
-
tags: {
|
17
|
-
'message_bus.destination' => destination,
|
18
|
-
'component' => 'freddy',
|
19
|
-
'span.kind' => 'producer' # Message Bus
|
20
|
-
})
|
14
|
+
def produce(routing_key, payload, properties)
|
15
|
+
span = Tracing.span_for_produce(@topic_exchange, routing_key, payload)
|
21
16
|
|
22
17
|
properties = properties.merge(
|
23
|
-
routing_key:
|
18
|
+
routing_key: routing_key,
|
24
19
|
content_type: CONTENT_TYPE
|
25
20
|
)
|
26
|
-
|
27
|
-
|
21
|
+
Tracing.inject_tracing_information_to_properties!(properties)
|
22
|
+
|
23
|
+
json_payload = Freddy::Encoding.compress(
|
24
|
+
Payload.dump(payload),
|
25
|
+
properties[:content_encoding]
|
26
|
+
)
|
28
27
|
|
29
28
|
# Connection adapters handle thread safety for #publish themselves. No
|
30
29
|
# need to lock these.
|
@@ -33,8 +32,6 @@ class Freddy
|
|
33
32
|
ensure
|
34
33
|
# We don't know how many listeners there are and we do not know when
|
35
34
|
# this message gets processed. Instead we close the span immediately.
|
36
|
-
# Listeners should use FollowsFrom to add trace information.
|
37
|
-
# https://github.com/opentracing/specification/blob/master/specification.md
|
38
35
|
span.finish
|
39
36
|
end
|
40
37
|
end
|
@@ -12,7 +12,6 @@ class Freddy
|
|
12
12
|
@request_manager = RequestManager.new(@logger)
|
13
13
|
|
14
14
|
@exchange = @channel.default_exchange
|
15
|
-
@topic_exchange = @channel.topic Freddy::FREDDY_TOPIC_EXCHANGE_NAME
|
16
15
|
|
17
16
|
@channel.on_no_route do |correlation_id|
|
18
17
|
@request_manager.no_route(correlation_id)
|
@@ -24,43 +23,37 @@ class Freddy
|
|
24
23
|
@response_consumer.consume(@channel, @response_queue, &method(:handle_response))
|
25
24
|
end
|
26
25
|
|
27
|
-
def produce(
|
26
|
+
def produce(routing_key, payload, timeout_in_seconds:, delete_on_timeout:, **properties)
|
28
27
|
correlation_id = SecureRandom.uuid
|
29
28
|
|
30
|
-
span =
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
'message_bus.response_queue' => @response_queue.name,
|
37
|
-
'message_bus.correlation_id' => correlation_id,
|
38
|
-
'freddy.timeout_in_seconds' => timeout_in_seconds
|
39
|
-
})
|
29
|
+
span = Tracing.span_for_produce(
|
30
|
+
@exchange,
|
31
|
+
routing_key,
|
32
|
+
payload,
|
33
|
+
correlation_id: correlation_id, timeout_in_seconds: timeout_in_seconds
|
34
|
+
)
|
40
35
|
|
41
36
|
container = SyncResponseContainer.new(
|
42
|
-
on_timeout(correlation_id,
|
37
|
+
on_timeout(correlation_id, routing_key, timeout_in_seconds, span)
|
43
38
|
)
|
44
39
|
|
45
40
|
@request_manager.store(correlation_id,
|
46
41
|
callback: container,
|
47
42
|
span: span,
|
48
|
-
destination:
|
43
|
+
destination: routing_key)
|
49
44
|
|
50
45
|
properties[:expiration] = (timeout_in_seconds * 1000).to_i if delete_on_timeout
|
51
46
|
|
52
47
|
properties = properties.merge(
|
53
|
-
routing_key:
|
48
|
+
routing_key: routing_key, content_type: CONTENT_TYPE,
|
54
49
|
correlation_id: correlation_id, reply_to: @response_queue.name,
|
55
50
|
mandatory: true, type: 'request'
|
56
51
|
)
|
57
|
-
|
58
|
-
json_payload = Payload.dump(payload)
|
52
|
+
Tracing.inject_tracing_information_to_properties!(properties)
|
59
53
|
|
60
54
|
# Connection adapters handle thread safety for #publish themselves. No
|
61
|
-
# need to lock
|
62
|
-
@
|
63
|
-
@exchange.publish json_payload, properties.dup
|
55
|
+
# need to lock this.
|
56
|
+
@exchange.publish Payload.dump(payload), properties.dup
|
64
57
|
|
65
58
|
container.wait_for_response(timeout_in_seconds)
|
66
59
|
end
|
@@ -82,28 +75,21 @@ class Freddy
|
|
82
75
|
"with correlation_id #{delivery.correlation_id}"
|
83
76
|
request[:callback].call(delivery.payload, delivery)
|
84
77
|
rescue InvalidRequestError => e
|
85
|
-
request[:span].
|
86
|
-
request[:span].
|
87
|
-
event: 'invalid request',
|
88
|
-
message: e.message,
|
89
|
-
'error.object': e
|
90
|
-
)
|
78
|
+
request[:span].record_exception(e)
|
79
|
+
request[:span].status = OpenTelemetry::Trace::Status.error
|
91
80
|
raise e
|
92
81
|
ensure
|
93
82
|
request[:span].finish
|
94
83
|
end
|
95
84
|
|
96
|
-
def on_timeout(correlation_id,
|
85
|
+
def on_timeout(correlation_id, routing_key, timeout_in_seconds, span)
|
97
86
|
proc do
|
98
|
-
@logger.warn "Request timed out waiting response from #{
|
87
|
+
@logger.warn "Request timed out waiting response from #{routing_key}"\
|
99
88
|
", correlation id #{correlation_id}, timeout #{timeout_in_seconds}s"
|
100
89
|
|
101
90
|
@request_manager.delete(correlation_id)
|
102
|
-
span.
|
103
|
-
span.
|
104
|
-
event: 'timed out',
|
105
|
-
message: "Timed out waiting response from #{destination}"
|
106
|
-
)
|
91
|
+
span.add_event('timeout')
|
92
|
+
span.status = OpenTelemetry::Trace::Status.error("Timed out waiting response from #{routing_key}")
|
107
93
|
span.finish
|
108
94
|
end
|
109
95
|
end
|
data/lib/freddy/producers.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
class Freddy
|
4
4
|
class RequestManager
|
5
5
|
def initialize(logger)
|
6
|
-
@requests =
|
6
|
+
@requests = {}
|
7
7
|
@logger = logger
|
8
8
|
end
|
9
9
|
|
@@ -22,13 +22,5 @@ class Freddy
|
|
22
22
|
def delete(correlation_id)
|
23
23
|
@requests.delete(correlation_id)
|
24
24
|
end
|
25
|
-
|
26
|
-
class ConcurrentHash < Hash
|
27
|
-
# CRuby hash does not need any locks. Only adding when using JRuby.
|
28
|
-
if RUBY_PLATFORM == 'java'
|
29
|
-
require 'jruby/synchronized'
|
30
|
-
include JRuby::Synchronized
|
31
|
-
end
|
32
|
-
end
|
33
25
|
end
|
34
26
|
end
|