peatio 0.6.3 → 2.4.0.pre.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Peatio
2
4
  class Logger
3
5
  class << self
@@ -6,30 +8,30 @@ module Peatio
6
8
  end
7
9
 
8
10
  def level
9
- (ENV["LOG_LEVEL"] || "debug").downcase.to_sym
11
+ (ENV["LOG_LEVEL"] || "info").downcase.to_sym
10
12
  end
11
13
 
12
- def debug(progname = nil, &block)
14
+ def debug(progname=nil, &block)
13
15
  logger.debug(progname, &block)
14
16
  end
15
17
 
16
- def info(progname = nil, &block)
18
+ def info(progname=nil, &block)
17
19
  logger.info(progname, &block)
18
20
  end
19
21
 
20
- def warn(progname = nil, &block)
22
+ def warn(progname=nil, &block)
21
23
  logger.warn(progname, &block)
22
24
  end
23
25
 
24
- def error(progname = nil, &block)
26
+ def error(progname=nil, &block)
25
27
  logger.error(progname, &block)
26
28
  end
27
29
 
28
- def fatal(progname = nil, &block)
30
+ def fatal(progname=nil, &block)
29
31
  logger.fatal(progname, &block)
30
32
  end
31
33
 
32
- def unknown(progname = nil, &block)
34
+ def unknown(progname=nil, &block)
33
35
  logger.unknown(progname, &block)
34
36
  end
35
37
  end
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Peatio::Metrics
4
+ class Server
5
+ def self.app(registry)
6
+ Rack::Builder.new do |builder|
7
+ builder.use Rack::CommonLogger
8
+ builder.use Rack::ShowExceptions
9
+ builder.use Rack::Deflater
10
+ builder.use Prometheus::Middleware::Exporter, registry: registry
11
+ builder.run ->(_) { [404, {"Content-Type" => "text/html"}, ["Not found\n"]] }
12
+ end
13
+ end
14
+ end
15
+ end
@@ -1,30 +1,51 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Peatio::MQ
2
4
  class Client
3
5
  class << self
4
- attr_accessor :channel, :connection
6
+ attr_accessor :connection
5
7
 
6
- def new
7
- @options = {
8
- host: ENV["RABBITMQ_HOST"] || "0.0.0.0",
9
- port: ENV["RABBITMQ_PORT"] || "5672",
8
+ def connect!
9
+ options = {
10
+ host: ENV["RABBITMQ_HOST"] || "0.0.0.0",
11
+ port: ENV["RABBITMQ_PORT"] || "5672",
10
12
  username: ENV["RABBITMQ_USER"],
11
13
  password: ENV["RABBITMQ_PASSWORD"],
12
14
  }
13
- end
14
-
15
- def connect!
16
- @connection = Bunny.new(@options)
15
+ @connection = Bunny.new(options)
17
16
  @connection.start
18
17
  end
19
18
 
20
- def create_channel!
21
- @channel = @connection.create_channel
22
- end
23
-
24
19
  def disconnect
25
20
  @connection.close
26
- yield if block_given?
27
21
  end
28
22
  end
23
+
24
+ def initialize
25
+ Client.connect! unless Peatio::MQ::Client.connection
26
+ @channel = Client.connection.create_channel
27
+ @exchanges = {}
28
+ end
29
+
30
+ def exchange(name, type="topic")
31
+ @exchanges[name] ||= @channel.exchange(name, type: type)
32
+ end
33
+
34
+ def publish(ex_name, type, id, event, payload)
35
+ routing_key = [type, id, event].join(".")
36
+ serialized_data = JSON.dump(payload)
37
+ exchange(ex_name).publish(serialized_data, routing_key: routing_key)
38
+ Peatio::Logger.debug { "published event to #{routing_key} " }
39
+ end
40
+
41
+ def subscribe(ex_name, &callback)
42
+ suffix = "#{Socket.gethostname.split(/-/).last}#{Random.rand(10_000)}"
43
+ queue_name = "ranger.#{suffix}"
44
+
45
+ @channel
46
+ .queue(queue_name, durable: false, auto_delete: true)
47
+ .bind(exchange(ex_name), routing_key: "#").subscribe(&callback)
48
+ Peatio::Logger.info "Subscribed to exchange #{ex_name}"
49
+ end
29
50
  end
30
51
  end
@@ -0,0 +1,110 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Peatio::Ranger
4
+ class Connection
5
+ attr_reader :socket, :user, :authorized, :streams, :logger, :id
6
+
7
+ def initialize(router, socket, logger)
8
+ @id = SecureRandom.hex(10)
9
+ @router = router
10
+ @socket = socket
11
+ @logger = logger
12
+ @user = nil
13
+ @authorized = false
14
+ @streams = {}
15
+ end
16
+
17
+ def inspect
18
+ if authorized
19
+ "<Connection id=#{id} user=#{user}>"
20
+ else
21
+ "<Connection id=#{id}>"
22
+ end
23
+ end
24
+
25
+ def send_raw(payload)
26
+ logger.debug { "sending to user #{user.inspect} payload: #{payload}" }
27
+ @socket.send(payload)
28
+ end
29
+
30
+ def send(method, data)
31
+ payload = JSON.dump(method => data)
32
+ send_raw(payload)
33
+ end
34
+
35
+ def authenticate(authenticator, jwt)
36
+ payload = {}
37
+ authorized = false
38
+ begin
39
+ payload = authenticator.authenticate!(jwt)
40
+ authorized = true
41
+ rescue Peatio::Auth::Error => e
42
+ logger.warn e.message
43
+ end
44
+ [authorized, payload]
45
+ end
46
+
47
+ def subscribe(subscribed_streams)
48
+ raise "Streams must be an array of strings" unless subscribed_streams.is_a?(Array)
49
+
50
+ subscribed_streams.each do |stream|
51
+ stream = stream.to_s
52
+ next if stream.empty?
53
+
54
+ unless @streams[stream]
55
+ @streams[stream] = true
56
+ @router.on_subscribe(self, stream)
57
+ end
58
+ end
59
+ send(:success, message: "subscribed", streams: streams.keys)
60
+ end
61
+
62
+ def unsubscribe(unsubscribed_streams)
63
+ raise "Streams must be an array of strings" unless unsubscribed_streams.is_a?(Array)
64
+
65
+ unsubscribed_streams.each do |stream|
66
+ stream = stream.to_s
67
+ next if stream.empty?
68
+
69
+ if @streams[stream]
70
+ @streams.delete(stream)
71
+ @router.on_unsubscribe(self, stream)
72
+ end
73
+ end
74
+ send(:success, message: "unsubscribed", streams: streams.keys)
75
+ end
76
+
77
+ def handle(msg)
78
+ return if msg.to_s.empty?
79
+
80
+ data = JSON.parse(msg)
81
+ case data["event"]
82
+ when "subscribe"
83
+ subscribe(data["streams"])
84
+ when "unsubscribe"
85
+ unsubscribe(data["streams"])
86
+ end
87
+ rescue JSON::ParserError => e
88
+ logger.debug { "#{e}, msg: `#{msg}`" }
89
+ end
90
+
91
+ def handshake(authenticator, hs)
92
+ query = URI.decode_www_form(hs.query_string)
93
+ subscribe(query.map {|item| item.last if item.first == "stream" })
94
+ logger.debug "WebSocket connection openned"
95
+ headers = hs.headers_downcased
96
+ return unless headers.key?("authorization")
97
+
98
+ authorized, payload = authenticate(authenticator, headers["authorization"])
99
+
100
+ if !authorized
101
+ logger.debug "Authentication failed for UID:#{payload[:uid]}"
102
+ raise EM::WebSocket::HandshakeError, "Authorization failed"
103
+ else
104
+ @user = payload[:uid]
105
+ @authorized = true
106
+ logger.debug "User #{@user} authenticated #{@streams}"
107
+ end
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Peatio::Ranger
4
+ class Events
5
+ def self.publish(type, id, event, payload, opts={})
6
+ ex_name = opts[:ex_name] || "peatio.events.ranger"
7
+ @client ||= Peatio::MQ::Client.new
8
+ @client.publish(ex_name, type, id, event, payload)
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,234 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Peatio::Ranger
4
+ class Router
5
+ attr_reader :connections
6
+ attr_reader :connections_by_userid
7
+ attr_reader :streams_sockets
8
+ attr_reader :logger
9
+
10
+ class ConnectionArray < Array
11
+ def delete(connection)
12
+ delete_if do |c|
13
+ c.id == connection.id
14
+ end
15
+ end
16
+ end
17
+
18
+ def initialize(prometheus=nil)
19
+ @connections = {}
20
+ @connections_by_userid = {}
21
+ @streams_sockets = {}
22
+ @logger = Peatio::Logger.logger
23
+ @stores = {}
24
+ init_metrics(prometheus)
25
+ end
26
+
27
+ def init_metrics(prometheus)
28
+ return unless prometheus
29
+
30
+ @prometheus = prometheus
31
+ @metric_connections_total = @prometheus.counter(
32
+ :ranger_connections_total,
33
+ docstring: "Total number of connections to ranger from the start",
34
+ labels: [:auth]
35
+ )
36
+ @metric_connections_current = @prometheus.gauge(
37
+ :ranger_connections_current,
38
+ docstring: "Current number of connections to ranger",
39
+ labels: [:auth]
40
+ )
41
+ @metric_subscriptions_current = @prometheus.gauge(
42
+ :ranger_subscriptions_current,
43
+ docstring: "Current number of streams subscriptions to ranger",
44
+ labels: [:stream]
45
+ )
46
+ end
47
+
48
+ def snapshot?(stream)
49
+ stream.end_with?("-snap")
50
+ end
51
+
52
+ def increment?(stream)
53
+ stream.end_with?("-inc")
54
+ end
55
+
56
+ def storekey(stream)
57
+ stream.gsub(/-(snap|inc)$/, "")
58
+ end
59
+
60
+ def stats
61
+ [
62
+ "==== Metrics ====",
63
+ "ranger_connections_total{auth=\"public\"}: %d" % [@metric_connections_total.get(labels: {auth: "public"})],
64
+ "ranger_connections_total{auth=\"private\"}: %d" % [@metric_connections_total.get(labels: {auth: "private"})],
65
+ "ranger_connections_current{auth=\"public\"}: %d" % [@metric_connections_current.get(labels: {auth: "public"})],
66
+ "ranger_connections_current{auth=\"private\"}: %d" % [@metric_connections_current.get(labels: {auth: "private"})],
67
+ "ranger_subscriptions_current: %d" % [compute_streams_subscriptions()],
68
+ "ranger_streams_kinds: %d" % [compute_streams_kinds()],
69
+ ].join("\n")
70
+ end
71
+
72
+ def debug
73
+ [
74
+ "==== Debug ====",
75
+ "connections: %s" % [@connections.inspect],
76
+ "connections_by_userid: %s" % [@connections_by_userid],
77
+ "streams_sockets: %s" % [@streams_sockets],
78
+ ].join("\n")
79
+ end
80
+
81
+ def compute_connections_all
82
+ @connections.size
83
+ end
84
+
85
+ def compute_connections_private
86
+ @connections_by_userid.each_value.map(&:size).reduce(0, :+)
87
+ end
88
+
89
+ def compute_stream_subscriptions(stream)
90
+ @streams_sockets[stream]&.size || 0
91
+ end
92
+
93
+ def compute_streams_subscriptions
94
+ @streams_sockets.each_value.map(&:size).reduce(0, :+)
95
+ end
96
+
97
+ def compute_streams_kinds
98
+ @streams_sockets.size
99
+ end
100
+
101
+ def sanity_check_metrics_connections
102
+ return unless @metric_connections_current
103
+
104
+ connections_current_all = @metric_connections_current.values.values.reduce(0, :+)
105
+ return if connections_current_all == compute_connections_all()
106
+
107
+ logger.warn "slip detected in metric_connections_current, recalculating"
108
+ connections_current_private = compute_connections_private()
109
+ @metric_connections_current.set(connections_current_private, labels: {auth: "private"})
110
+ @metric_connections_current.set(compute_connections_all() - connections_current_private, labels: {auth: "public"})
111
+ end
112
+
113
+ def on_connection_open(connection)
114
+ @connections[connection.id] = connection
115
+ unless connection.authorized
116
+ @metric_connections_current&.increment(labels: {auth: "public"})
117
+ @metric_connections_total&.increment(labels: {auth: "public"})
118
+ return
119
+ end
120
+ @metric_connections_current&.increment(labels: {auth: "private"})
121
+ @metric_connections_total&.increment(labels: {auth: "private"})
122
+
123
+ @connections_by_userid[connection.user] ||= ConnectionArray.new
124
+ @connections_by_userid[connection.user] << connection
125
+ end
126
+
127
+ def on_connection_close(connection)
128
+ @connections.delete(connection.id)
129
+ connection.streams.keys.each do |stream|
130
+ on_unsubscribe(connection, stream)
131
+ end
132
+
133
+ unless connection.authorized
134
+ @metric_connections_current&.decrement(labels: {auth: "public"})
135
+ sanity_check_metrics_connections
136
+ return
137
+ end
138
+ @metric_connections_current&.decrement(labels: {auth: "private"})
139
+
140
+ @connections_by_userid[connection.user].delete(connection)
141
+ @connections_by_userid.delete(connection.user) \
142
+ if @connections_by_userid[connection.user].empty?
143
+ sanity_check_metrics_connections
144
+ end
145
+
146
+ def on_subscribe(connection, stream)
147
+ @streams_sockets[stream] ||= ConnectionArray.new
148
+ @streams_sockets[stream] << connection
149
+ send_snapshot_and_increments(connection, storekey(stream)) if increment?(stream)
150
+ @metric_subscriptions_current&.set(compute_stream_subscriptions(stream), labels: {stream: stream})
151
+ end
152
+
153
+ def send_snapshot_and_increments(connection, key)
154
+ return unless @stores[key]
155
+ return unless @stores[key][:snapshot]
156
+
157
+ connection.send_raw(@stores[key][:snapshot])
158
+ @stores[key][:increments]&.each {|inc| connection.send_raw(inc) }
159
+ end
160
+
161
+ def on_unsubscribe(connection, stream)
162
+ return unless @streams_sockets[stream]
163
+
164
+ @streams_sockets[stream].delete(connection)
165
+ @streams_sockets.delete(stream) if @streams_sockets[stream].empty?
166
+ @metric_subscriptions_current&.set(compute_stream_subscriptions(stream), labels: {stream: stream})
167
+ end
168
+
169
+ def send_private_message(user_id, event, payload_decoded)
170
+ Array(@connections_by_userid[user_id]).each do |connection|
171
+ connection.send(event, payload_decoded) if connection.streams.include?(event)
172
+ end
173
+ end
174
+
175
+ def send_public_message(stream, raw_message)
176
+ Array(@streams_sockets[stream]).each do |connection|
177
+ connection.send_raw(raw_message)
178
+ end
179
+ end
180
+
181
+ #
182
+ # routing key format: type.id.event
183
+ # * `type` can be *public* or *private*
184
+ # * `id` can be user id or market id
185
+ # * `event` is the event identifier, ex: order_completed, trade, ...
186
+ #
187
+ def on_message(delivery_info, _metadata, payload)
188
+ routing_key = delivery_info.routing_key
189
+ if routing_key.count(".") != 2
190
+ logger.error { "invalid routing key from amqp: #{routing_key}" }
191
+ return
192
+ end
193
+
194
+ type, id, event = routing_key.split(".")
195
+ payload_decoded = JSON.parse(payload)
196
+
197
+ if type == "private"
198
+ send_private_message(id, event, payload_decoded)
199
+ return
200
+ end
201
+
202
+ stream = [id, event].join(".")
203
+ message = JSON.dump(stream => payload_decoded)
204
+
205
+ if snapshot?(event)
206
+ key = storekey(stream)
207
+
208
+ unless @stores[key]
209
+ # Send the snapshot to subscribers of -inc stream if there were no snapshot before
210
+ send_public_message("#{key}-inc", message)
211
+ end
212
+
213
+ @stores[key] = {
214
+ snapshot: message,
215
+ increments: [],
216
+ }
217
+ return
218
+ end
219
+
220
+ if increment?(event)
221
+ key = storekey(stream)
222
+
223
+ unless @stores[key]
224
+ logger.warn { "Discard increment received before snapshot for store:#{key}" }
225
+ return
226
+ end
227
+
228
+ @stores[key][:increments] << message
229
+ end
230
+
231
+ send_public_message(stream, message)
232
+ end
233
+ end
234
+ end