peatio 0.6.3 → 2.4.0.pre.alpha

data/lib/peatio/logger.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Peatio
   class Logger
     class << self
@@ -6,30 +8,30 @@ module Peatio
       end
 
       def level
-        (ENV["LOG_LEVEL"] || "debug").downcase.to_sym
+        (ENV["LOG_LEVEL"] || "info").downcase.to_sym
       end
 
-      def debug(progname = nil, &block)
+      def debug(progname=nil, &block)
         logger.debug(progname, &block)
       end
 
-      def info(progname = nil, &block)
+      def info(progname=nil, &block)
         logger.info(progname, &block)
       end
 
-      def warn(progname = nil, &block)
+      def warn(progname=nil, &block)
         logger.warn(progname, &block)
       end
 
-      def error(progname = nil, &block)
+      def error(progname=nil, &block)
         logger.error(progname, &block)
       end
 
-      def fatal(progname = nil, &block)
+      def fatal(progname=nil, &block)
         logger.fatal(progname, &block)
       end
 
-      def unknown(progname = nil, &block)
+      def unknown(progname=nil, &block)
         logger.unknown(progname, &block)
       end
     end

data/lib/peatio/metrics/server.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Peatio::Metrics
+  class Server
+    def self.app(registry)
+      Rack::Builder.new do |builder|
+        builder.use Rack::CommonLogger
+        builder.use Rack::ShowExceptions
+        builder.use Rack::Deflater
+        builder.use Prometheus::Middleware::Exporter, registry: registry
+        builder.run ->(_) { [404, {"Content-Type" => "text/html"}, ["Not found\n"]] }
+      end
+    end
+  end
+end

data/lib/peatio/mq/client.rb

@@ -1,30 +1,51 @@
+# frozen_string_literal: true
+
 module Peatio::MQ
   class Client
     class << self
-      attr_accessor :channel, :connection
+      attr_accessor :connection
 
-      def new
-        @options = {
-          host: ENV["RABBITMQ_HOST"] || "0.0.0.0",
-          port: ENV["RABBITMQ_PORT"] || "5672",
+      def connect!
+        options = {
+          host:     ENV["RABBITMQ_HOST"] || "0.0.0.0",
+          port:     ENV["RABBITMQ_PORT"] || "5672",
           username: ENV["RABBITMQ_USER"],
           password: ENV["RABBITMQ_PASSWORD"],
         }
-      end
-
-      def connect!
-        @connection = Bunny.new(@options)
+        @connection = Bunny.new(options)
         @connection.start
       end
 
-      def create_channel!
-        @channel = @connection.create_channel
-      end
-
       def disconnect
         @connection.close
-        yield if block_given?
       end
     end
+
+    def initialize
+      Client.connect! unless Peatio::MQ::Client.connection
+      @channel = Client.connection.create_channel
+      @exchanges = {}
+    end
+
+    def exchange(name, type="topic")
+      @exchanges[name] ||= @channel.exchange(name, type: type)
+    end
+
+    def publish(ex_name, type, id, event, payload)
+      routing_key = [type, id, event].join(".")
+      serialized_data = JSON.dump(payload)
+      exchange(ex_name).publish(serialized_data, routing_key: routing_key)
+      Peatio::Logger.debug { "published event to #{routing_key} " }
+    end
+
+    def subscribe(ex_name, &callback)
+      suffix = "#{Socket.gethostname.split(/-/).last}#{Random.rand(10_000)}"
+      queue_name = "ranger.#{suffix}"
+
+      @channel
+        .queue(queue_name, durable: false, auto_delete: true)
+        .bind(exchange(ex_name), routing_key: "#").subscribe(&callback)
+      Peatio::Logger.info "Subscribed to exchange #{ex_name}"
+    end
   end
 end

data/lib/peatio/ranger/connection.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Peatio::Ranger
+  class Connection
+    attr_reader :socket, :user, :authorized, :streams, :logger, :id
+
+    def initialize(router, socket, logger)
+      @id = SecureRandom.hex(10)
+      @router = router
+      @socket = socket
+      @logger = logger
+      @user = nil
+      @authorized = false
+      @streams = {}
+    end
+
+    def inspect
+      if authorized
+        "<Connection id=#{id} user=#{user}>"
+      else
+        "<Connection id=#{id}>"
+      end
+    end
+
+    def send_raw(payload)
+      logger.debug { "sending to user #{user.inspect} payload: #{payload}" }
+      @socket.send(payload)
+    end
+
+    def send(method, data)
+      payload = JSON.dump(method => data)
+      send_raw(payload)
+    end
+
+    def authenticate(authenticator, jwt)
+      payload = {}
+      authorized = false
+      begin
+        payload = authenticator.authenticate!(jwt)
+        authorized = true
+      rescue Peatio::Auth::Error => e
+        logger.warn e.message
+      end
+      [authorized, payload]
+    end
+
+    def subscribe(subscribed_streams)
+      raise "Streams must be an array of strings" unless subscribed_streams.is_a?(Array)
+
+      subscribed_streams.each do |stream|
+        stream = stream.to_s
+        next if stream.empty?
+
+        unless @streams[stream]
+          @streams[stream] = true
+          @router.on_subscribe(self, stream)
+        end
+      end
+      send(:success, message: "subscribed", streams: streams.keys)
+    end
+
+    def unsubscribe(unsubscribed_streams)
+      raise "Streams must be an array of strings" unless unsubscribed_streams.is_a?(Array)
+
+      unsubscribed_streams.each do |stream|
+        stream = stream.to_s
+        next if stream.empty?
+
+        if @streams[stream]
+          @streams.delete(stream)
+          @router.on_unsubscribe(self, stream)
+        end
+      end
+      send(:success, message: "unsubscribed", streams: streams.keys)
+    end
+
+    def handle(msg)
+      return if msg.to_s.empty?
+
+      data = JSON.parse(msg)
+      case data["event"]
+      when "subscribe"
+        subscribe(data["streams"])
+      when "unsubscribe"
+        unsubscribe(data["streams"])
+      end
+    rescue JSON::ParserError => e
+      logger.debug { "#{e}, msg: `#{msg}`" }
+    end
+
+    def handshake(authenticator, hs)
+      query = URI.decode_www_form(hs.query_string)
+      subscribe(query.map {|item| item.last if item.first == "stream" })
+      logger.debug "WebSocket connection openned"
+      headers = hs.headers_downcased
+      return unless headers.key?("authorization")
+
+      authorized, payload = authenticate(authenticator, headers["authorization"])
+
+      if !authorized
+        logger.debug "Authentication failed for UID:#{payload[:uid]}"
+        raise EM::WebSocket::HandshakeError, "Authorization failed"
+      else
+        @user = payload[:uid]
+        @authorized = true
+        logger.debug "User #{@user} authenticated #{@streams}"
+      end
+    end
+  end
+end

data/lib/peatio/ranger/events.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Peatio::Ranger
+  class Events
+    def self.publish(type, id, event, payload, opts={})
+      ex_name = opts[:ex_name] || "peatio.events.ranger"
+      @client ||= Peatio::MQ::Client.new
+      @client.publish(ex_name, type, id, event, payload)
+    end
+  end
+end

data/lib/peatio/ranger/router.rb

@@ -0,0 +1,234 @@
+# frozen_string_literal: true
+
+module Peatio::Ranger
+  class Router
+    attr_reader :connections
+    attr_reader :connections_by_userid
+    attr_reader :streams_sockets
+    attr_reader :logger
+
+    class ConnectionArray < Array
+      def delete(connection)
+        delete_if do |c|
+          c.id == connection.id
+        end
+      end
+    end
+
+    def initialize(prometheus=nil)
+      @connections = {}
+      @connections_by_userid = {}
+      @streams_sockets = {}
+      @logger = Peatio::Logger.logger
+      @stores = {}
+      init_metrics(prometheus)
+    end
+
+    def init_metrics(prometheus)
+      return unless prometheus
+
+      @prometheus = prometheus
+      @metric_connections_total = @prometheus.counter(
+        :ranger_connections_total,
+        docstring: "Total number of connections to ranger from the start",
+        labels:    [:auth]
+      )
+      @metric_connections_current = @prometheus.gauge(
+        :ranger_connections_current,
+        docstring: "Current number of connections to ranger",
+        labels:    [:auth]
+      )
+      @metric_subscriptions_current = @prometheus.gauge(
+        :ranger_subscriptions_current,
+        docstring: "Current number of streams subscriptions to ranger",
+        labels:    [:stream]
+      )
+    end
+
+    def snapshot?(stream)
+      stream.end_with?("-snap")
+    end
+
+    def increment?(stream)
+      stream.end_with?("-inc")
+    end
+
+    def storekey(stream)
+      stream.gsub(/-(snap|inc)$/, "")
+    end
+
+    def stats
+      [
+        "==== Metrics ====",
+        "ranger_connections_total{auth=\"public\"}: %d" % [@metric_connections_total.get(labels: {auth: "public"})],
+        "ranger_connections_total{auth=\"private\"}: %d" % [@metric_connections_total.get(labels: {auth: "private"})],
+        "ranger_connections_current{auth=\"public\"}: %d" % [@metric_connections_current.get(labels: {auth: "public"})],
+        "ranger_connections_current{auth=\"private\"}: %d" % [@metric_connections_current.get(labels: {auth: "private"})],
+        "ranger_subscriptions_current: %d" % [compute_streams_subscriptions()],
+        "ranger_streams_kinds: %d" % [compute_streams_kinds()],
+      ].join("\n")
+    end
+
+    def debug
+      [
+        "==== Debug ====",
+        "connections: %s" % [@connections.inspect],
+        "connections_by_userid: %s" % [@connections_by_userid],
+        "streams_sockets: %s" % [@streams_sockets],
+      ].join("\n")
+    end
+
+    def compute_connections_all
+      @connections.size
+    end
+
+    def compute_connections_private
+      @connections_by_userid.each_value.map(&:size).reduce(0, :+)
+    end
+
+    def compute_stream_subscriptions(stream)
+      @streams_sockets[stream]&.size || 0
+    end
+
+    def compute_streams_subscriptions
+      @streams_sockets.each_value.map(&:size).reduce(0, :+)
+    end
+
+    def compute_streams_kinds
+      @streams_sockets.size
+    end
+
+    def sanity_check_metrics_connections
+      return unless @metric_connections_current
+
+      connections_current_all = @metric_connections_current.values.values.reduce(0, :+)
+      return if connections_current_all == compute_connections_all()
+
+      logger.warn "slip detected in metric_connections_current, recalculating"
+      connections_current_private = compute_connections_private()
+      @metric_connections_current.set(connections_current_private, labels: {auth: "private"})
+      @metric_connections_current.set(compute_connections_all() - connections_current_private, labels: {auth: "public"})
+    end
+
+    def on_connection_open(connection)
+      @connections[connection.id] = connection
+      unless connection.authorized
+        @metric_connections_current&.increment(labels: {auth: "public"})
+        @metric_connections_total&.increment(labels: {auth: "public"})
+        return
+      end
+      @metric_connections_current&.increment(labels: {auth: "private"})
+      @metric_connections_total&.increment(labels: {auth: "private"})
+
+      @connections_by_userid[connection.user] ||= ConnectionArray.new
+      @connections_by_userid[connection.user] << connection
+    end
+
+    def on_connection_close(connection)
+      @connections.delete(connection.id)
+      connection.streams.keys.each do |stream|
+        on_unsubscribe(connection, stream)
+      end
+
+      unless connection.authorized
+        @metric_connections_current&.decrement(labels: {auth: "public"})
+        sanity_check_metrics_connections
+        return
+      end
+      @metric_connections_current&.decrement(labels: {auth: "private"})
+
+      @connections_by_userid[connection.user].delete(connection)
+      @connections_by_userid.delete(connection.user) \
+        if @connections_by_userid[connection.user].empty?
+      sanity_check_metrics_connections
+    end
+
+    def on_subscribe(connection, stream)
+      @streams_sockets[stream] ||= ConnectionArray.new
+      @streams_sockets[stream] << connection
+      send_snapshot_and_increments(connection, storekey(stream)) if increment?(stream)
+      @metric_subscriptions_current&.set(compute_stream_subscriptions(stream), labels: {stream: stream})
+    end
+
+    def send_snapshot_and_increments(connection, key)
+      return unless @stores[key]
+      return unless @stores[key][:snapshot]
+
+      connection.send_raw(@stores[key][:snapshot])
+      @stores[key][:increments]&.each {|inc| connection.send_raw(inc) }
+    end
+
+    def on_unsubscribe(connection, stream)
+      return unless @streams_sockets[stream]
+
+      @streams_sockets[stream].delete(connection)
+      @streams_sockets.delete(stream) if @streams_sockets[stream].empty?
+      @metric_subscriptions_current&.set(compute_stream_subscriptions(stream), labels: {stream: stream})
+    end
+
+    def send_private_message(user_id, event, payload_decoded)
+      Array(@connections_by_userid[user_id]).each do |connection|
+        connection.send(event, payload_decoded) if connection.streams.include?(event)
+      end
+    end
+
+    def send_public_message(stream, raw_message)
+      Array(@streams_sockets[stream]).each do |connection|
+        connection.send_raw(raw_message)
+      end
+    end
+
+    #
+    # routing key format: type.id.event
+    # * `type` can be *public* or *private*
+    # * `id` can be user id or market id
+    # * `event` is the event identifier, ex: order_completed, trade, ...
+    #
+    def on_message(delivery_info, _metadata, payload)
+      routing_key = delivery_info.routing_key
+      if routing_key.count(".") != 2
+        logger.error { "invalid routing key from amqp: #{routing_key}" }
+        return
+      end
+
+      type, id, event = routing_key.split(".")
+      payload_decoded = JSON.parse(payload)
+
+      if type == "private"
+        send_private_message(id, event, payload_decoded)
+        return
+      end
+
+      stream = [id, event].join(".")
+      message = JSON.dump(stream => payload_decoded)
+
+      if snapshot?(event)
+        key = storekey(stream)
+
+        unless @stores[key]
+          # Send the snapshot to subscribers of -inc stream if there were no snapshot before
+          send_public_message("#{key}-inc", message)
+        end
+
+        @stores[key] = {
+          snapshot:   message,
+          increments: [],
+        }
+        return
+      end
+
+      if increment?(event)
+        key = storekey(stream)
+
+        unless @stores[key]
+          logger.warn { "Discard increment received before snapshot for store:#{key}" }
+          return
+        end
+
+        @stores[key][:increments] << message
+      end
+
+      send_public_message(stream, message)
+    end
+  end
+end