peatio-dao 3.1.3

data/lib/peatio/injectors/peatio_events.rb

@@ -0,0 +1,240 @@
+# frozen_string_literal: true
+
+module Peatio::Injectors
+  class PeatioEvents
+    attr_accessor :market, :market_name, :base_unit, :quote_unit, :seller_uid, :buyer_uid, :logger
+
+    def run!(exchange_name)
+      require "time"
+      @logger = Peatio::Logger.logger
+      @market = "eurusd"
+      @market_name = "EUR/USD"
+      @base_unit = "eur"
+      @quote_unit = "usd"
+      @seller_uid = 21
+      @buyer_uid = 42
+      @messages = create_messages
+      @opts = {
+        ex_name: exchange_name
+      }
+      EventMachine.run do
+        inject_message()
+      end
+    end
+
+    def inject_message()
+      if message = @messages.shift
+        type, id, event, data = message
+        Peatio::Ranger::Events.publish(type, id, event, data, @opts)
+        EM.next_tick do
+          inject_message()
+        end
+      else
+        Peatio::MQ::Client.disconnect
+        EventMachine.stop
+      end
+    end
+
+    def create_messages
+      [
+        public_tickers,
+        public_orderbook,
+        private_order,
+        private_trade_user1,
+        private_trade_user2,
+        public_trade,
+        public_orderbook_increment1,
+        public_orderbook_snapshot1,
+        public_orderbook_increment2,
+        public_orderbook_increment3,
+      ]
+    end
+
+    def created_at
+      Time.now - 600
+    end
+
+    def updated_at
+      Time.now
+    end
+
+    alias completed_at updated_at
+    alias canceled_at updated_at
+
+    def public_orderbook
+      [
+        "public",
+        market,
+        "update",
+        {
+          "asks": [
+            ["1020.0", "0.005"],
+            ["1026.0", "0.03"]
+          ],
+          "bids": [
+            ["1000.0", "0.25"],
+            ["999.0", "0.005"],
+            ["994.0", "0.005"],
+            ["1.0", "11.0"]
+          ]
+        }
+      ]
+    end
+
+    def public_orderbook_snapshot1
+      [
+        "public",
+        market,
+        "ob-snap",
+        {
+          "asks": [
+            ["1020.0", "0.005"],
+            ["1026.0", "0.03"]
+          ],
+          "bids": [
+            ["1000.0", "0.25"],
+            ["999.0", "0.005"],
+            ["994.0", "0.005"],
+            ["1.0", "11.0"]
+          ]
+        }
+      ]
+    end
+
+    def public_orderbook_increment1
+      [
+        "public",
+        market,
+        "ob-inc",
+        {
+          "asks": [
+            ["1020.0", "0.015"],
+          ],
+        }
+      ]
+    end
+
+    def public_orderbook_increment2
+      [
+        "public",
+        market,
+        "ob-inc",
+        {
+          "bids": [
+            ["1000.0", "0"],
+          ],
+        }
+      ]
+    end
+
+    def public_orderbook_increment3
+      [
+        "public",
+        market,
+        "ob-inc",
+        {
+          "bids": [
+            ["999.0", "0.001"],
+          ],
+        }
+      ]
+    end
+
+    def public_tickers
+      [
+        "public",
+        "global",
+        "tickers",
+        {
+          market => {
+            "name":       market_name,
+            "base_unit":  base_unit,
+            "quote_unit": quote_unit,
+            "low":        "1000.0",
+            "high":       "10000.0",
+            "last":       "1000.0",
+            "open":       1000.0,
+            "volume":     "0.0",
+            "sell":       "1020.0",
+            "buy":        "1000.0",
+            "at":         Time.now.to_i
+          }
+        }
+      ]
+    end
+
+    def private_order
+      [
+        "private",
+        "IDABC0000001",
+        "order",
+        {
+          "id":            22,
+          "at":            created_at.to_i,
+          "market":        market,
+          "kind":          "bid",
+          "price":         "1026.0",
+          "state":         "wait",
+          "volume":        "0.001",
+          "origin_volume": "0.001"
+        }
+      ]
+    end
+
+    def private_trade_user1
+      [
+        "private",
+        "IDABC0000001",
+        "trade",
+        {
+          "id":     7,
+          "kind":   "ask",
+          "at":     created_at.to_i,
+          "price":  "1020.0",
+          "volume": "0.001",
+          "ask_id": 15,
+          "bid_id": 22,
+          "market": market
+        }
+      ]
+    end
+
+    def private_trade_user2
+      [
+        "private",
+        "IDABC0000002",
+        "trade",
+        {
+          "id":     7,
+          "kind":   "bid",
+          "at":     created_at.to_i,
+          "price":  "1020.0",
+          "volume": "0.001",
+          "ask_id": 15,
+          "bid_id": 22,
+          "market": market
+        }
+      ]
+    end
+
+    def public_trade
+      [
+        "public",
+        market,
+        "trades",
+        {
+          "trades": [
+            {
+              "tid":        7,
+              "taker_type": "buy",
+              "date":       created_at.to_i,
+              "price":      "1020.0",
+              "amount":
+                            "0.001"
+            }
+          ]
+        }
+      ]
+    end
+  end
+end

data/lib/peatio/logger.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Peatio
+  class Logger
+    class << self
+      def logger
+        @logger ||= ::Logger.new(STDERR, level: level)
+      end
+
+      def level
+        (ENV["LOG_LEVEL"] || "info").downcase.to_sym
+      end
+
+      def debug(progname=nil, &block)
+        logger.debug(progname, &block)
+      end
+
+      def info(progname=nil, &block)
+        logger.info(progname, &block)
+      end
+
+      def warn(progname=nil, &block)
+        logger.warn(progname, &block)
+      end
+
+      def error(progname=nil, &block)
+        logger.error(progname, &block)
+      end
+
+      def fatal(progname=nil, &block)
+        logger.fatal(progname, &block)
+      end
+
+      def unknown(progname=nil, &block)
+        logger.unknown(progname, &block)
+      end
+    end
+  end
+end

data/lib/peatio/metrics/server.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Peatio::Metrics
+  class Server
+    def self.app(registry)
+      Rack::Builder.new do |builder|
+        builder.use Rack::CommonLogger
+        builder.use Rack::ShowExceptions
+        builder.use Rack::Deflater
+        builder.use Prometheus::Middleware::Exporter, registry: registry
+        builder.run ->(_) { [404, {"Content-Type" => "text/html"}, ["Not found\n"]] }
+      end
+    end
+  end
+end

data/lib/peatio/mq/client.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Peatio::MQ
+  class Client
+    class << self
+      attr_accessor :connection
+
+      def connect!
+        options = {
+          host:     ENV["RABBITMQ_HOST"] || "0.0.0.0",
+          port:     ENV["RABBITMQ_PORT"] || "5672",
+          username: ENV["RABBITMQ_USER"],
+          password: ENV["RABBITMQ_PASSWORD"],
+        }
+        @connection = Bunny.new(options)
+        @connection.start
+      end
+
+      def disconnect
+        @connection.close
+      end
+    end
+
+    def initialize
+      Client.connect! unless Peatio::MQ::Client.connection
+      @channel = Client.connection.create_channel
+      @exchanges = {}
+    end
+
+    def exchange(name, type="topic")
+      @exchanges[name] ||= @channel.exchange(name, type: type)
+    end
+
+    def publish(ex_name, type, id, event, payload)
+      routing_key = [type, id, event].join(".")
+      serialized_data = JSON.dump(payload)
+      exchange(ex_name).publish(serialized_data, routing_key: routing_key)
+      Peatio::Logger.debug { "published event to #{routing_key} " }
+    end
+
+    def subscribe(ex_name, &callback)
+      suffix = "#{Socket.gethostname.split(/-/).last}#{Random.rand(10_000)}"
+      queue_name = "ranger.#{suffix}"
+
+      @channel
+        .queue(queue_name, durable: false, auto_delete: true)
+        .bind(exchange(ex_name), routing_key: "#").subscribe(&callback)
+      Peatio::Logger.info "Subscribed to exchange #{ex_name}"
+    end
+  end
+end

data/lib/peatio/ranger/connection.rb

@@ -0,0 +1,117 @@
+module Peatio::Ranger
+  class Connection
+    attr_reader :socket, :user, :authorized, :streams, :logger, :id
+
+    def initialize(router, socket, logger)
+      @id = SecureRandom.hex(10)
+      @router = router
+      @socket = socket
+      @logger = logger
+      @user = nil
+      @authorized = false
+      @streams = {}
+    end
+
+    def inspect
+      if authorized
+        "<Connection id=#{id} user=#{user}>"
+      else
+        "<Connection id=#{id}>"
+      end
+    end
+
+    def send_raw(payload)
+      if user
+        logger.debug { "sending to user #{user.inspect} payload: #{payload}" }
+      else
+        logger.debug { "sending to anonymous payload: #{payload}" }
+      end
+      @socket.send(payload)
+    end
+
+    def send(method, data)
+      payload = JSON.dump(method => data)
+      send_raw(payload)
+    end
+
+    def authenticate(authenticator, jwt)
+      payload = {}
+      authorized = false
+      begin
+        payload = authenticator.authenticate!(jwt)
+        authorized = true
+      rescue Peatio::Auth::Error => e
+        logger.warn e.message
+      end
+      [authorized, payload]
+    end
+
+    def subscribe(subscribed_streams)
+      raise "Streams must be an array of strings" unless subscribed_streams.is_a?(Array)
+
+      subscribed_streams.each do |stream|
+        stream = stream.to_s
+        next if stream.empty?
+
+        unless @streams[stream]
+          @streams[stream] = true
+          @router.on_subscribe(self, stream)
+        end
+      end
+      send(:success, message: "subscribed", streams: streams.keys)
+    end
+
+    def unsubscribe(unsubscribed_streams)
+      raise "Streams must be an array of strings" unless unsubscribed_streams.is_a?(Array)
+
+      unsubscribed_streams.each do |stream|
+        stream = stream.to_s
+        next if stream.empty?
+
+        if @streams[stream]
+          @streams.delete(stream)
+          @router.on_unsubscribe(self, stream)
+        end
+      end
+      send(:success, message: "unsubscribed", streams: streams.keys)
+    end
+
+    def handle(msg)
+      return if msg.to_s.empty?
+
+      if msg =~ /^ping/
+        send_raw("pong")
+        return
+      end
+
+      data = JSON.parse(msg)
+      case data["event"]
+      when "subscribe"
+        subscribe(data["streams"])
+      when "unsubscribe"
+        unsubscribe(data["streams"])
+      end
+    rescue JSON::ParserError => e
+      logger.debug { "#{e}, msg: `#{msg}`" }
+    end
+
+    def handshake(authenticator, hs)
+      query = URI.decode_www_form(hs.query_string)
+      subscribe(query.map {|item| item.last if item.first == "stream" })
+      logger.debug "WebSocket connection openned"
+      headers = hs.headers_downcased
+      return unless headers.key?("authorization")
+
+      authorized, payload = authenticate(authenticator, headers["authorization"])
+
+      if !authorized
+        logger.debug "Authentication failed for UID:#{payload[:uid]}"
+        raise EM::WebSocket::HandshakeError, "Authorization failed"
+      else
+        @user = payload[:uid]
+        @authorized = true
+        logger.debug "User #{@user} authenticated #{@streams}"
+      end
+    end
+  end
+end

data/lib/peatio/ranger/events.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Peatio::Ranger
+  class Events
+    def self.publish(type, id, event, payload, opts={})
+      ex_name = opts[:ex_name] || "peatio.events.ranger"
+      @client ||= Peatio::MQ::Client.new
+      @client.publish(ex_name, type, id, event, payload)
+    end
+  end
+end

data/lib/peatio/ranger/router.rb

@@ -0,0 +1,234 @@
+# frozen_string_literal: true
+
+module Peatio::Ranger
+  class Router
+    attr_reader :connections
+    attr_reader :connections_by_userid
+    attr_reader :streams_sockets
+    attr_reader :logger
+
+    class ConnectionArray < Array
+      def delete(connection)
+        delete_if do |c|
+          c.id == connection.id
+        end
+      end
+    end
+
+    def initialize(prometheus=nil)
+      @connections = {}
+      @connections_by_userid = {}
+      @streams_sockets = {}
+      @logger = Peatio::Logger.logger
+      @stores = {}
+      init_metrics(prometheus)
+    end
+
+    def init_metrics(prometheus)
+      return unless prometheus
+
+      @prometheus = prometheus
+      @metric_connections_total = @prometheus.counter(
+        :ranger_connections_total,
+        docstring: "Total number of connections to ranger from the start",
+        labels:    [:auth]
+      )
+      @metric_connections_current = @prometheus.gauge(
+        :ranger_connections_current,
+        docstring: "Current number of connections to ranger",
+        labels:    [:auth]
+      )
+      @metric_subscriptions_current = @prometheus.gauge(
+        :ranger_subscriptions_current,
+        docstring: "Current number of streams subscriptions to ranger",
+        labels:    [:stream]
+      )
+    end
+
+    def snapshot?(stream)
+      stream.end_with?("-snap")
+    end
+
+    def increment?(stream)
+      stream.end_with?("-inc")
+    end
+
+    def storekey(stream)
+      stream.gsub(/-(snap|inc)$/, "")
+    end
+
+    def stats
+      [
+        "==== Metrics ====",
+        "ranger_connections_total{auth=\"public\"}: %d" % [@metric_connections_total.get(labels: {auth: "public"})],
+        "ranger_connections_total{auth=\"private\"}: %d" % [@metric_connections_total.get(labels: {auth: "private"})],
+        "ranger_connections_current{auth=\"public\"}: %d" % [@metric_connections_current.get(labels: {auth: "public"})],
+        "ranger_connections_current{auth=\"private\"}: %d" % [@metric_connections_current.get(labels: {auth: "private"})],
+        "ranger_subscriptions_current: %d" % [compute_streams_subscriptions()],
+        "ranger_streams_kinds: %d" % [compute_streams_kinds()],
+      ].join("\n")
+    end
+
+    def debug
+      [
+        "==== Debug ====",
+        "connections: %s" % [@connections.inspect],
+        "connections_by_userid: %s" % [@connections_by_userid],
+        "streams_sockets: %s" % [@streams_sockets],
+      ].join("\n")
+    end
+
+    def compute_connections_all
+      @connections.size
+    end
+
+    def compute_connections_private
+      @connections_by_userid.each_value.map(&:size).reduce(0, :+)
+    end
+
+    def compute_stream_subscriptions(stream)
+      @streams_sockets[stream]&.size || 0
+    end
+
+    def compute_streams_subscriptions
+      @streams_sockets.each_value.map(&:size).reduce(0, :+)
+    end
+
+    def compute_streams_kinds
+      @streams_sockets.size
+    end
+
+    def sanity_check_metrics_connections
+      return unless @metric_connections_current
+
+      connections_current_all = @metric_connections_current.values.values.reduce(0, :+)
+      return if connections_current_all == compute_connections_all()
+
+      logger.warn "slip detected in metric_connections_current, recalculating"
+      connections_current_private = compute_connections_private()
+      @metric_connections_current.set(connections_current_private, labels: {auth: "private"})
+      @metric_connections_current.set(compute_connections_all() - connections_current_private, labels: {auth: "public"})
+    end
+
+    def on_connection_open(connection)
+      @connections[connection.id] = connection
+      unless connection.authorized
+        @metric_connections_current&.increment(labels: {auth: "public"})
+        @metric_connections_total&.increment(labels: {auth: "public"})
+        return
+      end
+      @metric_connections_current&.increment(labels: {auth: "private"})
+      @metric_connections_total&.increment(labels: {auth: "private"})
+
+      @connections_by_userid[connection.user] ||= ConnectionArray.new
+      @connections_by_userid[connection.user] << connection
+    end
+
+    def on_connection_close(connection)
+      @connections.delete(connection.id)
+      connection.streams.keys.each do |stream|
+        on_unsubscribe(connection, stream)
+      end
+
+      unless connection.authorized
+        @metric_connections_current&.decrement(labels: {auth: "public"})
+        sanity_check_metrics_connections
+        return
+      end
+      @metric_connections_current&.decrement(labels: {auth: "private"})
+
+      @connections_by_userid[connection.user].delete(connection)
+      @connections_by_userid.delete(connection.user) \
+        if @connections_by_userid[connection.user].empty?
+      sanity_check_metrics_connections
+    end
+
+    def on_subscribe(connection, stream)
+      @streams_sockets[stream] ||= ConnectionArray.new
+      @streams_sockets[stream] << connection
+      send_snapshot_and_increments(connection, storekey(stream)) if increment?(stream)
+      @metric_subscriptions_current&.set(compute_stream_subscriptions(stream), labels: {stream: stream})
+    end
+
+    def send_snapshot_and_increments(connection, key)
+      return unless @stores[key]
+      return unless @stores[key][:snapshot]
+
+      connection.send_raw(@stores[key][:snapshot])
+      @stores[key][:increments]&.each {|inc| connection.send_raw(inc) }
+    end
+
+    def on_unsubscribe(connection, stream)
+      return unless @streams_sockets[stream]
+
+      @streams_sockets[stream].delete(connection)
+      @streams_sockets.delete(stream) if @streams_sockets[stream].empty?
+      @metric_subscriptions_current&.set(compute_stream_subscriptions(stream), labels: {stream: stream})
+    end
+
+    def send_private_message(user_id, event, payload_decoded)
+      Array(@connections_by_userid[user_id]).each do |connection|
+        connection.send(event, payload_decoded) if connection.streams.include?(event)
+      end
+    end
+
+    def send_public_message(stream, raw_message)
+      Array(@streams_sockets[stream]).each do |connection|
+        connection.send_raw(raw_message)
+      end
+    end
+
+    #
+    # routing key format: type.id.event
+    # * `type` can be *public* or *private*
+    # * `id` can be user id or market id
+    # * `event` is the event identifier, ex: order_completed, trade, ...
+    #
+    def on_message(delivery_info, _metadata, payload)
+      routing_key = delivery_info.routing_key
+      if routing_key.count(".") != 2
+        logger.error { "invalid routing key from amqp: #{routing_key}" }
+        return
+      end
+
+      type, id, event = routing_key.split(".")
+      payload_decoded = JSON.parse(payload)
+
+      if type == "private"
+        send_private_message(id, event, payload_decoded)
+        return
+      end
+
+      stream = [id, event].join(".")
+      message = JSON.dump(stream => payload_decoded)
+
+      if snapshot?(event)
+        key = storekey(stream)
+
+        unless @stores[key]
+          # Send the snapshot to subscribers of -inc stream if there were no snapshot before
+          send_public_message("#{key}-inc", message)
+        end
+
+        @stores[key] = {
+          snapshot:   message,
+          increments: [],
+        }
+        return
+      end
+
+      if increment?(event)
+        key = storekey(stream)
+
+        unless @stores[key]
+          logger.warn { "Discard increment received before snapshot for store:#{key}" }
+          return
+        end
+
+        @stores[key][:increments] << message
+      end
+
+      send_public_message(stream, message)
+    end
+  end
+end