bbk-app 1.0.0.72899

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b47e943b8f8075f53603a17fdbe11c1c71de2dd330e9a338f7ab5f4901602ecc
+  data.tar.gz: f0f7ebd29ed819ebc6e2e0d7999ff138989bd637bc382bb8be5c42397a80dc32
+SHA512:
+  metadata.gz: 261f9ce6914823596cd18afc01ff4ec8717c472ba5b956e3ee3fcfdde69c21bdee6193ff48d959aeab6211fdda8e44e4f47e21a7c2f2be2990377a6f5129cd29
+  data.tar.gz: ca5a3164cc571ba52c9d04b704ab199b700027ce65c935b08e220cb364cd047f27b6613f12de152a7b26c8d989f1cf1faa7b7307ae8bc570ea7978e235f4c3fb

data/Gemfile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in bbk-app.gemspec
+gemspec
+

data/Gemfile.lock

@@ -0,0 +1,177 @@
+PATH
+  remote: .
+  specs:
+    bbk-app (1.0.0.72899)
+      activesupport
+      bbk-utils (> 1.0.1)
+      timeouter
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.1.4.4)
+      activesupport (= 6.1.4.4)
+    activerecord (6.1.4.4)
+      activemodel (= 6.1.4.4)
+      activesupport (= 6.1.4.4)
+    activesupport (6.1.4.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    amq-protocol (2.3.2)
+    ansi (1.5.0)
+    ast (2.4.2)
+    axiom-types (0.1.1)
+      descendants_tracker (~> 0.0.4)
+      ice_nine (~> 0.11.0)
+      thread_safe (~> 0.3, >= 0.3.1)
+    bbk-utils (1.0.1.72735)
+      activesupport (~> 6.0)
+      russian
+    bunny (2.19.0)
+      amq-protocol (~> 2.3, >= 2.3.1)
+      sorted_set (~> 1, >= 1.0.2)
+    bunny-mock (1.7.0)
+      bunny (>= 1.7)
+    byebug (11.1.3)
+    coercible (1.0.0)
+      descendants_tracker (~> 0.0.1)
+    concurrent-ruby (1.1.9)
+    database_cleaner (1.99.0)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    diff-lcs (1.5.0)
+    docile (1.4.0)
+    equalizer (0.0.11)
+    erubis (2.7.0)
+    faker (2.19.0)
+      i18n (>= 1.6, < 2)
+    flay (2.12.1)
+      erubis (~> 2.7.0)
+      path_expander (~> 1.0)
+      ruby_parser (~> 3.0)
+      sexp_processor (~> 4.0)
+    flog (4.6.4)
+      path_expander (~> 1.0)
+      ruby_parser (~> 3.1, > 3.1.0)
+      sexp_processor (~> 4.8)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    ice_nine (0.11.2)
+    kwalify (0.7.2)
+    launchy (2.5.0)
+      addressable (~> 2.7)
+    minitest (5.15.0)
+    parallel (1.21.0)
+    parser (3.0.3.2)
+      ast (~> 2.4.1)
+    path_expander (1.1.0)
+    public_suffix (4.0.6)
+    rainbow (3.1.1)
+    rake (12.3.3)
+    rbtree (0.4.4)
+    reek (6.0.6)
+      kwalify (~> 0.7.0)
+      parser (~> 3.0.0)
+      rainbow (>= 2.0, < 4.0)
+    regexp_parser (2.2.0)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.3)
+    rspec_junit_formatter (0.5.1)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    rubocop (1.24.1)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.15.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.15.1)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    ruby_parser (3.18.1)
+      sexp_processor (~> 4.16)
+    rubycritic (4.6.1)
+      flay (~> 2.8)
+      flog (~> 4.4)
+      launchy (>= 2.0.0)
+      parser (>= 2.6.0)
+      rainbow (~> 3.0)
+      reek (~> 6.0, < 7.0)
+      ruby_parser (~> 3.8)
+      simplecov (>= 0.17.0)
+      tty-which (~> 0.4.0)
+      virtus (~> 1.0)
+    russian (0.6.0)
+      i18n (>= 0.5.0)
+    set (1.0.2)
+    sexp_processor (4.16.0)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-console (0.9.1)
+      ansi
+      simplecov
+      terminal-table
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    sorted_set (1.0.3)
+      rbtree
+      set (~> 1.0)
+    sqlite3 (1.4.2)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
+    thread_safe (0.3.6)
+    timeouter (0.1.3.38794)
+    tty-which (0.4.2)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.1.0)
+    virtus (1.0.5)
+      axiom-types (~> 0.1)
+      coercible (~> 1.0)
+      descendants_tracker (~> 0.0, >= 0.0.3)
+      equalizer (~> 0.0, >= 0.0.9)
+    zeitwerk (2.5.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord (~> 6.0)
+  bbk-app!
+  bundler (~> 2.0)
+  bunny-mock (~> 1.7.0)
+  byebug
+  database_cleaner (~> 1.7)
+  faker (~> 2)
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  rspec_junit_formatter
+  rubocop
+  rubycritic
+  simplecov
+  simplecov-console
+  sqlite3 (~> 1.4)
+
+BUNDLED WITH
+   2.2.33

data/README.md

@@ -0,0 +1,38 @@
+# BBK::App
+
+Classes for building services based on BBK stack.
+
+## Installation
+
+Adding to a gem:
+
+```ruby
+# my-cool-gem.gemspec
+
+Gem::Specification.new do |spec|
+  # ...
+  spec.add_dependency "bbk-app", "~> 1.0.0"
+  # ...
+end
+```
+
+Or adding to your project:
+
+```ruby
+# Gemfile
+
+gem "bbk-app", "~> 1.0.0"
+```
+
+### Supported Ruby versions
+
+* Ruby (MRI) >= 2.5.0
+
+### Tested Ruby versions
+
+* Ruby (MRI) 2.5.x
+* Ruby (MRI) 3.0.x
+
+## License
+
+The gem is available as open source under the terms of the MIT License.

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'bbk/app'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)
+

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/bbk/app/dispatcher/message.rb

@@ -0,0 +1,47 @@
+module BBK
+  module App
+    class Dispatcher
+      class Message
+
+        attr_reader :consumer, :delivery_info, :headers, :payload, :body
+
+        def initialize(consumer, delivery_info, headers, body, *_args, **_kwargs)
+          @consumer = consumer
+          @delivery_info = delivery_info
+          @headers = headers.to_h.with_indifferent_access
+          @body = body
+          @payload = begin
+            JSON(body).with_indifferent_access
+          rescue StandardError
+            {}.with_indifferent_access
+          end
+        end
+
+        def ack(*args, answer: nil, **kwargs)
+          consumer.ack(self, *args, answer: answer, **kwargs)
+        end
+
+        def nack(*args, error: nil, **kwargs)
+          consumer.nack(self, *args, error: error, **kwargs)
+        end
+
+        def message_id
+          raise NotImplementedError
+        end
+
+        def reply_message_id(addon)
+          Digest::SHA1.hexdigest("#{addon}#{message_id}")
+        end
+
+        def to_h
+          {
+            headers: headers,
+            body:    body
+          }
+        end
+
+      end
+    end
+  end
+end
+

data/lib/bbk/app/dispatcher/message_stream.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module BBK
+  module App
+    class Dispatcher
+      class MessageStream
+
+        CLOSE_VALUE = :close
+        attr_reader :queue, :stream
+
+        def initialize(size: 10)
+          @queue = SizedQueue.new(size)
+          @closed = false
+        end
+
+        def push(message)
+          @queue.push(message) unless @closed
+        end
+        alias << push
+
+        def each
+          return to_enum unless block_given?
+          return nil if @closed
+
+          loop do
+            value = @queue.pop
+            break if value == CLOSE_VALUE
+
+            yield(value)
+          end
+        end
+
+        def close
+          @closed = true
+          @queue << CLOSE_VALUE
+        end
+
+      end
+    end
+  end
+end
+

data/lib/bbk/app/dispatcher/queue_stream_strategy.rb

@@ -0,0 +1,54 @@
+require 'bbk/app/dispatcher/message_stream'
+
+module BBK
+  module App
+    class Dispatcher
+      class QueueStreamStrategy
+
+        def initialize(pool, logger:)
+          @pool = pool
+          @logger = logger
+        end
+
+        def run(consumers, &block)
+          @unblocker = Queue.new
+          @stream = BBK::App::Dispatcher::MessageStream.new(size: 10)
+
+          consumers.each {|cons| cons.run(@stream) }
+          @stream.each do |msg|
+            @logger.debug "[#{self.class}] Consumed message #{msg.headers}"
+            @pool.post(msg) do |m|
+              block.call(m)
+            end
+          end
+
+          begin
+            @pool.shutdown
+          rescue StandardError
+            nil
+          end
+          @pool.kill unless @pool.wait_for_termination(@stop_queue_timeout)
+        ensure
+          @unblocker.push(:ok)
+        end
+
+        def push(*args)
+          @stream.push(*args)
+        end
+
+        def stop(timeout = 5)
+          @stop_queue_timeout = timeout
+
+          begin
+            @stream.close
+          rescue StandardError
+            nil
+          end
+          @unblocker.pop
+        end
+
+      end
+    end
+  end
+end
+

data/lib/bbk/app/dispatcher/result.rb

@@ -0,0 +1,20 @@
+module BBK
+  module App
+    class Dispatcher
+      class Result
+
+        attr_accessor :route, :message
+
+        def initialize(route, message)
+          @route = route.is_a?(String) ? Dispatcher::Route.new(route) : route
+
+          raise 'route must be of type Dispatcher::Route' unless @route.is_a?(Dispatcher::Route)
+
+          @message = message
+        end
+
+      end
+    end
+  end
+end
+

data/lib/bbk/app/dispatcher/route.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module BBK
+  module App
+    class Dispatcher
+      class Route
+
+        attr_reader :uri, :scheme, :domain, :routing_key
+
+        # Example: mq://gw@service.smev.request
+        def initialize(string)
+          @uri = URI(string)
+          @scheme = uri.scheme
+          @domain = uri.user
+          @routing_key = "#{uri.host}#{uri.path}"
+
+          # raise 'domain must present in route' if @domain.blank?
+          raise 'routing_key must present in route' if @routing_key.blank?
+        end
+
+        def to_s
+          @uri.to_s
+        end
+
+        def ==(other)
+          if other.is_a?(String)
+            to_s == other
+          else
+            super
+          end
+        end
+
+      end
+    end
+  end
+end
+

data/lib/bbk/app/dispatcher.rb

@@ -0,0 +1,210 @@
+require 'concurrent'
+require 'bbk/app/thread_pool'
+require 'bbk/app/dispatcher/message_stream'
+require 'bbk/app/dispatcher/message'
+require 'bbk/app/dispatcher/queue_stream_strategy'
+require 'bbk/app/dispatcher/result'
+require 'bbk/app/dispatcher/route'
+require 'bbk/utils/proxy_logger'
+
+module BBK
+  module App
+
+    class SimplePoolFactory
+
+      def self.call(pool_size, queue_size)
+        BBK::App::ThreadPool.new(pool_size, queue: queue_size)
+      end
+
+    end
+
+    class ConcurrentPoolFactory
+
+      def self.call(pool_size, queue_size)
+        Concurrent::FixedThreadPool.new(pool_size, max_queue:       queue_size,
+                                                   fallback_policy: :caller_runs)
+      end
+
+    end
+
+    class Dispatcher
+
+      attr_accessor :force_quit
+      attr_reader :consumers, :publishers, :observer, :middlewares, :logger
+
+      ANSWER_DOMAIN = 'answer'
+
+      def initialize(observer, pool_size: 3, logger: BBK::App.logger, pool_factory: SimplePoolFactory, stream_strategy: QueueStreamStrategy)
+        @observer = observer
+        @pool_size = pool_size
+        logger = logger.respond_to?(:tagged) ? logger : ActiveSupport::TaggedLogging.new(logger)
+        @logger = BBK::Utils::ProxyLogger.new(logger, tags: 'Dispatcher')
+        @consumers = []
+        @publishers = []
+        @middlewares = []
+        @pool_factory = pool_factory
+        @stream_strategy_class = stream_strategy
+        @force_quit = false
+      end
+
+      def register_consumer(consumer)
+        consumers << consumer
+      end
+
+      def register_publisher(publisher)
+        publishers << publisher
+      end
+
+      def register_middleware(middleware)
+        middlewares << middleware
+      end
+
+      # Run all consumers and blocks on message processing
+      def run
+        @pool = @pool_factory.call(@pool_size, 10)
+        @stream_strategy = @stream_strategy_class.new(@pool, logger: logger)
+        ActiveSupport::Notifications.instrument 'dispatcher.run', dispatcher: self
+
+        @stream_strategy.run(consumers) do |msg|
+          begin
+            logger.tagged(msg.headers[:message_id]) do
+              process msg
+            end
+          rescue StandardError => e
+            logger.fatal "E[#{@stream_strategy_class}]: #{e}"
+            logger.fatal "E[#{@stream_strategy_class}]: #{e.backtrace.join("\n")}"
+          end
+        end
+      end
+
+      # stop dispatcher and wait for termination
+      # Чтоб остановить диспетчер надо:
+      # 1. остановить консьюмеры
+      # 2. остановить прием новых сообщений - @stream.close
+      # 3. дождаться обработки всего в очереди или таймаут
+      # 4. остановить потоки
+      # 5. остановить паблишеры
+      def close(_timeout = 5)
+        ActiveSupport::Notifications.instrument 'dispatcher.close', dispatcher: self
+        consumers.each do |cons|
+          begin
+            cons.stop
+          rescue StandardError => e
+            logger.error "Consumer #{cons} stop error: #{e}"
+            logger.debug e.backtrace
+          end
+        end
+
+        @stream_strategy.stop(5)
+
+        consumers.each do |cons|
+          begin
+            cons.close
+          rescue StandardError => e
+            logger.error "Consumer #{cons} close error: #{e}"
+            logger.debug e.backtrace
+          end
+        end
+
+        publishers.each do |pub|
+          begin
+            pub.close
+          rescue StandardError => e
+            logger.error "Publisher #{pub} close error: #{e}"
+            logger.debug e.backtrace
+          end
+        end
+      end
+
+      protected
+
+        def process(message)
+          results = build_processing_stack.call(message).select do |e|
+            e.is_a? BBK::App::Dispatcher::Result
+          end
+          logger.debug "There are #{results.count} results to send from #{message.headers[:message_id]}..."
+          send_results(message, results).value
+        rescue StandardError => e
+          ActiveSupport::Notifications.instrument 'dispatcher.exception', msg: message, exception: e
+          message.nack(error: e)
+          close if force_quit
+        end
+
+        def process_message(message)
+          matched, processor = find_processor(message)
+          results = []
+          begin
+            is_unknown = @observer.instance_variable_get('@default') == processor
+            ActiveSupport::Notifications.instrument 'dispatcher.request.process', msg: message, match: matched, unknown: is_unknown do
+              processor.call(message, results: results)
+            end
+          rescue StandardError => e
+            if processor.respond_to?(:on_error)
+              results = processor.on_error(message, e)
+            else
+              raise
+            end
+          end
+          [results].flatten
+        rescue StandardError => e
+          ActiveSupport::Notifications.instrument 'dispatcher.request.exception', msg: message, match: matched, processor: processor, exception: e
+          raise
+        end
+
+        def find_processor(msg)
+          matched, callback = @observer.match(msg.headers, msg.payload, msg.delivery_info)
+          [matched, callback.is_a?(BBK::App::Factory) ? callback.create : callback]
+        end
+
+        def build_processing_stack
+          stack = proc{|msg| process_message(msg) }
+          middlewares.reduce(stack) do |stack, middleware|
+            if middleware.respond_to?(:build)
+              middleware.build(stack)
+            else
+              middleware.new(stack)
+            end
+          end
+        end
+
+        def send_results(incoming, results)
+          message_id = incoming.headers[:message_id]
+
+          answer = results.find {|msg| msg.route.domain == ANSWER_DOMAIN }
+          Concurrent::Promises.zip_futures(*results.map do |result|
+                                             publish_result(result)
+                                           end).then do |_successes|
+            incoming.ack(answer: answer)
+          end.rescue do |*errors|
+            error = errors.compact.first
+            ActiveSupport::Notifications.instrument 'dispatcher.request.result_rejected',
+                                                    msg: incoming, message: error.inspect
+            logger.error "[Message#{message_id}] Publish failed: #{error.inspect}"
+            incoming.nack(error: error)
+            close if force_quit
+          rescue StandardError => e
+            warn e.backtrace
+            warn "[CRITICAL] #{self.class} [#{Process.pid}] failure exiting: #{e.inspect}"
+            ActiveSupport::Notifications.instrument 'dispatcher.exception', msg:       incoming,
+                                                                            exception: e
+            sleep(10)
+            exit!(1)
+          end
+        end
+
+        # @return [Concurrent::Promises::ResolvableFuture]
+        def publish_result(result)
+          route = result.route
+          logger.debug "Publish result to #{route} ..."
+          publisher = publishers.find {|pub| pub.protocols.include?(route.scheme) }
+          raise "Not found publisher for scheme #{route.scheme}" if publisher.nil?
+
+          # return Concurrent::Promises.resolvable_future
+          publisher.publish(result)
+        end
+
+    end
+
+  end
+end
+

data/lib/bbk/app/factory.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module BBK
+  module App
+    class Factory
+
+      attr_accessor :klass, :instanceargs, :instancekwargs
+
+      def initialize(klass, *args, **kwargs)
+        @klass = klass
+        @instanceargs = args
+        @instancekwargs = kwargs
+      end
+
+      def create
+        klass.new(*instanceargs, **instancekwargs)
+      end
+
+      def call(*args, **kwargs)
+        create.call(*args, **kwargs)
+      end
+
+    end
+  end
+end
+