shivam 0.0.0-java

data/bin/ci/before_build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env sh
+
+CTL=${BUNNY_RABBITMQCTL:-"sudo rabbitmqctl"}
+PLUGINS=${BUNNY_RABBITMQ_PLUGINS:-"sudo rabbitmq-plugins"}
+
+echo "Will use rabbitmqctl at ${CTL}"
+echo "Will use rabbitmq-plugins at ${PLUGINS}"
+
+$PLUGINS enable rabbitmq_management
+
+sleep 3
+
+# guest:guest has full access to /
+$CTL add_vhost /
+$CTL add_user guest guest
+$CTL set_permissions -p / guest ".*" ".*" ".*"
+
+# Reduce retention policy for faster publishing of stats
+$CTL eval 'supervisor2:terminate_child(rabbit_mgmt_sup_sup, rabbit_mgmt_sup), application:set_env(rabbitmq_management,       sample_retention_policies, [{global, [{605, 1}]}, {basic, [{605, 1}]}, {detailed, [{10, 1}]}]), rabbit_mgmt_sup_sup:start_child().' || true
+$CTL eval  'supervisor2:terminate_child(rabbit_mgmt_agent_sup_sup, rabbit_mgmt_agent_sup), application:set_env(rabbitmq_management_agent, sample_retention_policies, [{global, [{605, 1}]}, {basic, [{605, 1}]}, {detailed, [{10, 1}]}]), rabbit_mgmt_agent_sup_sup:start_child().' || true

data/bin/ci/before_build_docker.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+CTL=${MARCH_HARE_RABBITMQCTL:="docker exec rabbitmq rabbitmqctl"}
+PLUGINS=${MARCH_HARE_RABBITMQ_PLUGINS:="docker exec rabbitmq rabbitmq-plugins"}
+
+$PLUGINS enable rabbitmq_management
+
+sleep 3
+
+# guest:guest has full access to /
+
+$CTL add_vhost /
+# $CTL add_user guest guest # already exists
+$CTL set_permissions -p / guest ".*" ".*" ".*"
+
+# Reduce retention policy for faster publishing of stats
+$CTL eval 'supervisor2:terminate_child(rabbit_mgmt_sup_sup, rabbit_mgmt_sup), application:set_env(rabbitmq_management,       sample_retention_policies, [{global, [{605, 1}]}, {basic, [{605, 1}]}, {detailed, [{10, 1}]}]), rabbit_mgmt_sup_sup:start_child().'
+$CTL eval 'supervisor2:terminate_child(rabbit_mgmt_agent_sup_sup, rabbit_mgmt_agent_sup), application:set_env(rabbitmq_management_agent, sample_retention_policies, [{global, [{605, 1}]}, {basic, [{605, 1}]}, {detailed, [{10, 1}]}]), rabbit_mgmt_agent_sup_sup:start_child().'
+
+sleep 3

data/bin/ci/install_on_debian.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+sudo apt-get install curl gnupg debian-keyring debian-archive-keyring apt-transport-https -y
+
+## Team RabbitMQ's main signing key
+sudo apt-key adv --keyserver "hkps://keys.openpgp.org" --recv-keys "0x0A9AF2115F4687BD29803A206B73A36E6026DFCA"
+## Launchpad PPA that provides modern Erlang releases
+sudo apt-key adv --keyserver "keyserver.ubuntu.com" --recv-keys "F77F1EDA57EBB1CC"
+## PackageCloud RabbitMQ repository
+sudo apt-key adv --keyserver "keyserver.ubuntu.com" --recv-keys "F6609E60DC62814E"
+
+## Add apt repositories maintained by Team RabbitMQ
+sudo tee /etc/apt/sources.list.d/rabbitmq.list <<EOF
+## Provides modern Erlang/OTP releases
+##
+## "bionic" as distribution name should work for any reasonably recent Ubuntu or Debian release.
+## See the release to distribution mapping table in RabbitMQ doc guides to learn more.
+deb http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main
+deb-src http://ppa.launchpad.net/rabbitmq/rabbitmq-erlang/ubuntu bionic main
+
+## Provides RabbitMQ
+##
+## "bionic" as distribution name should work for any reasonably recent Ubuntu or Debian release.
+## See the release to distribution mapping table in RabbitMQ doc guides to learn more.
+deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main
+deb-src https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ bionic main
+EOF
+
+## Update package indices
+sudo apt-get update -y
+
+## Install Erlang packages
+sudo apt-get install -y erlang-base \
+                        erlang-asn1 erlang-crypto erlang-eldap erlang-ftp erlang-inets \
+                        erlang-mnesia erlang-os-mon erlang-parsetools erlang-public-key \
+                        erlang-runtime-tools erlang-snmp erlang-ssl \
+                        erlang-syntax-tools erlang-tftp erlang-tools erlang-xmerl
+
+## Install rabbitmq-server and its dependencies
+sudo apt-get install rabbitmq-server -y --fix-missing
+
+sudo service rabbitmq-server start
+
+sudo rabbitmqctl await_startup --timeout 120
+
+until sudo lsof -i:5672; do echo "Waiting for RabbitMQ to start..."; sleep 1; done

data/bin/hutch

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/hutch'
+require_relative '../lib/hutch/cli'
+
+cli = Hutch::CLI.new
+cli.run(ARGV)
+

data/examples/consumer.rb

@@ -0,0 +1,13 @@
+require 'hutch'
+
+class TestConsumer
+  include Hutch::Consumer
+  consume 'hutch.test'
+
+  def process(message)
+    puts "TestConsumer got a message: #{message}"
+    puts "Processing..."
+    sleep(1)
+    puts "Done"
+  end
+end

data/examples/producer.rb

@@ -0,0 +1,10 @@
+require 'hutch'
+
+Hutch.connect
+loop do
+  print "Press return to send test message..."
+  gets
+  Hutch.publish('hutch.test', subject: 'test message')
+  puts "Send message with routing key 'hutch.test'"
+end
+

data/hutch.gemspec

@@ -0,0 +1,27 @@
+require File.expand_path('../lib/hutch/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  if defined?(JRUBY_VERSION)
+    gem.platform = 'java'
+    gem.add_runtime_dependency 'march_hare', '>= 3.0.0'
+  else
+    gem.platform = Gem::Platform::RUBY
+    gem.add_runtime_dependency 'bunny', '>= 2.20', '< 3.0'
+  end
+  gem.add_runtime_dependency 'carrot-top', '~> 0.0.7'
+  gem.add_runtime_dependency 'multi_json', '~> 1.15'
+  gem.add_runtime_dependency 'activesupport', '>= 4.2', '< 8'
+
+  gem.name = 'shivam'
+  gem.summary = 'Opinionated asynchronous inter-service communication using RabbitMQ'
+  gem.description = 'Hutch is a Ruby library for enabling asynchronous inter-service communication using RabbitMQ'
+  gem.version = "0.0.0"
+  gem.required_ruby_version = '>= 2.6'
+  gem.authors = ['Harry Marr', 'Michael Klishin']
+  gem.homepage = 'https://github.com/nulllvoid/hutch'
+  gem.require_paths = ['lib']
+  gem.license = 'MIT'
+  gem.executables = ['hutch']
+  gem.files = `git ls-files`.split("\n")
+  gem.test_files = `git ls-files -- spec/*`.split("\n")
+end

data/lib/hutch/acknowledgements/base.rb

@@ -0,0 +1,16 @@
+module Hutch
+  module Acknowledgements
+    # Defines acknowledgement handler interface.
+    class Base
+      # Implements negative acknowledgement/requeueing logic
+      # and returns a boolean to indicate whether acknowledgement
+      # was performed. If false is returned, next handler in the
+      # chain will be invoked.
+      #
+      # The chain always falls back to unconditional nacking.
+      def handle(delivery_info, properties, broker, ex)
+        raise NotImplementedError.new
+      end
+    end
+  end
+end

data/lib/hutch/acknowledgements/nack_on_all_failures.rb

@@ -0,0 +1,19 @@
+require 'hutch/logging'
+require 'hutch/acknowledgements/base'
+
+module Hutch
+  module Acknowledgements
+    class NackOnAllFailures < Base
+      include Logging
+
+      def handle(delivery_info, properties, broker, ex)
+        prefix = "message(#{properties.message_id || '-'}): "
+        logger.debug "#{prefix} nacking message"
+
+        broker.nack(delivery_info.delivery_tag)
+
+        true
+      end
+    end
+  end
+end

data/lib/hutch/adapter.rb

@@ -0,0 +1,11 @@
+if defined?(JRUBY_VERSION)
+  require 'hutch/adapters/march_hare'
+  module Hutch
+    Adapter = Adapters::MarchHareAdapter
+  end
+else
+  require 'hutch/adapters/bunny'
+  module Hutch
+    Adapter = Adapters::BunnyAdapter
+  end
+end

data/lib/hutch/adapters/bunny.rb

@@ -0,0 +1,37 @@
+require 'bunny'
+require 'forwardable'
+
+module Hutch
+  module Adapters
+    class BunnyAdapter
+      extend Forwardable
+
+      DEFAULT_VHOST = Bunny::Session::DEFAULT_VHOST
+
+      ConnectionRefused = Bunny::TCPConnectionFailed
+      PreconditionFailed = Bunny::PreconditionFailed
+
+      def_delegators :@connection, :start, :disconnect, :close, :create_channel, :open?
+
+      def initialize(opts={})
+        @connection = Bunny.new(opts)
+      end
+
+      def self.decode_message(delivery_info, properties, payload)
+        [delivery_info, properties, payload]
+      end
+
+      def prefetch_channel(ch, prefetch)
+        ch.prefetch(prefetch) if prefetch
+      end
+
+      def current_timestamp
+        Time.now.to_i
+      end
+
+      def self.new_exchange(ch, exchange_type, exchange_name, exchange_options)
+        Bunny::Exchange.new(ch, exchange_type, exchange_name, exchange_options)
+      end
+    end
+  end
+end

data/lib/hutch/adapters/march_hare.rb

@@ -0,0 +1,41 @@
+require 'march_hare'
+require 'forwardable'
+
+module Hutch
+  module Adapters
+    class MarchHareAdapter
+      extend Forwardable
+
+      DEFAULT_VHOST = "/"
+
+      ConnectionRefused = MarchHare::ConnectionRefused
+      PreconditionFailed = MarchHare::PreconditionFailed
+
+      def_delegators :@connection, :start, :disconnect, :close, :open?
+
+      def initialize(opts = {})
+        @connection = MarchHare.connect(opts)
+      end
+
+      def self.decode_message(delivery_info, payload)
+        [delivery_info, delivery_info.properties, payload]
+      end
+
+      def prefetch_channel(ch, prefetch)
+        ch.prefetch = prefetch if prefetch
+      end
+
+      def create_channel(n = nil, consumer_pool_size = 1, consumer_pool_abort_on_exception = false)
+        @connection.create_channel(n)
+      end
+
+      def current_timestamp
+        Time.now
+      end
+
+      def self.new_exchange(ch, exchange_type, exchange_name, exchange_options)
+        MarchHare::Exchange.new(ch, exchange_name, exchange_options.merge(type: exchange_type))
+      end
+    end
+  end
+end

data/lib/hutch/broker.rb

@@ -0,0 +1,384 @@
+require 'active_support/core_ext/object/blank'
+
+require 'carrot-top'
+require 'hutch/logging'
+require 'hutch/exceptions'
+require 'hutch/publisher'
+
+module Hutch
+  class Broker
+    include Logging
+
+    attr_accessor :connection, :channel, :exchange, :api_client
+
+
+    DEFAULT_AMQP_PORT =
+      case RUBY_ENGINE
+      when "jruby" then
+        com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_PORT
+      when "ruby" then
+        AMQ::Protocol::DEFAULT_PORT
+      end
+
+    DEFAULT_AMQPS_PORT =
+      case RUBY_ENGINE
+      when "jruby" then
+        com.rabbitmq.client.ConnectionFactory::DEFAULT_AMQP_OVER_SSL_PORT
+      when "ruby" then
+        AMQ::Protocol::TLS_PORT
+      end
+
+
+    # @param config [nil,Hash] Configuration override
+    def initialize(config = nil)
+      @config = config || Hutch::Config
+    end
+
+    # Connect to broker
+    #
+    # @example
+    #   Hutch::Broker.new.connect(enable_http_api_use: true) do
+    #     # will disconnect after this block
+    #   end
+    #
+    # @param [Hash] options The options to connect with
+    # @option options [Boolean] :enable_http_api_use
+    def connect(options = {})
+      @options = options
+      set_up_amqp_connection
+      if http_api_use_enabled?
+        logger.info "HTTP API use is enabled"
+        set_up_api_connection
+      else
+        logger.info "HTTP API use is disabled"
+      end
+
+      if tracing_enabled?
+        logger.info "tracing is enabled using #{@config[:tracer]}"
+      else
+        logger.info "tracing is disabled"
+      end
+
+      if block_given?
+        begin
+          yield
+        ensure
+          disconnect
+        end
+      end
+    end
+
+    def disconnect
+      @channel.close    if @channel
+      @connection.close if @connection
+      @channel = nil
+      @connection = nil
+      @exchange = nil
+      @api_client = nil
+    end
+
+    # Connect to RabbitMQ via AMQP
+    #
+    # This sets up the main connection and channel we use for talking to
+    # RabbitMQ. It also ensures the existence of the exchange we'll be using.
+    def set_up_amqp_connection
+      open_connection!
+      open_channel!
+      declare_exchange!
+      declare_publisher!
+    end
+
+    def open_connection
+      logger.info "connecting to rabbitmq (#{sanitized_uri})"
+
+      connection = Hutch::Adapter.new(connection_params)
+
+      with_bunny_connection_handler(sanitized_uri) do
+        connection.start
+      end
+
+      logger.info "connected to RabbitMQ at #{connection_params[:host]} as #{connection_params[:username]}"
+      connection
+    end
+
+    def open_connection!
+      @connection = open_connection
+    end
+
+    def open_channel
+      logger.info "opening rabbitmq channel with pool size #{consumer_pool_size}, abort on exception #{consumer_pool_abort_on_exception}"
+      connection.create_channel(nil, consumer_pool_size, consumer_pool_abort_on_exception).tap do |ch|
+        connection.prefetch_channel(ch, @config[:channel_prefetch])
+        if @config[:publisher_confirms] || @config[:force_publisher_confirms]
+          logger.info 'enabling publisher confirms'
+          ch.confirm_select
+        end
+      end
+    end
+
+    def open_channel!
+      @channel = open_channel
+    end
+
+    def declare_exchange(name,channel)
+      exchange_name = name or @config[:mq_exchange]
+      exchange_type = @config[:mq_exchange_type]
+      exchange_options = { durable: true }.merge(@config[:mq_exchange_options])
+      logger.info "using topic exchange '#{exchange_name}'"
+
+      with_bunny_precondition_handler('exchange') do
+        Adapter.new_exchange(channel, exchange_type, exchange_name, exchange_options)
+      end
+    end
+
+    def declare_exchange!
+      name = @config[:mq_exchange]
+      @exchange = declare_exchange(name,channel)
+    end
+
+    def declare_publisher!
+      @publisher = Hutch::Publisher.new(connection, channel, exchange, @config)
+    end
+
+    # Set up the connection to the RabbitMQ management API. Unfortunately, this
+    # is necessary to do a few things that are impossible over AMQP. E.g.
+    # listing queues and bindings.
+    def set_up_api_connection
+      logger.info "connecting to rabbitmq HTTP API (#{api_config.sanitized_uri})"
+
+      with_authentication_error_handler do
+        with_connection_error_handler do
+          @api_client = CarrotTop.new(host: api_config.host, port: api_config.port,
+                                      user: api_config.username, password: api_config.password,
+                                      ssl: api_config.ssl)
+          @api_client.exchanges
+        end
+      end
+    end
+
+    def http_api_use_enabled?
+      op = @options.fetch(:enable_http_api_use, true)
+      cf = if @config[:enable_http_api_use].nil?
+             true
+           else
+             @config[:enable_http_api_use]
+           end
+
+      op && cf
+    end
+
+    def tracing_enabled?
+      @config[:tracer] && @config[:tracer] != Hutch::Tracers::NullTracer
+    end
+
+    # Create / get a durable queue and apply namespace if it exists.
+    def queue(name, options = {})
+      with_bunny_precondition_handler('queue') do
+        namespace = @config[:namespace].to_s.downcase.gsub(/[^-_:\.\w]/, "")
+        queue_name = namespace.present? ? "#{namespace}:#{name}" : name
+        channel.queue(queue_name, **options)
+      end
+    end
+
+    # Return a mapping of queue names to the routing keys they're bound to.
+    def bindings
+      results = Hash.new { |hash, key| hash[key] = [] }
+
+      filtered = api_client.bindings.
+        reject { |b| b['destination'] == b['routing_key'] }.
+        select { |b| b['source'] == @config[:mq_exchange] && b['vhost'] == @config[:mq_vhost] }
+
+      filtered.each do |binding|
+        results[binding['destination']] << binding['routing_key']
+      end
+
+      results
+    end
+
+    # Find the existing bindings, and unbind any redundant bindings
+    def unbind_redundant_bindings(queue, routing_keys)
+      return unless http_api_use_enabled?
+
+      filtered = bindings.select { |dest, keys| dest == queue.name }
+      filtered.each do |dest, keys|
+        keys.reject { |key| routing_keys.include?(key) }.each do |key|
+          logger.debug "removing redundant binding #{queue.name} <--> #{key}"
+          queue.unbind(exchange, routing_key: key)
+        end
+      end
+    end
+
+    # Bind a queue to the broker's exchange on the routing keys provided. Any
+    # existing bindings on the queue that aren't present in the array of
+    # routing keys will be unbound.
+    def bind_queue(queue, routing_keys)
+      unbind_redundant_bindings(queue, routing_keys)
+
+      # Ensure all the desired bindings are present
+      routing_keys.each do |routing_key|
+        logger.debug "creating binding #{queue.name} <--> #{routing_key}"
+        queue.bind(exchange, routing_key: routing_key)
+      end
+    end
+
+    def stop
+      if defined?(JRUBY_VERSION)
+        channel.close
+      else
+        # Enqueue a failing job that kills the consumer loop
+        channel_work_pool.shutdown
+        # Give `timeout` seconds to jobs that are still being processed
+        channel_work_pool.join(@config[:graceful_exit_timeout])
+        # If after `timeout` they are still running, they are killed
+        channel_work_pool.kill
+      end
+    end
+
+    def requeue(delivery_tag)
+      channel.reject(delivery_tag, true)
+    end
+
+    def reject(delivery_tag, requeue=false)
+      channel.reject(delivery_tag, requeue)
+    end
+
+    def ack(delivery_tag)
+      channel.ack(delivery_tag, false)
+    end
+
+    def nack(delivery_tag)
+      channel.nack(delivery_tag, false, false)
+    end
+
+    def publish(*args)
+      @publisher.publish(*args)
+    end
+
+    def confirm_select(*args)
+      channel.confirm_select(*args)
+    end
+
+    def wait_for_confirms
+      channel.wait_for_confirms
+    end
+
+    # @return [Boolean] True if channel is set up to use publisher confirmations.
+    def using_publisher_confirmations?
+      channel.using_publisher_confirmations?
+    end
+
+    private
+
+    def api_config
+      @api_config ||= OpenStruct.new.tap do |config|
+        config.host = @config[:mq_api_host]
+        config.port = @config[:mq_api_port]
+        config.username = @config[:mq_username]
+        config.password = @config[:mq_password]
+        config.ssl = @config[:mq_api_ssl]
+        config.protocol = config.ssl ? "https://" : "http://"
+        config.sanitized_uri = "#{config.protocol}#{config.username}@#{config.host}:#{config.port}/"
+      end
+    end
+
+    def connection_params
+      parse_uri
+
+      {}.tap do |params|
+        params[:host]               = @config[:mq_host]
+        params[:port]               = @config[:mq_port]
+        params[:vhost]              = @config[:mq_vhost].presence || Hutch::Adapter::DEFAULT_VHOST
+        params[:auth_mechanism]     = @config[:mq_auth_mechanism]
+        params[:username]           = @config[:mq_username]
+        params[:password]           = @config[:mq_password]
+        params[:tls]                = @config[:mq_tls]
+        params[:tls_key]            = @config[:mq_tls_key]
+        params[:tls_cert]           = @config[:mq_tls_cert]
+        params[:verify_peer]        = @config[:mq_verify_peer]
+        if @config[:mq_tls_ca_certificates]
+          params[:tls_ca_certificates] = @config[:mq_tls_ca_certificates]
+        end
+        params[:heartbeat]          = @config[:heartbeat]
+        params[:connection_timeout] = @config[:connection_timeout]
+        params[:read_timeout]       = @config[:read_timeout]
+        params[:write_timeout]      = @config[:write_timeout]
+
+
+        params[:automatically_recover] = @config[:automatically_recover]
+        params[:network_recovery_interval] = @config[:network_recovery_interval]
+
+        params[:logger] = @config[:client_logger] if @config[:client_logger]
+      end
+    end
+
+    def parse_uri
+      return if @config[:uri].blank?
+
+      u = URI.parse(@config[:uri])
+
+      @config[:mq_tls]      = u.scheme == 'amqps'
+      @config[:mq_host]     = u.host
+      @config[:mq_port]     = u.port || default_mq_port
+      @config[:mq_vhost]    = u.path.sub(/^\//, "")
+      @config[:mq_username] = u.user
+      @config[:mq_password] = u.password
+    end
+
+    def default_mq_port
+      @config[:mq_tls] ? DEFAULT_AMQPS_PORT : DEFAULT_AMQP_PORT
+    end
+
+    def sanitized_uri
+      p = connection_params
+      scheme = p[:tls] ? "amqps" : "amqp"
+
+      "#{scheme}://#{p[:username]}@#{p[:host]}:#{p[:port]}/#{p[:vhost].sub(/^\//, '')}"
+    end
+
+    def with_authentication_error_handler
+      yield
+    rescue Net::HTTPServerException => ex
+      logger.error "HTTP API connection error: #{ex.message.downcase}"
+      if ex.response.code == '401'
+        raise AuthenticationError.new('invalid HTTP API credentials')
+      else
+        raise
+      end
+    end
+
+    def with_connection_error_handler
+      yield
+    rescue Errno::ECONNREFUSED => ex
+      logger.error "HTTP API connection error: #{ex.message.downcase}"
+      raise ConnectionError.new("couldn't connect to HTTP API at #{api_config.sanitized_uri}")
+    end
+
+    def with_bunny_precondition_handler(item)
+      yield
+    rescue Hutch::Adapter::PreconditionFailed => ex
+      logger.error ex.message
+      s = "RabbitMQ responded with 406 Precondition Failed when creating this #{item}. " +
+          "Perhaps it is being redeclared with non-matching attributes"
+      raise WorkerSetupError.new(s)
+    end
+
+    def with_bunny_connection_handler(uri)
+      yield
+    rescue Hutch::Adapter::ConnectionRefused => ex
+      logger.error "amqp connection error: #{ex.message.downcase}"
+      raise ConnectionError.new("couldn't connect to rabbitmq at #{uri}. Check your configuration, network connectivity and RabbitMQ logs.")
+    end
+
+    def channel_work_pool
+      channel.work_pool
+    end
+
+    def consumer_pool_size
+      @config[:consumer_pool_size]
+    end
+
+    def consumer_pool_abort_on_exception
+      @config[:consumer_pool_abort_on_exception]
+    end
+  end
+end