rabbitek 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 48e1e5b2bc2ea1d5a8de3e5e6e35ec5f8ef3e6961a99987f544f87dfc043d48f
+  data.tar.gz: b413bb3111ab56d0cf135bc5af3c6a754c675d945cf5919932881e57f31c6d94
+SHA512:
+  metadata.gz: 04dc3c2f72c7157d9d7eb987aa91c9e97fcbd588c2a06079e37db8fba3a5d2ef202cb52337cc9b7065d4ec711e1976a132efbd53dce5fba8f01a037b1afe9585
+  data.tar.gz: c5b4d5946c8ebf3a8f374de46ec73ce40504d32a0fed75329fc568e027b12697d19c2c2dab1562da03a9ea60abce242c9ebe907c96a8f66b5f9963fbcff0aced

data/.codeclimate.yml

@@ -0,0 +1,4 @@
+plugins:
+  rubocop:
+    enabled: true
+    channel: rubocop-0-58

data/.gitignore

@@ -0,0 +1,58 @@
+# Created by https://www.gitignore.io/api/ruby
+
+### Ruby ###
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+
+# End of https://www.gitignore.io/api/ruby
+
+.idea

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,19 @@
+AllCops:
+  Exclude:
+  - 'vendor/**/*'
+  - 'spec/fixtures/**/*'
+  - 'tmp/**/*'
+  TargetRubyVersion: 2.4
+
+Metrics/LineLength:
+  Max: 120
+
+Metrics/BlockLength:
+  Exclude:
+  - 'rabbitek.gemspec'
+  - 'Rakefile'
+  - '**/*.rake'
+  - 'spec/**/*.rb'
+
+Metrics/MethodLength:
+  Max: 15

data/.travis.yml

@@ -0,0 +1,7 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.0
+  - 2.4.0
+  - 2.5.1
+before_install: gem install bundler -N

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in rabbitek.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,76 @@
+PATH
+  remote: .
+  specs:
+    rabbitek (0.1.0)
+      activesupport (> 3.0)
+      bunny (~> 2.11.0)
+      oj (~> 3.6)
+      opentracing (~> 0.4)
+      slop (~> 4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    amq-protocol (2.3.0)
+    ast (2.4.0)
+    bunny (2.11.0)
+      amq-protocol (~> 2.3.0)
+    concurrent-ruby (1.0.5)
+    diff-lcs (1.3)
+    i18n (1.1.1)
+      concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.1)
+    minitest (5.11.3)
+    oj (3.7.0)
+    opentracing (0.4.3)
+    parallel (1.12.1)
+    parser (2.5.1.2)
+      ast (~> 2.4.0)
+    powerpack (0.1.2)
+    rainbow (3.0.0)
+    rake (10.5.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
+    rubocop (0.58.2)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.10.0)
+    slop (4.6.2)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    unicode-display_width (1.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  rabbitek!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rubocop (~> 0.58.0)
+
+BUNDLED WITH
+   1.16.4

data/LICENSE.txt

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Boostcom
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,97 @@
+# Rabbitek
+
+High performance, easy to use background job processing library for Ruby using RabbitMQ queues.
+
+* Consumers (servers) are configurable via yaml files.
+* Retries
+* Delayed jobs
+* Jobs priority (through RabbitMQ Priority Queues)
+* Scalable and failure-safe
+* Client & Server hooks
+* OpenTracing (http://opentracing.io/) instrumentation
+* NewRelic instrumentation for sending errors
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rabbitek'
+```
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+First, you need configuration file:
+
+```yaml
+# config/rabbitek.yml
+consumers:
+  - FirstConsumer
+  - SecondConsumer  
+threads: 25
+parameters:
+  queue: myqueue
+  basic_qos: 20
+  queue_attributes:
+    arguments:
+      x-max-priority: 10
+```
+
+Create consumer and include `Rabbitek::Consumer`, add `rabbit_options` with path to config file
+and create `perform` method same way as on example:
+
+```
+  class ExampleConsumer
+    include Rabbitek::Consumer
+
+    rabbit_options config_file: 'config/rabbitek.yml'
+
+    def perform(payload, delivery_info, properties)
+      ack!(delivery_info)
+    end
+  end
+```
+
+Lastly, run server:
+
+```
+bundle exec rabbitek
+```
+
+You can schedule jobs e.g.: `ExampleCustomer.perform_async(some: :payload)`
+
+
+## Roadmap
+
+* more tests!
+* dead queue
+* CRON jobs
+* job batching (run consumer with e.g. max 100 messages at once)
+* extended docs and how to
+* prometheus metrics
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/Boostcom/rabbitek.
+
+## License
+
+Please see [LICENSE.txt](LICENSE.txt)
+
+## Author
+
+![Boostcom](boostcom-logo.png)
+
+**[Boostcom](https://boostcom.com/)** - we provide the most powerful management- and loyalty platform built for the needs of shopping centres.
+

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+task default: :spec

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'rabbitek'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/rabbitek

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/rabbitek/cli'
+
+begin
+  Rabbitek::CLI.new.run
+rescue StandardError => e
+  raise(e) if $DEBUG
+  STDERR.puts(e.message)
+  STDERR.puts(e.backtrace.join("\n"))
+  exit(1)
+end

data/lib/rabbitek/bunny_connection.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  ##
+  # Bunny connection setup
+  class BunnyConnection
+    def self.initialize_connection
+      connection = Bunny.new(Rabbitek.config.bunny_configuration)
+      connection.start
+      connection
+    end
+  end
+end

data/lib/rabbitek/cli/signal_handlers.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  class CLI
+    ##
+    # OS signal handlers
+    class SignalHandlers
+      SIGNALS = {
+        INT: :shutdown,
+        TERM: :shutdown
+      }.freeze
+
+      def self.setup(io_w)
+        SIGNALS.each do |signal, hook|
+          Signal.trap(signal) { io_w.write("#{hook}\n") }
+        end
+      end
+
+      def self.shutdown
+        raise Interrupt
+      end
+    end
+  end
+end

data/lib/rabbitek/cli.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'slop'
+require 'yaml'
+
+require_relative './cli/signal_handlers'
+require 'rabbitek'
+
+module Rabbitek
+  ##
+  # Rabbitek server CLI
+  class CLI
+    include Loggable
+
+    def run
+      opts
+      require_application
+      map_consumer_workers!
+
+      start_log
+
+      io_r, io_w = IO.pipe
+      SignalHandlers.setup(io_w)
+
+      begin
+        consumers = boot_consumers
+
+        while io_resp = IO.select([io_r]) # rubocop:disable Lint/AssignmentInCondition
+          SignalHandlers.public_send(io_resp.first.first.gets.strip)
+        end
+      rescue Interrupt
+        execute_shutdown(consumers)
+      end
+    end
+
+    private
+
+    def start_log # rubocop:disable Metrics/AbcSize
+      info "Rabbit consumers '[#{configuration[:workers].map(&:to_s).join(', ')}]' started with PID #{Process.pid}"
+      info "Client hooks: [#{Rabbitek.config.client_hooks.map(&:class).join(', ')}]"
+      info "Server hooks: [#{Rabbitek.config.server_hooks.map(&:class).join(', ')}]"
+    end
+
+    def opts
+      @opts ||= Slop.parse do |o|
+        o.string '-c', '--config', 'config file path. Default: "config/rabbitek.yaml"', default: 'config/rabbitek.yml'
+        o.string '-r', '--require', 'file to require while booting. Default: "config/environment.rb"',
+                 default: 'config/environment.rb'
+        o.on '--version', 'print the version' do
+          puts VERSION
+          exit
+        end
+        o.on '-h', '--help' do
+          puts o
+          exit
+        end
+      end
+    end
+
+    def configuration
+      @configuration ||= YAML.load_file(opts[:config]).with_indifferent_access
+    end
+
+    def require_application
+      require File.expand_path(opts[:require])
+    end
+
+    def map_consumer_workers!
+      configuration[:workers].map!(&:constantize)
+    end
+
+    def boot_consumers
+      (1..configuration[:threads]).each_with_object([]) do |_, arr|
+        arr << Starter.new(Rabbitek.bunny_connection, configuration).start
+      end
+    end
+
+    def execute_shutdown(consumers)
+      info 'Shutting down gracefully...'
+
+      consumers.map(&:cancel)
+      Rabbitek.close_bunny_connection
+      info 'Graceful shutdown completed'
+
+      exit(0)
+    end
+  end
+end

data/lib/rabbitek/client/client_hook.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  ##
+  # Base client hook class
+  class ClientHook
+    def call(payload, params)
+      yield(payload, params)
+    end
+  end
+end

data/lib/rabbitek/client/hooks/opentracing.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative '../client_hook'
+
+module Rabbitek
+  module Client
+    module Hooks
+      ##
+      # OpenTracing client hook
+      class OpenTracing < Rabbitek::ClientHook
+        def call(payload, params)
+          result = nil
+
+          ::OpenTracing.start_active_span(params[:routing_key], opentracing_options(params)) do |scope|
+            begin
+              params[:headers] ||= {}
+              Utils::OpenTracing.inject!(scope.span, params[:headers])
+
+              result = super
+            rescue StandardError => e
+              raise unless scope.span
+
+              Utils::OpenTracing.log_error(scope.span, e)
+              raise
+            end
+          end
+
+          result
+        end
+
+        def opentracing_options(params)
+          Utils::OpenTracing.client_options(params)
+        end
+      end
+    end
+  end
+end

data/lib/rabbitek/client/publisher.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  ##
+  # Handles publishing message to RabbitMQ
+  class Publisher
+    def initialize(exchange_name, exchange_type: 'direct', channel: nil)
+      @channel = channel || Rabbitek.create_channel
+      @exchange_name = exchange_name
+      @exchange_type = exchange_type
+    end
+
+    def publish(payload, params = {})
+      Utils::HookWalker.new(Rabbitek.config.client_hooks).call!(payload, params) do |parsed_payload, parsed_params|
+        exchange.publish(Utils::Oj.dump(parsed_payload), parsed_params)
+      end
+    end
+
+    def close
+      @channel.close
+    end
+
+    def exchange
+      @exchange ||= Utils::Common.exchange(@channel, @exchange_type, @exchange_name)
+    end
+  end
+end

data/lib/rabbitek/config.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  ##
+  # Rabbitek configuration
+  class Config
+    DEFAULTS = {
+      bunny_configuration: { hosts: 'localhost:5672', vhost: '/' },
+      log_format: 'json',
+      enable_newrelic: true
+    }.freeze
+
+    attr_accessor(*DEFAULTS.keys)
+
+    def initialize
+      DEFAULTS.each { |k, v| public_send("#{k}=", v) }
+
+      @client_hooks_config = []
+      @server_hooks_config = []
+    end
+
+    def add_client_hook(hook_object)
+      @client_hooks_config << hook_object
+    end
+
+    def add_server_hook(hook_object)
+      @server_hooks_config << hook_object
+    end
+
+    def client_hooks
+      @client_hooks ||= begin
+        @client_hooks_config << Client::Hooks::OpenTracing.new
+        @client_hooks_config.reverse
+      end
+    end
+
+    def server_hooks
+      @server_hooks ||= begin
+        @server_hooks_config.unshift(Server::Hooks::TimeTracker.new)
+        @server_hooks_config.push(Server::Hooks::OpenTracing.new, Server::Hooks::Retry.new)
+        @server_hooks_config.reverse
+      end
+    end
+  end
+end

data/lib/rabbitek/loggable.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  ##
+  # Log helpers
+  module Loggable
+    def self.logger
+      Rabbitek.logger
+    end
+
+    def logger
+      Rabbitek.logger
+    end
+
+    def error(msg)
+      log_msg(:error, msg)
+      NewRelic::Agent.notice_error(msg) if Rabbitek.config.enable_newrelic && Object.const_defined?('NewRelic')
+      true
+    end
+
+    def warn(msg)
+      log_msg(:warn, msg)
+    end
+
+    def debug(msg)
+      log_msg(:debug, msg)
+    end
+
+    def info(msg)
+      log_msg(:info, msg)
+    end
+
+    private
+
+    def log_msg(severity, msg)
+      if logger.respond_to?(:tagged)
+        logger.tagged(class_name(msg)) { logger.send(severity, msg) }
+      else
+        logger.send(severity, msg)
+      end
+
+      true
+    end
+
+    def class_name(msg)
+      msg.is_a?(Hash) && msg[:class_name] ? msg.delete(:class_name) : self.class.name
+    end
+  end
+end