rt-tackle 0.8.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c97d2ff7d67880695b2d97d2ecad9b07abafaca6
+  data.tar.gz: 33cb64ad00cc670d12d675103e2f0a81f1c4ca9b
+SHA512:
+  metadata.gz: 536a9691867f1a5d25f1cc35785f99c777e4172f826bd65e03d42432647301de49ecdb6d02c7000560e3284859141b5e11c5249337ac164f24bb9b640a878f7f
+  data.tar.gz: 3b6f70028efc46b0ce4fda79a0305354ebc749a6a580f7bcf117e2d0e09487629841dad15281b41b18c16233ea3026943a86b9d0489d2ecf0dff907d84b55583

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+vendor/bundle

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in tackle.gemspec
+gemspec

data/README.md

@@ -0,0 +1,68 @@
+# Tackle
+
+[![Build Status](https://semaphoreci.com/api/v1/projects/b39e2ae2-2516-4fd7-9e2c-f5be1a043ff5/732979/badge.svg)](https://semaphoreci.com/renderedtext/tackle)
+
+Tackles the problem of processing asynchronous jobs in reliable manner
+by relying on RabbitMQ.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "rt-tackle", :require => "tackle"
+```
+
+## Usage
+
+### Subscribe consumer:
+
+```ruby
+require "tackle"
+
+options = {
+  :url         => "amqp://localhost", # optional
+  :exchange    => "test-exchange",    # required
+  :routing_key => "test-messages",    # required
+  :queue       => "test-queue",       # required
+  :retry_limit => 8,                  # optional
+  :retry_delay => 30,                 # optional
+  :logger      => Logger.new(STDOUT)  # optional
+}
+
+Tackle.subscribe(options) do |message|
+  # Do something with message
+end
+```
+
+### Publish message:
+
+```ruby
+
+options = {
+  :url         => "amqp://localhost", # optional
+  :exchange    => "test-exchange",    # required
+  :routing_key => "test-messages",    # required
+  :logger      => Logger.new(STDOUT)  # optional
+}
+
+Tackle.publish("Hello, world!", options)
+```
+
+## Development
+
+
+After checking out the repo, run `bin/setup` to install dependencies. Then,
+run `rake rspec` to run the tests. You can also run `bin/console` for an
+interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+To release a new version, update the version number in `version.rb`, and
+then run `bundle exec rake release`, which will create a git tag for the
+version, push git commits and tags, and push the `.gem` file
+to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/renderedtext/tackle.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "tackle"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/tackle/delayed_retry.rb

@@ -0,0 +1,36 @@
+require "tackle/tackle_logger"
+
+module Tackle
+
+  class DelayedRetry
+    include Tackle::TackleLogger
+
+    def initialize(dead_letter_queue, properties, payload, retry_limit, logger)
+      @dead_letter_queue = dead_letter_queue
+      @properties = properties
+      @payload = payload
+      @retry_limit = retry_limit
+      @logger = logger
+    end
+
+    def schedule_retry
+      if retry_count < @retry_limit
+        tackle_log("Adding message to retry queue for retry #{retry_count + 1}/#{@retry_limit}")
+        @dead_letter_queue.publish(@payload, :headers => {:retry_count => retry_count + 1})
+      else
+        tackle_log("Reached #{retry_count} retries. Discarding message.")
+      end
+    end
+
+    private
+
+    def retry_count
+      if @properties.headers && @properties.headers["retry_count"]
+        @properties.headers["retry_count"]
+      else
+        0
+      end
+    end
+
+  end
+end

data/lib/tackle/publisher.rb

@@ -0,0 +1,50 @@
+module Tackle
+  class Publisher
+    include Tackle::TackleLogger
+
+    def initialize(exchange_name, routing_key, url, logger)
+      @exchange_name = exchange_name
+      @routing_key = routing_key
+      @url = url
+      @logger = logger
+    end
+
+    def publish(message)
+      tackle_log("Publishing message started exchange='#{@exchange_name}' routing_key='#{@routing_key}'")
+
+      with_rabbit_connection do |conn|
+        channel = conn.create_channel
+        tackle_log("Created a communication channel")
+
+        exchange = channel.direct(@exchange_name, :durable => true)
+        tackle_log("Declared the exchange")
+
+        exchange.publish(message, :routing_key => @routing_key, :persistent => true)
+      end
+
+      tackle_log("Publishing message finished exchange='#{@exchange_name}' routing_key='#{@routing_key}'")
+    end
+
+    private
+
+    def with_rabbit_connection
+      tackle_log("Establishing rabbit connection")
+
+      conn = Bunny.new(@url)
+      conn.start
+
+      yield(conn)
+
+      tackle_log("Established rabbit connection")
+    rescue StandardError => ex
+      tackle_log("An exception occured while sending the message exception='#{ex.class.name}' message='#{ex.message}'")
+
+      raise ex
+    ensure
+      tackle_log("Clossing rabbit connection")
+
+      conn.close if conn
+    end
+
+  end
+end

data/lib/tackle/rabbit.rb

@@ -0,0 +1,87 @@
+require "bunny"
+require "tackle/tackle_logger"
+
+module Tackle
+
+  class Rabbit
+    include Tackle::TackleLogger
+
+    attr_reader :channel, :dead_letter_queue, :queue, :exchange
+
+    def initialize(exchange_name, routing_key, queue_name, amqp_url, retry_delay, logger)
+      @exchange_name = exchange_name
+      @routing_key = routing_key
+      @queue_name = queue_name
+      @amqp_url = amqp_url
+      @retry_delay = retry_delay
+      @logger = logger
+    end
+
+    def connect
+      @conn = Bunny.new(@amqp_url)
+      @conn.start
+      tackle_log("Connected to RabbitMQ")
+
+      @channel = @conn.create_channel
+      @channel.prefetch(1)
+
+      tackle_log("Connected to channel")
+      connect_queue
+      connect_dead_letter_queue
+    rescue StandardError => ex
+      tackle_log("An exception occured while connecting to the server message='#{ex.message}'")
+
+      raise ex
+    end
+
+    def on_uncaught_exception(blk)
+      @channel.on_uncaught_exception(&blk)
+    end
+
+    def close
+      @channel.close
+      tackle_log("Closed channel")
+      @conn.close
+      tackle_log("Closed connection to RabbitMQ")
+    end
+
+    def dead_letter_exchange_name
+      "#{@exchange_name}.dead_letter_exchange"
+    end
+
+    def dead_letter_queue_name
+      "#{@queue_name}_dead_letters"
+    end
+
+    private
+
+    def connect_queue
+      @exchange = @channel.direct(@exchange_name, :durable => true)
+      tackle_log("Connected to exchange '#{@exchange_name}'")
+
+      @queue = @channel.queue(@queue_name, :durable => true).bind(@exchange, :routing_key => @routing_key)
+
+      tackle_log("Connected to queue '#{@queue_name}' with routing key '#{@routing_key}'")
+    end
+
+    def connect_dead_letter_queue
+      tackle_log("Connected to dead letter exchange '#{dead_letter_exchange_name}'")
+
+      dead_letter_exchange = @channel.fanout(dead_letter_exchange_name)
+
+      queue_options = {
+        :durable => true,
+        :arguments => {
+          "x-dead-letter-exchange" => @exchange_name,
+          "x-dead-letter-routing-key" => @routing_key,
+          "x-message-ttl" => @retry_delay
+        }
+      }
+
+      @dead_letter_queue = @channel.queue(dead_letter_queue_name, queue_options).bind(dead_letter_exchange, :routing_key => @routing_key)
+
+      tackle_log("Connected to dead letter queue '#{dead_letter_queue_name}'")
+    end
+
+  end
+end

data/lib/tackle/tackle_logger.rb

@@ -0,0 +1,13 @@
+module Tackle
+
+  module TackleLogger
+
+    def tackle_log(message)
+      pid = Process.pid
+
+      whole_message = "tackle - pid=#{pid} message=#{message}"
+
+      @logger.info(whole_message)
+    end
+  end
+end

data/lib/tackle/version.rb

@@ -0,0 +1,3 @@
+module Tackle
+  VERSION = "0.8.0"
+end

data/lib/tackle/worker.rb

@@ -0,0 +1,86 @@
+require "tackle/rabbit"
+require "tackle/delayed_retry"
+
+module Tackle
+  class Worker
+    include Tackle::TackleLogger
+
+    attr_reader :rabbit
+
+    # Initializes now worker
+    #
+    # @param [String] exchange_name Name of the exchange queue is connected to.
+    # @param [String] routing_key Routing key for binding queue to exchange
+    # @param [String] queue_name Name of the queue worker is processing.
+    # @param [Hash] options Worker options for RabbitMQ connection, retries and logger.
+    #
+    # @option options [String] :url AMQP connection url. Defaults to 'localhost'
+    # @option options [Integer] :retry_limit Number of times message processing should be retried in case of an exception.
+    # @option options [Integer] :retry_delay Delay between processing retries. Dafaults to 30 seconds. Cannot be changed without deleting or renameing a queue.
+    # @option options [Logger] :logger Logger instance. Defaults to standard output.
+    #
+    # @api public
+    def initialize(exchange_name, routing_key, queue_name, options = {})
+      @queue_name = queue_name
+      @amqp_url = options[:url] || "amqp://localhost:5672"
+      @retry_limit = options[:retry_limit] || 8
+      @retry_delay = (options[:retry_delay] || 30) * 1000 #ms
+      @logger = options[:logger] || Logger.new(STDOUT)
+
+      @rabbit = Tackle::Rabbit.new(exchange_name,
+                                   routing_key,
+                                   @queue_name,
+                                   @amqp_url,
+                                   @retry_delay,
+                                   @logger)
+
+      @rabbit.connect
+      @rabbit.on_uncaught_exception(options[:on_uncaught_exception]) if options[:on_uncaught_exception]
+    end
+
+    # Subscribes for message deliveries
+    #
+    # @param [Block] Accepts a block that accepts message
+    #
+    # @api public
+    def subscribe(&block)
+      tackle_log("Subscribing to queue '#{@queue_name}'...")
+      rabbit.queue.subscribe(:manual_ack => true,
+                             :block => true) do |delivery_info, properties, payload|
+
+        tackle_log("Received message. Processing...")
+        process_message(delivery_info, properties, payload, block)
+        tackle_log("Done with processing message.")
+
+      end
+    rescue Interrupt => _
+      rabbit.close
+    rescue StandardError => ex
+      tackle_log("An exception occured message='#{ex.message}'")
+
+      raise ex
+    end
+
+    def process_message(delivery_info, properties, payload, block)
+      begin
+        tackle_log("Calling message processor...")
+        block.call(payload)
+        @rabbit.channel.ack(delivery_info.delivery_tag)
+        tackle_log("Successfully processed message")
+      rescue Exception => ex
+        tackle_log("Failed to process message. Received exception '#{ex}'")
+        try_again = Tackle::DelayedRetry.new(@rabbit.dead_letter_queue,
+                                             properties,
+                                             payload,
+                                             @retry_limit,
+                                             @logger)
+        try_again.schedule_retry
+        tackle_log("Sending negative acknowledgement to source queue...")
+        @rabbit.channel.nack(delivery_info.delivery_tag)
+        tackle_log("Negative acknowledgement sent")
+
+        raise ex
+      end
+    end
+  end
+end

data/lib/tackle.rb

@@ -0,0 +1,45 @@
+require "tackle/version"
+
+module Tackle
+  require "tackle/worker"
+  require "tackle/publisher"
+
+  def self.subscribe(options = {}, &block)
+    # required
+    exchange_name = options.fetch(:exchange)
+    routing_key   = options.fetch(:routing_key)
+    queue_name    = options.fetch(:queue)
+
+    # optional
+    amqp_url    = options[:url]
+    retry_limit = options[:retry_limit]
+    retry_delay = options[:retry_delay]
+    logger      = options[:logger]
+    on_uncaught_exception = options[:on_uncaught_exception]
+
+    worker = Tackle::Worker.new(exchange_name,
+                                routing_key,
+                                queue_name,
+                                :url => amqp_url,
+                                :retry_limit => retry_limit,
+                                :retry_delay => retry_delay,
+                                :logger => logger,
+                                :on_uncaught_exception => on_uncaught_exception)
+
+    worker.subscribe(&block)
+  end
+
+  def self.publish(message, options = {})
+    # required
+    exchange_name = options.fetch(:exchange)
+    routing_key   = options.fetch(:routing_key)
+
+    # optional
+    amqp_url    = options[:url] || "amqp://localhost:5672"
+    logger      = options[:logger] || Logger.new(STDOUT)
+
+    publisher = Tackle::Publisher.new(exchange_name, routing_key, amqp_url, logger)
+
+    publisher.publish(message)
+  end
+end

data/reset_rabbit.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+sudo rabbitmqctl delete_vhost '/'
+sudo rabbitmqctl stop_app
+sudo rabbitmqctl reset
+sudo rabbitmqctl start_app

data/tackle.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'tackle/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rt-tackle"
+  spec.version       = Tackle::VERSION
+  spec.licenses      = ['MIT']
+  spec.authors       = ["Rendered Text"]
+  spec.email         = ["devops@renderedtext.com"]
+
+  spec.summary       = %q{RabbitMQ based single-thread worker}
+  spec.description   = %q{RabbitMQ based single-thread worker}
+  spec.homepage      = "https://semaphoreci.com"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency             "bunny"
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "byebug"
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: rt-tackle
+version: !ruby/object:Gem::Version
+  version: 0.8.0
+platform: ruby
+authors:
+- Rendered Text
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-09-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: RabbitMQ based single-thread worker
+email:
+- devops@renderedtext.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/tackle.rb
+- lib/tackle/delayed_retry.rb
+- lib/tackle/publisher.rb
+- lib/tackle/rabbit.rb
+- lib/tackle/tackle_logger.rb
+- lib/tackle/version.rb
+- lib/tackle/worker.rb
+- reset_rabbit.sh
+- tackle.gemspec
+homepage: https://semaphoreci.com
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: RabbitMQ based single-thread worker
+test_files: []