twingly-amqp 3.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a9fa7fea2188c1f01d9707114404ff18fb6ac3ec
+  data.tar.gz: a63a38b0be52d72241b69788fdeef5c489d5fd34
+SHA512:
+  metadata.gz: dab3d09f43682c6c3b580ad754113b58d5d582551a997931665988cd4abe5879257202f1f2d33d5a3fc0441d71d825b000c69fa3259428ff52d147e7d59dae87
+  data.tar.gz: 2e48a940d9361acc7e76e099a040d4956a679bd30b12312da4e7afff5497441c03a793935ceec7a09b290e6babf12096b696b1523221360545c281ccaaef4326

data/README.md

@@ -0,0 +1,122 @@
+# Twingly::AMQP
+
+[![Build Status](https://travis-ci.org/twingly/twingly-amqp.svg?branch=master)](https://travis-ci.org/twingly/twingly-amqp)
+
+
+A gem for subscribing and publishing messages via RabbitMQ.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "twingly-amqp", :git => "git@github.com:twingly/twingly-amqp.git"
+```
+
+## Usage
+
+Environment variables:
+
+* `RABBITMQ_N_HOST`
+* `AMQP_USERNAME`
+* `AMQP_PASSWORD`
+* `AMQP_TLS` # Use TLS connection if set
+
+### Customize options
+
+If you don't have the RabbitMQ hosts in your ENV you can set them with `Twingly::AMQP::Connection.options=` before you create an instance of `Subscription` or `Ping`.
+
+```ruby
+Twingly::AMQP::Connection.options = {
+  hosts: "localhost",
+}
+```
+
+### Subscribe to a queue
+
+```ruby
+subscription = Twingly::AMQP::Subscription.new(
+  queue_name:       "crawler-urls",
+  exchange_topic:   "url-exchange", # Optional, uses the default exchange if omitted
+  routing_key:      "url.blog",     # Optional, uses the default exchange if omitted
+  consumer_threads: 4,              # Optional
+  prefetch:         20,             # Optional
+)
+
+subscription.on_exception { |exception| puts "Oh noes! #{exception.message}" }
+subscription.before_handle_message { |raw_message_payload| puts raw_message }
+
+subscription.subscribe do |message| # An instance of Twingly::AMQP::Message
+  begin
+    response = client.post(payload.fetch(:url))
+
+    case response.code
+    when 200 then message.ack     # No error
+    when 404 then message.reject  # Permanent error, discard
+    when 500 then message.requeue # Transient error, retry
+    end
+  rescue
+    # It's up to the client to handle all exceptions
+    message.reject                # Unknown error, discard
+  end
+end
+```
+
+### Ping urls
+
+```ruby
+pinger = Twingly::AMQP::Ping.new(
+  provider_name: "a-provider-name",
+  queue_name:    "provider-ping",
+  priority:      1,
+  source_ip:     "?.?.?.?", # Optional, can be given to #ping
+  url_cache:     url_cache, # Optional, see below
+)
+
+urls = [
+  "http://blog.twingly.com",
+]
+
+pinger.ping(urls) do |pinged_url|
+  # Optional block that gets called for each pinged url
+end
+
+# Send a ping using another source ip
+pinger.ping(urls, source_ip: "1.2.3.4")
+```
+
+#### Url cache
+
+`Twingly::AMQP::Ping.new` can optionally take an url cache which caches the urls and only pings in the urls that isn't already cached. The cache needs to respond to the two following methods:
+
+```ruby
+class UrlCache
+  def cached?(url)
+    # return true/false
+  end
+
+  def cache!(url)
+    # cache url
+  end
+end
+```
+
+## Tests
+
+The tests require a local RabbitMQ server to run.
+
+Run tests with
+
+```shell
+bundle exec rake
+```
+
+## Release workflow
+
+**Note**: Make sure you are logged in as [twingly][twingly-rubygems] at RubyGems.org.
+
+Build and [publish](http://guides.rubygems.org/publishing/) the gem.
+
+    bundle exec rake release
+
+[twingly-rubygems]: https://rubygems.org/profiles/twingly

data/lib/twingly/amqp.rb

@@ -0,0 +1,8 @@
+require "twingly/amqp/version"
+
+ENV["RUBY_ENV"] ||= "development"
+
+require "twingly/amqp/session"
+require "twingly/amqp/connection"
+require "twingly/amqp/subscription"
+require "twingly/amqp/ping"

data/lib/twingly/amqp/connection.rb

@@ -0,0 +1,26 @@
+require "twingly/amqp/session"
+require "bunny"
+
+module Twingly
+  module AMQP
+    class Connection
+      private_class_method :new
+
+      @@lock     = Mutex.new
+      @@instance = nil
+      @@options  = {}
+
+      def self.options=(options)
+        @@options = options
+      end
+
+      def self.instance
+        return @@instance if @@instance
+        @@lock.synchronize do
+          return @@instance if @@instance
+          @@instance = Session.new(@@options).connection
+        end
+      end
+    end
+  end
+end

data/lib/twingly/amqp/message.rb

@@ -0,0 +1,34 @@
+require "json"
+
+module Twingly
+  module AMQP
+    class Message
+      attr_reader :delivery_info, :metadata, :payload
+
+      def initialize(delivery_info:, metadata:, payload:, channel:)
+        @delivery_info = delivery_info
+        @metadata      = metadata
+        @payload       = parse_payload(payload)
+        @channel       = channel
+      end
+
+      def ack
+        @channel.ack(@delivery_info.delivery_tag)
+      end
+
+      def requeue
+        @channel.reject(@delivery_info.delivery_tag, true)
+      end
+
+      def reject
+        @channel.reject(@delivery_info.delivery_tag, false)
+      end
+
+      private
+
+      def parse_payload(payload)
+        JSON.parse(payload, symbolize_names: true)
+      end
+    end
+  end
+end

data/lib/twingly/amqp/ping.rb

@@ -0,0 +1,77 @@
+require "twingly/amqp/connection"
+require "json"
+
+module Twingly
+  module AMQP
+    class Ping
+      def initialize(provider_name:, queue_name:, priority:, source_ip: nil, url_cache: NullCache, connection: nil)
+        @url_cache = url_cache
+
+        @provider_name = provider_name
+        @queue_name    = queue_name
+        @source_ip     = source_ip
+        @priority      = priority
+
+        connection ||= Connection.instance
+        @channel = connection.create_channel
+      end
+
+      def ping(urls, options = {})
+        Array(urls).each do |url|
+          unless cached?(url)
+            publish(url, options)
+            cache!(url)
+
+            yield url if block_given?
+          end
+        end
+      end
+
+      private
+
+      def publish(url, options)
+        payload = message(url, options).to_json
+        @channel.default_exchange.publish(payload, amqp_publish_options)
+      end
+
+      def amqp_publish_options
+        {
+          key: @queue_name,
+          persistent: true,
+          content_type: "application/json",
+        }
+      end
+
+      def message(url, options)
+        source_ip = options.fetch(:source_ip) { @source_ip }
+        raise ArgumentError, ":source_ip not specified" unless source_ip
+
+        {
+          automatic_ping: false,
+          provider_name: @provider_name,
+          priority: @priority,
+          source_ip: source_ip,
+          url: url,
+        }
+      end
+
+      def cached?(url)
+        @url_cache.cached?(url)
+      end
+
+      def cache!(url)
+        @url_cache.cache!(url)
+      end
+
+      class NullCache
+        def self.cached?(url)
+          false
+        end
+
+        def self.cache!(url)
+          # Do nothing
+        end
+      end
+    end
+  end
+end

data/lib/twingly/amqp/session.rb

@@ -0,0 +1,45 @@
+module Twingly
+  module AMQP
+    class Session
+      attr_reader :connection, :hosts
+
+      def initialize(hosts: nil)
+        @hosts      = hosts || hosts_from_env
+        @connection = create_connection
+      end
+
+      private
+
+      def create_connection
+        if ruby_env == "development"
+          connection = Bunny.new
+        else
+          connection_options = {
+            hosts: hosts,
+            user: ENV.fetch("AMQP_USERNAME"),
+            pass: ENV.fetch("AMQP_PASSWORD"),
+            recover_from_connection_close: true,
+            tls: tls?,
+          }
+          connection = Bunny.new(connection_options)
+        end
+        connection.start
+        connection
+      end
+
+      def ruby_env
+        ENV.fetch("RUBY_ENV")
+      end
+
+      def tls?
+        ENV.has_key?("AMQP_TLS")
+      end
+
+      def hosts_from_env
+        # Matches env keys like `RABBITMQ_01_HOST`
+        environment_keys_with_host = ENV.keys.select { |key| key =~ /^rabbitmq_\d+_host$/i }
+        environment_keys_with_host.map { |key| ENV[key] }
+      end
+    end
+  end
+end

data/lib/twingly/amqp/subscription.rb

@@ -0,0 +1,119 @@
+require "twingly/amqp/connection"
+require "twingly/amqp/message"
+
+module Twingly
+  module AMQP
+    class Subscription
+      def initialize(queue_name:, exchange_topic: nil, routing_key: nil, consumer_threads: 4, prefetch: 20, connection: nil)
+        @queue_name       = queue_name
+        @exchange_topic   = exchange_topic
+        @routing_key      = routing_key
+        @consumer_threads = consumer_threads
+        @prefetch         = prefetch
+
+        connection ||= Connection.instance
+        @channel = create_channel(connection)
+        @queue   = @channel.queue(@queue_name, queue_options)
+
+        if @exchange_topic && @routing_key
+          exchange = @channel.topic(@exchange_topic, durable: true)
+          @queue.bind(exchange, routing_key: @routing_key)
+        end
+
+        @before_handle_message_callback = Proc.new {}
+        @on_exception_callback          = Proc.new {}
+      end
+
+      def subscribe(&block)
+        setup_traps
+
+        consumer = @queue.subscribe(subscribe_options) do |delivery_info, metadata, payload|
+          @before_handle_message_callback.call(payload)
+
+          message = Message.new(
+            delivery_info: delivery_info,
+            metadata:      metadata,
+            payload:       payload,
+            channel:       @channel,
+          )
+
+          block.call(message)
+        end
+
+        # The consumer isn't blocking, so we wait here
+        until cancel? do
+          sleep 0.5
+        end
+
+        puts "Shutting down" if development?
+        consumer.cancel
+      end
+
+      def before_handle_message(&block)
+        @before_handle_message_callback = block
+      end
+
+      def on_exception(&block)
+        @on_exception_callback = block
+      end
+
+      def cancel?
+        @cancel
+      end
+
+      def cancel!
+        @cancel = true
+      end
+
+      private
+
+      def create_channel(connection)
+        channel = connection.create_channel(nil, @consumer_threads)
+        channel.prefetch(@prefetch)
+        channel.on_uncaught_exception do |exception, _|
+          puts exception.message, exception.backtrace if development?
+          @on_exception_callback.call(exception)
+        end
+        channel
+      end
+
+      def development?
+        ruby_env == "development"
+      end
+
+      def ruby_env
+        ENV.fetch("RUBY_ENV")
+      end
+
+      def queue_options
+        {
+          durable: true,
+        }
+      end
+
+      def subscribe_options
+        {
+          manual_ack:   true,
+          consumer_tag: consumer_tag,
+        }
+      end
+
+      def consumer_tag
+        tag_name = [Socket.gethostname, ruby_env].join('-')
+        @channel.generate_consumer_tag(tag_name)
+      end
+
+      def setup_traps
+        [:INT, :TERM].each do |signal|
+          Signal.trap(signal) do
+            # Exit fast if we've already got a signal since before
+            exit!(true) if cancel?
+
+            # Set cancel flag, cancels consumers
+            cancel!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/twingly/amqp/version.rb

@@ -0,0 +1,5 @@
+module Twingly
+  module Amqp
+    VERSION = "3.2.0"
+  end
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: twingly-amqp
+version: !ruby/object:Gem::Version
+  version: 3.2.0
+platform: ruby
+authors:
+- Twingly AB
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Pings urls via RabbitMQ.
+email:
+- support@twingly.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/twingly/amqp.rb
+- lib/twingly/amqp/connection.rb
+- lib/twingly/amqp/message.rb
+- lib/twingly/amqp/ping.rb
+- lib/twingly/amqp/session.rb
+- lib/twingly/amqp/subscription.rb
+- lib/twingly/amqp/version.rb
+homepage: https://github.com/twingly/twingly-amqp
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Ruby library for talking to RabbitMQ.
+test_files: []