twingly-amqp 3.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a9fa7fea2188c1f01d9707114404ff18fb6ac3ec
4
+ data.tar.gz: a63a38b0be52d72241b69788fdeef5c489d5fd34
5
+ SHA512:
6
+ metadata.gz: dab3d09f43682c6c3b580ad754113b58d5d582551a997931665988cd4abe5879257202f1f2d33d5a3fc0441d71d825b000c69fa3259428ff52d147e7d59dae87
7
+ data.tar.gz: 2e48a940d9361acc7e76e099a040d4956a679bd30b12312da4e7afff5497441c03a793935ceec7a09b290e6babf12096b696b1523221360545c281ccaaef4326
data/README.md ADDED
@@ -0,0 +1,122 @@
1
+ # Twingly::AMQP
2
+
3
+ [![Build Status](https://travis-ci.org/twingly/twingly-amqp.svg?branch=master)](https://travis-ci.org/twingly/twingly-amqp)
4
+
5
+
6
+ A gem for subscribing and publishing messages via RabbitMQ.
7
+
8
+ ## Installation
9
+
10
+ Add this line to your application's Gemfile:
11
+
12
+ ```ruby
13
+ gem "twingly-amqp", :git => "git@github.com:twingly/twingly-amqp.git"
14
+ ```
15
+
16
+ ## Usage
17
+
18
+ Environment variables:
19
+
20
+ * `RABBITMQ_N_HOST`
21
+ * `AMQP_USERNAME`
22
+ * `AMQP_PASSWORD`
23
+ * `AMQP_TLS` # Use TLS connection if set
24
+
25
+ ### Customize options
26
+
27
+ If you don't have the RabbitMQ hosts in your ENV you can set them with `Twingly::AMQP::Connection.options=` before you create an instance of `Subscription` or `Ping`.
28
+
29
+ ```ruby
30
+ Twingly::AMQP::Connection.options = {
31
+ hosts: "localhost",
32
+ }
33
+ ```
34
+
35
+ ### Subscribe to a queue
36
+
37
+ ```ruby
38
+ subscription = Twingly::AMQP::Subscription.new(
39
+ queue_name: "crawler-urls",
40
+ exchange_topic: "url-exchange", # Optional, uses the default exchange if omitted
41
+ routing_key: "url.blog", # Optional, uses the default exchange if omitted
42
+ consumer_threads: 4, # Optional
43
+ prefetch: 20, # Optional
44
+ )
45
+
46
+ subscription.on_exception { |exception| puts "Oh noes! #{exception.message}" }
47
+ subscription.before_handle_message { |raw_message_payload| puts raw_message }
48
+
49
+ subscription.subscribe do |message| # An instance of Twingly::AMQP::Message
50
+ begin
51
+ response = client.post(payload.fetch(:url))
52
+
53
+ case response.code
54
+ when 200 then message.ack # No error
55
+ when 404 then message.reject # Permanent error, discard
56
+ when 500 then message.requeue # Transient error, retry
57
+ end
58
+ rescue
59
+ # It's up to the client to handle all exceptions
60
+ message.reject # Unknown error, discard
61
+ end
62
+ end
63
+ ```
64
+
65
+ ### Ping urls
66
+
67
+ ```ruby
68
+ pinger = Twingly::AMQP::Ping.new(
69
+ provider_name: "a-provider-name",
70
+ queue_name: "provider-ping",
71
+ priority: 1,
72
+ source_ip: "?.?.?.?", # Optional, can be given to #ping
73
+ url_cache: url_cache, # Optional, see below
74
+ )
75
+
76
+ urls = [
77
+ "http://blog.twingly.com",
78
+ ]
79
+
80
+ pinger.ping(urls) do |pinged_url|
81
+ # Optional block that gets called for each pinged url
82
+ end
83
+
84
+ # Send a ping using another source ip
85
+ pinger.ping(urls, source_ip: "1.2.3.4")
86
+ ```
87
+
88
+ #### Url cache
89
+
90
+ `Twingly::AMQP::Ping.new` can optionally take an url cache which caches the urls and only pings in the urls that isn't already cached. The cache needs to respond to the two following methods:
91
+
92
+ ```ruby
93
+ class UrlCache
94
+ def cached?(url)
95
+ # return true/false
96
+ end
97
+
98
+ def cache!(url)
99
+ # cache url
100
+ end
101
+ end
102
+ ```
103
+
104
+ ## Tests
105
+
106
+ The tests require a local RabbitMQ server to run.
107
+
108
+ Run tests with
109
+
110
+ ```shell
111
+ bundle exec rake
112
+ ```
113
+
114
+ ## Release workflow
115
+
116
+ **Note**: Make sure you are logged in as [twingly][twingly-rubygems] at RubyGems.org.
117
+
118
+ Build and [publish](http://guides.rubygems.org/publishing/) the gem.
119
+
120
+ bundle exec rake release
121
+
122
+ [twingly-rubygems]: https://rubygems.org/profiles/twingly
@@ -0,0 +1,8 @@
1
+ require "twingly/amqp/version"
2
+
3
+ ENV["RUBY_ENV"] ||= "development"
4
+
5
+ require "twingly/amqp/session"
6
+ require "twingly/amqp/connection"
7
+ require "twingly/amqp/subscription"
8
+ require "twingly/amqp/ping"
@@ -0,0 +1,26 @@
1
+ require "twingly/amqp/session"
2
+ require "bunny"
3
+
4
+ module Twingly
5
+ module AMQP
6
+ class Connection
7
+ private_class_method :new
8
+
9
+ @@lock = Mutex.new
10
+ @@instance = nil
11
+ @@options = {}
12
+
13
+ def self.options=(options)
14
+ @@options = options
15
+ end
16
+
17
+ def self.instance
18
+ return @@instance if @@instance
19
+ @@lock.synchronize do
20
+ return @@instance if @@instance
21
+ @@instance = Session.new(@@options).connection
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,34 @@
1
+ require "json"
2
+
3
+ module Twingly
4
+ module AMQP
5
+ class Message
6
+ attr_reader :delivery_info, :metadata, :payload
7
+
8
+ def initialize(delivery_info:, metadata:, payload:, channel:)
9
+ @delivery_info = delivery_info
10
+ @metadata = metadata
11
+ @payload = parse_payload(payload)
12
+ @channel = channel
13
+ end
14
+
15
+ def ack
16
+ @channel.ack(@delivery_info.delivery_tag)
17
+ end
18
+
19
+ def requeue
20
+ @channel.reject(@delivery_info.delivery_tag, true)
21
+ end
22
+
23
+ def reject
24
+ @channel.reject(@delivery_info.delivery_tag, false)
25
+ end
26
+
27
+ private
28
+
29
+ def parse_payload(payload)
30
+ JSON.parse(payload, symbolize_names: true)
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,77 @@
1
+ require "twingly/amqp/connection"
2
+ require "json"
3
+
4
+ module Twingly
5
+ module AMQP
6
+ class Ping
7
+ def initialize(provider_name:, queue_name:, priority:, source_ip: nil, url_cache: NullCache, connection: nil)
8
+ @url_cache = url_cache
9
+
10
+ @provider_name = provider_name
11
+ @queue_name = queue_name
12
+ @source_ip = source_ip
13
+ @priority = priority
14
+
15
+ connection ||= Connection.instance
16
+ @channel = connection.create_channel
17
+ end
18
+
19
+ def ping(urls, options = {})
20
+ Array(urls).each do |url|
21
+ unless cached?(url)
22
+ publish(url, options)
23
+ cache!(url)
24
+
25
+ yield url if block_given?
26
+ end
27
+ end
28
+ end
29
+
30
+ private
31
+
32
+ def publish(url, options)
33
+ payload = message(url, options).to_json
34
+ @channel.default_exchange.publish(payload, amqp_publish_options)
35
+ end
36
+
37
+ def amqp_publish_options
38
+ {
39
+ key: @queue_name,
40
+ persistent: true,
41
+ content_type: "application/json",
42
+ }
43
+ end
44
+
45
+ def message(url, options)
46
+ source_ip = options.fetch(:source_ip) { @source_ip }
47
+ raise ArgumentError, ":source_ip not specified" unless source_ip
48
+
49
+ {
50
+ automatic_ping: false,
51
+ provider_name: @provider_name,
52
+ priority: @priority,
53
+ source_ip: source_ip,
54
+ url: url,
55
+ }
56
+ end
57
+
58
+ def cached?(url)
59
+ @url_cache.cached?(url)
60
+ end
61
+
62
+ def cache!(url)
63
+ @url_cache.cache!(url)
64
+ end
65
+
66
+ class NullCache
67
+ def self.cached?(url)
68
+ false
69
+ end
70
+
71
+ def self.cache!(url)
72
+ # Do nothing
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,45 @@
1
+ module Twingly
2
+ module AMQP
3
+ class Session
4
+ attr_reader :connection, :hosts
5
+
6
+ def initialize(hosts: nil)
7
+ @hosts = hosts || hosts_from_env
8
+ @connection = create_connection
9
+ end
10
+
11
+ private
12
+
13
+ def create_connection
14
+ if ruby_env == "development"
15
+ connection = Bunny.new
16
+ else
17
+ connection_options = {
18
+ hosts: hosts,
19
+ user: ENV.fetch("AMQP_USERNAME"),
20
+ pass: ENV.fetch("AMQP_PASSWORD"),
21
+ recover_from_connection_close: true,
22
+ tls: tls?,
23
+ }
24
+ connection = Bunny.new(connection_options)
25
+ end
26
+ connection.start
27
+ connection
28
+ end
29
+
30
+ def ruby_env
31
+ ENV.fetch("RUBY_ENV")
32
+ end
33
+
34
+ def tls?
35
+ ENV.has_key?("AMQP_TLS")
36
+ end
37
+
38
+ def hosts_from_env
39
+ # Matches env keys like `RABBITMQ_01_HOST`
40
+ environment_keys_with_host = ENV.keys.select { |key| key =~ /^rabbitmq_\d+_host$/i }
41
+ environment_keys_with_host.map { |key| ENV[key] }
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,119 @@
1
+ require "twingly/amqp/connection"
2
+ require "twingly/amqp/message"
3
+
4
+ module Twingly
5
+ module AMQP
6
+ class Subscription
7
+ def initialize(queue_name:, exchange_topic: nil, routing_key: nil, consumer_threads: 4, prefetch: 20, connection: nil)
8
+ @queue_name = queue_name
9
+ @exchange_topic = exchange_topic
10
+ @routing_key = routing_key
11
+ @consumer_threads = consumer_threads
12
+ @prefetch = prefetch
13
+
14
+ connection ||= Connection.instance
15
+ @channel = create_channel(connection)
16
+ @queue = @channel.queue(@queue_name, queue_options)
17
+
18
+ if @exchange_topic && @routing_key
19
+ exchange = @channel.topic(@exchange_topic, durable: true)
20
+ @queue.bind(exchange, routing_key: @routing_key)
21
+ end
22
+
23
+ @before_handle_message_callback = Proc.new {}
24
+ @on_exception_callback = Proc.new {}
25
+ end
26
+
27
+ def subscribe(&block)
28
+ setup_traps
29
+
30
+ consumer = @queue.subscribe(subscribe_options) do |delivery_info, metadata, payload|
31
+ @before_handle_message_callback.call(payload)
32
+
33
+ message = Message.new(
34
+ delivery_info: delivery_info,
35
+ metadata: metadata,
36
+ payload: payload,
37
+ channel: @channel,
38
+ )
39
+
40
+ block.call(message)
41
+ end
42
+
43
+ # The consumer isn't blocking, so we wait here
44
+ until cancel? do
45
+ sleep 0.5
46
+ end
47
+
48
+ puts "Shutting down" if development?
49
+ consumer.cancel
50
+ end
51
+
52
+ def before_handle_message(&block)
53
+ @before_handle_message_callback = block
54
+ end
55
+
56
+ def on_exception(&block)
57
+ @on_exception_callback = block
58
+ end
59
+
60
+ def cancel?
61
+ @cancel
62
+ end
63
+
64
+ def cancel!
65
+ @cancel = true
66
+ end
67
+
68
+ private
69
+
70
+ def create_channel(connection)
71
+ channel = connection.create_channel(nil, @consumer_threads)
72
+ channel.prefetch(@prefetch)
73
+ channel.on_uncaught_exception do |exception, _|
74
+ puts exception.message, exception.backtrace if development?
75
+ @on_exception_callback.call(exception)
76
+ end
77
+ channel
78
+ end
79
+
80
+ def development?
81
+ ruby_env == "development"
82
+ end
83
+
84
+ def ruby_env
85
+ ENV.fetch("RUBY_ENV")
86
+ end
87
+
88
+ def queue_options
89
+ {
90
+ durable: true,
91
+ }
92
+ end
93
+
94
+ def subscribe_options
95
+ {
96
+ manual_ack: true,
97
+ consumer_tag: consumer_tag,
98
+ }
99
+ end
100
+
101
+ def consumer_tag
102
+ tag_name = [Socket.gethostname, ruby_env].join('-')
103
+ @channel.generate_consumer_tag(tag_name)
104
+ end
105
+
106
+ def setup_traps
107
+ [:INT, :TERM].each do |signal|
108
+ Signal.trap(signal) do
109
+ # Exit fast if we've already got a signal since before
110
+ exit!(true) if cancel?
111
+
112
+ # Set cancel flag, cancels consumers
113
+ cancel!
114
+ end
115
+ end
116
+ end
117
+ end
118
+ end
119
+ end
@@ -0,0 +1,5 @@
1
+ module Twingly
2
+ module Amqp
3
+ VERSION = "3.2.0"
4
+ end
5
+ end
metadata ADDED
@@ -0,0 +1,93 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: twingly-amqp
3
+ version: !ruby/object:Gem::Version
4
+ version: 3.2.0
5
+ platform: ruby
6
+ authors:
7
+ - Twingly AB
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-10-30 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bunny
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rspec
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '3'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '3'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '10.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '10.0'
55
+ description: Pings urls via RabbitMQ.
56
+ email:
57
+ - support@twingly.com
58
+ executables: []
59
+ extensions: []
60
+ extra_rdoc_files: []
61
+ files:
62
+ - README.md
63
+ - lib/twingly/amqp.rb
64
+ - lib/twingly/amqp/connection.rb
65
+ - lib/twingly/amqp/message.rb
66
+ - lib/twingly/amqp/ping.rb
67
+ - lib/twingly/amqp/session.rb
68
+ - lib/twingly/amqp/subscription.rb
69
+ - lib/twingly/amqp/version.rb
70
+ homepage: https://github.com/twingly/twingly-amqp
71
+ licenses: []
72
+ metadata: {}
73
+ post_install_message:
74
+ rdoc_options: []
75
+ require_paths:
76
+ - lib
77
+ required_ruby_version: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - ">="
80
+ - !ruby/object:Gem::Version
81
+ version: '0'
82
+ required_rubygems_version: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - ">="
85
+ - !ruby/object:Gem::Version
86
+ version: '0'
87
+ requirements: []
88
+ rubyforge_project:
89
+ rubygems_version: 2.4.5
90
+ signing_key:
91
+ specification_version: 4
92
+ summary: Ruby library for talking to RabbitMQ.
93
+ test_files: []