stapfen 2.2.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +19 -0
- data/CHANGES.md +16 -0
- data/Gemfile +16 -0
- data/LICENSE.txt +22 -0
- data/README.md +106 -0
- data/Rakefile +10 -0
- data/examples/simple.rb +44 -0
- data/lib/stapfen.rb +25 -0
- data/lib/stapfen/client.rb +8 -0
- data/lib/stapfen/client/jms.rb +118 -0
- data/lib/stapfen/client/kafka.rb +76 -0
- data/lib/stapfen/client/stomp.rb +35 -0
- data/lib/stapfen/destination.rb +59 -0
- data/lib/stapfen/logger.rb +48 -0
- data/lib/stapfen/message.rb +58 -0
- data/lib/stapfen/version.rb +3 -0
- data/lib/stapfen/worker.rb +265 -0
- data/spec/client/jms_spec.rb +199 -0
- data/spec/client/kafka_spec.rb +54 -0
- data/spec/client/stomp_spec.rb +5 -0
- data/spec/client_spec.rb +5 -0
- data/spec/destination_spec.rb +71 -0
- data/spec/logger_spec.rb +41 -0
- data/spec/message_spec.rb +96 -0
- data/spec/spec_helper.rb +21 -0
- data/spec/worker_spec.rb +275 -0
- data/stapfen.gemspec +24 -0
- metadata +93 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 9ca4badd6b7c0c0e44a1a101242b2fd87eb2a30d
|
4
|
+
data.tar.gz: 87865dda53e78a451e17acfa310a44471a546852
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6e659e0357574a33fe5b806106c00bbae8ef2cf1d5f3dc4148af817b2fa72860e700f64d9a0271e116bba5648ca9e3251f934f767aac102743dda02990eba77d
|
7
|
+
data.tar.gz: f95e44cd65ddf1ab1625161934f03781c14276779dd1389d168c1b28f004aff76d4dc8af2589b6addecca69b25957b7dff7be2e047758e33461fc3fa4dc9094e
|
data/.gitignore
ADDED
data/CHANGES.md
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
# Changes to Stapfen
|
2
|
+
|
3
|
+
## 2.0.2
|
4
|
+
|
5
|
+
* Fix for unreceive handling
|
6
|
+
|
7
|
+
## 2.0.1
|
8
|
+
|
9
|
+
* More verbose exit handlers
|
10
|
+
* Invoke `java.lang.System.exit` on the JVM when exiting
|
11
|
+
|
12
|
+
## 2.0.0
|
13
|
+
|
14
|
+
* Add support for JMS-backed `Stapfen::Worker` classes
|
15
|
+
* Deep copy the configuration passed into `Stomp::Client` to work-around [stomp #80](https://github.com/stompgem/stomp/issues/80)
|
16
|
+
* Support per-instance log configuration [#3](https://github.com/lookout/stapfen/issues/3)
|
data/Gemfile
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
# Specify your gem's dependencies in stapfen.gemspec
|
4
|
+
gemspec
|
5
|
+
|
6
|
+
gem 'jruby-jms', :platform => :jruby
|
7
|
+
gem 'stomp', '>= 1.2.14'
|
8
|
+
|
9
|
+
group :development do
|
10
|
+
gem 'rake'
|
11
|
+
gem 'rspec'
|
12
|
+
gem 'rspec-its'
|
13
|
+
gem 'pry'
|
14
|
+
gem 'debugger', :platform => :mri
|
15
|
+
gem 'debugger-pry', :platform => :mri
|
16
|
+
end
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 R. Tyler Croy
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
# Stapfen
|
2
|
+
|
3
|
+
|
4
|
+
Stapfen is a simple gem to make writing workers that consume messages via
|
5
|
+
[STOMP](http://stomp.github.io/) or
|
6
|
+
[JMS](https://en.wikipedia.org/wiki/Java_Message_Service) easier.
|
7
|
+
|
8
|
+
Stapfen allows you to write one worker class, and use either protocol
|
9
|
+
depending on the environment and needs.
|
10
|
+
|
11
|
+
|
12
|
+
**[RDoc here](http://rdoc.info/github/lookout/stapfen/master/frames)**
|
13
|
+
|
14
|
+
## Usage
|
15
|
+
|
16
|
+
(Examples can be found in the `examples/` directory)
|
17
|
+
|
18
|
+
|
19
|
+
Consider the following `myworker.rb` file:
|
20
|
+
|
21
|
+
```ruby
|
22
|
+
class MyWorker < Stapfen::Worker
|
23
|
+
use_stomp!
|
24
|
+
|
25
|
+
configure do
|
26
|
+
{
|
27
|
+
:hosts => [
|
28
|
+
{
|
29
|
+
:host => 'localhost',
|
30
|
+
:port => 61613,
|
31
|
+
:login => 'guest',
|
32
|
+
:passcode => 'guest',
|
33
|
+
:ssl => false
|
34
|
+
}
|
35
|
+
]
|
36
|
+
}
|
37
|
+
end
|
38
|
+
|
39
|
+
# [Optional] Set up a logger for each worker instance
|
40
|
+
log do
|
41
|
+
Logger.new(STDOUT)
|
42
|
+
end
|
43
|
+
|
44
|
+
consume 'thequeue', :dead_letter_queue => '/queue/dlq',
|
45
|
+
:max_redeliveries => 0 do |message|
|
46
|
+
|
47
|
+
data = expensive_computation(message.body)
|
48
|
+
# Save my data, or do something worker-specific with it
|
49
|
+
persist(data)
|
50
|
+
|
51
|
+
# Send another message
|
52
|
+
client.publish('/topic/computation-acks', "finished with #{message.message_id}")
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
MyWorker.run!
|
58
|
+
```
|
59
|
+
|
60
|
+
|
61
|
+
When using the STOMP protocol, the value returned from the `configure` block is expected to be a valid
|
62
|
+
`Stomp::Client` [connection
|
63
|
+
hash](https://github.com/stompgem/stomp#hash-login-example-usage-this-is-the-recommended-login-technique).
|
64
|
+
|
65
|
+
In the case of the JMS protocol, the value returned from the `configure` block
|
66
|
+
is expected to be a valid [configuration
|
67
|
+
hash](https://github.com/reidmorrison/jruby-jms#consumer) for the
|
68
|
+
[jruby-jms](https://github.com/reidmorrison/jruby-jms) gem.
|
69
|
+
|
70
|
+
---
|
71
|
+
|
72
|
+
It is also important to note that the `consume` block will be invoked inside an
|
73
|
+
**instance** of `MyWorker` and will execute inside its own `Thread`, so take
|
74
|
+
care when accessing other shared resources.
|
75
|
+
|
76
|
+
### Fallback and dead-letter-queue support
|
77
|
+
|
78
|
+
The consume block accepts the usual subscriptions headers, as well as two
|
79
|
+
additional headers `:dead_letter_queue` and `:max_redeliveries`. If either of
|
80
|
+
the latter two is present, the consumer will unreceive any messages for which
|
81
|
+
the block returns `false`; after `:max_redeliveries`, it will send the message
|
82
|
+
to `:dead_letter_queue`. `consume` blocks without these headers will fail
|
83
|
+
silently rather than unreceive.
|
84
|
+
|
85
|
+
|
86
|
+
## Installation
|
87
|
+
|
88
|
+
Add this line to your application's Gemfile:
|
89
|
+
|
90
|
+
gem 'stapfen'
|
91
|
+
|
92
|
+
And then execute:
|
93
|
+
|
94
|
+
$ bundle
|
95
|
+
|
96
|
+
Or install it yourself as:
|
97
|
+
|
98
|
+
$ gem install stapfen
|
99
|
+
|
100
|
+
## Running Specs
|
101
|
+
|
102
|
+
Download this from jar from Maven Central
|
103
|
+
* [activemq-all-5.8.0.jar](http://search.maven.org/#artifactdetails%7Corg.apache.activemq%7Cactivemq-all%7C5.8.0%7Cjar)
|
104
|
+
* Put it in gem root
|
105
|
+
* ```rake spec```
|
106
|
+
|
data/Rakefile
ADDED
data/examples/simple.rb
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
|
2
|
+
require 'stapfen'
|
3
|
+
require ENV['activemq_jar'] if ENV['USE_JMS']
|
4
|
+
|
5
|
+
|
6
|
+
class Worker < Stapfen::Worker
|
7
|
+
|
8
|
+
configure do
|
9
|
+
{:hosts => [
|
10
|
+
{
|
11
|
+
:host => 'localhost',
|
12
|
+
:port => 61613,
|
13
|
+
:login => 'guest',
|
14
|
+
:passcode => 'guest',
|
15
|
+
:ssl => false
|
16
|
+
}
|
17
|
+
],
|
18
|
+
:factory => 'org.apache.activemq.ActiveMQConnectionFactory'}
|
19
|
+
end
|
20
|
+
|
21
|
+
use_jms! if ENV['USE_JMS']
|
22
|
+
|
23
|
+
if ENV['USE_JMS']
|
24
|
+
consume '/queue/jms.queue.test',
|
25
|
+
:max_redeliveries => 3,
|
26
|
+
:dead_letter_queue => '/queue/jms.queue.test/dlq' do |message|
|
27
|
+
puts "received: #{message}"
|
28
|
+
|
29
|
+
# False here forces an unreceive
|
30
|
+
return false
|
31
|
+
end
|
32
|
+
else # use stomp
|
33
|
+
consume '/queue/test',
|
34
|
+
:max_redeliveries => 3,
|
35
|
+
:dead_letter_queue => '/queue/test/dlq' do |message|
|
36
|
+
puts "received: #{message}"
|
37
|
+
|
38
|
+
# False here forces an unreceive
|
39
|
+
return false
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
Worker.run!
|
data/lib/stapfen.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
begin
|
2
|
+
require 'stomp'
|
3
|
+
rescue LoadError
|
4
|
+
# Can't process Stomp
|
5
|
+
end
|
6
|
+
|
7
|
+
begin
|
8
|
+
require 'java'
|
9
|
+
require 'jms'
|
10
|
+
rescue LoadError
|
11
|
+
# Can't process JMS
|
12
|
+
end
|
13
|
+
|
14
|
+
require 'stapfen/version'
|
15
|
+
require 'stapfen/client'
|
16
|
+
require 'stapfen/worker'
|
17
|
+
|
18
|
+
module Stapfen
|
19
|
+
class ConfigurationError < StandardError
|
20
|
+
end
|
21
|
+
class ConsumeError < StandardError
|
22
|
+
end
|
23
|
+
class InvalidMessageError < StandardError
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
require 'jms'
|
2
|
+
require 'stapfen/destination'
|
3
|
+
|
4
|
+
module Stapfen
|
5
|
+
module Client
|
6
|
+
class JMS
|
7
|
+
attr_reader :connection
|
8
|
+
|
9
|
+
def initialize(configuration)
|
10
|
+
super()
|
11
|
+
@config = configuration
|
12
|
+
@connection = nil
|
13
|
+
end
|
14
|
+
|
15
|
+
# Connect to the broker via JMS and start the JMS session
|
16
|
+
#
|
17
|
+
# @return [JMS::Connection]
|
18
|
+
def connect(*args)
|
19
|
+
@connection = ::JMS::Connection.new(@config)
|
20
|
+
@connection.start
|
21
|
+
return @connection
|
22
|
+
end
|
23
|
+
|
24
|
+
# Accessor method which will cache the session if we've already created
|
25
|
+
# it once
|
26
|
+
#
|
27
|
+
# @return [JMS::Session] Instantiated +JMS::Session+ for our
|
28
|
+
# +connection+
|
29
|
+
def session
|
30
|
+
@session ||= connection.create_session
|
31
|
+
end
|
32
|
+
|
33
|
+
def publish(destination, body, headers={})
|
34
|
+
destination = Stapfen::Destination.from_string(destination)
|
35
|
+
|
36
|
+
session.producer(destination.jms_opts) do |p|
|
37
|
+
# Create the JMS typed Message
|
38
|
+
message = session.message(body)
|
39
|
+
|
40
|
+
message.delivery_mode = ::JMS::DeliveryMode::PERSISTENT if headers.delete(:persistent)
|
41
|
+
|
42
|
+
# Take the remainder of the headers and push them into the message
|
43
|
+
# properties.
|
44
|
+
headers.each_pair do |key, value|
|
45
|
+
message.setStringProperty(key.to_s, value.to_s)
|
46
|
+
end
|
47
|
+
|
48
|
+
p.send(message)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def subscribe(destination, headers={}, &block)
|
53
|
+
destination = Stapfen::Destination.from_string(destination)
|
54
|
+
connection.on_message(destination.jms_opts) do |m|
|
55
|
+
block.call(m)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# Close the JMS::Connection and the JMS::Session if it's been created
|
60
|
+
# for this client
|
61
|
+
#
|
62
|
+
# @return [Boolean] True/false depending on whether we actually closed
|
63
|
+
# the connection
|
64
|
+
def close
|
65
|
+
return false unless @connection
|
66
|
+
@session.close if @session
|
67
|
+
@connection.close
|
68
|
+
@connection = nil
|
69
|
+
return true
|
70
|
+
end
|
71
|
+
|
72
|
+
# API compatibilty method, doesn't actually indicate that the connection
|
73
|
+
# is closed. Will only return true if no connection currently exists
|
74
|
+
#
|
75
|
+
# @return [Boolean]
|
76
|
+
def closed?
|
77
|
+
return connection.nil?
|
78
|
+
end
|
79
|
+
|
80
|
+
def runloop
|
81
|
+
loop do
|
82
|
+
sleep 1
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def can_unreceive?
|
87
|
+
true
|
88
|
+
end
|
89
|
+
|
90
|
+
# JMS doesn't implement unreceive in quite the way Stomp does, so we'll
|
91
|
+
# implement it here.
|
92
|
+
#
|
93
|
+
# Given a message and a set of unreceive headers, this will deliver the
|
94
|
+
# message back to its originating queue a limited number of times, then
|
95
|
+
# failover to the dead letter queue.
|
96
|
+
#
|
97
|
+
# @param [Stapfen::Message] message The message to unreceive.
|
98
|
+
# @param [Hash] unreceive_headers
|
99
|
+
# @option unreceive_headers [Integer] :max_redeliveries The number of times
|
100
|
+
# to attempt redelivery.
|
101
|
+
# @option unreceive_headers [String] :dead_letter_queue After giving up on
|
102
|
+
# redelivering, send the message here.
|
103
|
+
def unreceive(message, unreceive_headers)
|
104
|
+
return if unreceive_headers[:max_redeliveries].nil? && unreceive_headers[:max_redeliveries].nil?
|
105
|
+
|
106
|
+
destination = message.destination.to_s.sub('queue://','/queue/').sub('topic://','/topic')
|
107
|
+
retry_count = message.getStringProperty('retry_count').to_i || 0
|
108
|
+
retry_count +=1
|
109
|
+
|
110
|
+
if unreceive_headers[:max_redeliveries] && (retry_count <= unreceive_headers[:max_redeliveries])
|
111
|
+
self.publish(destination, message.data, {'retry_count' => retry_count.to_s})
|
112
|
+
elsif unreceive_headers[:dead_letter_queue] # Done retrying, send to DLQ
|
113
|
+
self.publish(unreceive_headers[:dead_letter_queue], message.data, {:original_destination => destination})
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
begin
|
2
|
+
require 'hermann'
|
3
|
+
require 'hermann/consumer'
|
4
|
+
rescue LoadError => e
|
5
|
+
if RUBY_PLATFORM == 'java'
|
6
|
+
raise
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
require 'stapfen/destination'
|
11
|
+
|
12
|
+
module Stapfen
|
13
|
+
module Client
|
14
|
+
class Kafka
|
15
|
+
attr_reader :connection, :producer
|
16
|
+
|
17
|
+
# Initialize a Kafka client object
|
18
|
+
#
|
19
|
+
# @params [Hash] configuration object
|
20
|
+
# @option configuration [String] :topic The kafka topic
|
21
|
+
# @option configuration [String] :groupId The kafka groupId
|
22
|
+
# @option configuration [String] :zookeepers List of zookeepers
|
23
|
+
#
|
24
|
+
# @raises [ConfigurationError] if required configs are not present
|
25
|
+
def initialize(configuration)
|
26
|
+
super()
|
27
|
+
@config = configuration
|
28
|
+
@topic = @config[:topic]
|
29
|
+
@groupId = @config[:groupId]
|
30
|
+
@zookeepers = @config[:zookeepers]
|
31
|
+
raise ConfigurationError unless @topic && @groupId && @zookeepers
|
32
|
+
@connection = Hermann::Consumer.new(@topic, @groupId, @zookeepers)
|
33
|
+
end
|
34
|
+
|
35
|
+
# This method is not implemenented
|
36
|
+
def connect(*args)
|
37
|
+
# No-op
|
38
|
+
end
|
39
|
+
|
40
|
+
# Cannot unreceive
|
41
|
+
def can_unreceive?
|
42
|
+
false
|
43
|
+
end
|
44
|
+
|
45
|
+
# Closes the consumer threads created by kafka.
|
46
|
+
#
|
47
|
+
# @return [Boolean] True/false depending on whether we actually closed
|
48
|
+
# the connection
|
49
|
+
def close
|
50
|
+
return false unless @connection
|
51
|
+
@connection.shutdown
|
52
|
+
@connection = nil
|
53
|
+
return true
|
54
|
+
end
|
55
|
+
|
56
|
+
# Subscribes to a destination (i.e. kafka topic) and consumes messages
|
57
|
+
#
|
58
|
+
# @params [Destination] source of messages to consume
|
59
|
+
#
|
60
|
+
# @params [Hash] Not used
|
61
|
+
#
|
62
|
+
# @params [block] block to yield consumed messages
|
63
|
+
def subscribe(destination, headers={}, &block)
|
64
|
+
destination = Stapfen::Destination.from_string(destination)
|
65
|
+
connection.consume(destination.as_kafka, &block)
|
66
|
+
end
|
67
|
+
|
68
|
+
def runloop
|
69
|
+
loop do
|
70
|
+
sleep 1
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|