emque-producing 0.0.3 → 1.0.0.beta2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -1
- data/README.md +10 -12
- data/emque-producing.gemspec +6 -7
- data/lib/emque/producing/configuration.rb +0 -3
- data/lib/emque/producing/producing.rb +1 -4
- data/lib/emque/producing/version.rb +1 -1
- data/spec/producing/configuration_spec.rb +0 -6
- metadata +18 -42
- data/Gemfile.lock +0 -64
- data/lib/emque/producing/publisher/kafka.rb +0 -25
- data/spec/bin/kafka-run-class.sh +0 -67
- data/spec/kafka_spec_helper.rb +0 -33
- data/spec/producing/publisher_spec.rb +0 -20
- data/spec/test_cluster.rb +0 -201
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b77c1aea181103402f745688487ef3de2b03741c
|
4
|
+
data.tar.gz: 97051666e37ea9f28bd626d038cb8c85f60121a3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a66f000eb81b489e4d655b475ddd6061c58a45190c17259aaa89f62224c35521a2f99ccee6fc8a2aceaf9ca7a11ffdfd815d2ceb909eb510df255f6ff906a51f
|
7
|
+
data.tar.gz: f9779c82e7f934dce063a0143410133c94e48325fb9a4376238c1617c5a8dbac78dd5c27f14bb2a7ec37b9d70896b3895aa70fd5b77627554d5d5a2e444bf2b8
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -4,8 +4,7 @@ teamsnap/emque-producing](https://www.codeship.io/projects/2ca7fd90-1785-0132-5f
|
|
4
4
|
# Emque Producing
|
5
5
|
|
6
6
|
Define and send messages with Ruby to a variety of [message brokers](http://en.wikipedia.org/wiki/Message_broker).
|
7
|
-
|
8
|
-
[Kafka](http://kafka.apache.org/).
|
7
|
+
The only currently supported message broker is [RabbitMQ](https://www.rabbitmq.com)
|
9
8
|
|
10
9
|
This is a library that pairs nicely with [Emque
|
11
10
|
Consuming](https://www.github.com/teamsnap/emque-consuming), a framework for
|
@@ -19,12 +18,6 @@ Add these lines to your application's Gemfile, depending on your message broker:
|
|
19
18
|
gem "emque-producing"
|
20
19
|
gem "bunny", "~> 1.4.1"
|
21
20
|
|
22
|
-
or
|
23
|
-
|
24
|
-
# for Kafka, poseidon is used
|
25
|
-
gem "emque-producing"
|
26
|
-
gem "poseidon", "0.0.4"
|
27
|
-
|
28
21
|
And then execute:
|
29
22
|
|
30
23
|
$ bundle
|
@@ -41,8 +34,6 @@ Or install it yourself as:
|
|
41
34
|
c.app_name = "app"
|
42
35
|
c.publishing_adapter = :rabbitmq
|
43
36
|
c.rabbitmq_options = { :url => "amqp://guest:guest@localhost:5672" }
|
44
|
-
#c.kafka_options = { :seed_brokers => [localhost:9092],
|
45
|
-
# :producer_options => {} }
|
46
37
|
c.error_handlers << Proc.new {|ex,context|
|
47
38
|
# notify/log
|
48
39
|
}
|
@@ -62,13 +53,15 @@ Or install it yourself as:
|
|
62
53
|
message = MyMessage.new({:first_property => 1, :another_property => "another"})
|
63
54
|
message.publish
|
64
55
|
|
56
|
+
For a more thorough guide to creating new messages and/or message producing
|
57
|
+
applications, [please read the wiki
|
58
|
+
entry](https://github.com/teamsnap/emque-producing/wiki/Creating-New-Producing-Applications)
|
59
|
+
|
65
60
|
## Requirements
|
66
61
|
|
67
62
|
* Ruby 1.9.3 or higher
|
68
63
|
* RabbitMQ 3.x
|
69
64
|
* Bunny 1.4.x
|
70
|
-
* Kafka 0.8.1
|
71
|
-
* Poseidon 0.0.4
|
72
65
|
|
73
66
|
## Tests
|
74
67
|
|
@@ -96,3 +89,8 @@ adapter which may be convenient in the case of CI environments.
|
|
96
89
|
3. Commit your changes (`git commit -am 'Add some feature'`)
|
97
90
|
4. Push to the branch (`git push origin my-new-feature`)
|
98
91
|
5. Create new Pull Request
|
92
|
+
|
93
|
+
## Roadmap
|
94
|
+
|
95
|
+
Kafka would make for a good adapter to be added to emque-producing. Anyone
|
96
|
+
wishing to submit a PR can use the RabbitMQ adapter as a model.
|
data/emque-producing.gemspec
CHANGED
@@ -21,13 +21,12 @@ Gem::Specification.new do |spec|
|
|
21
21
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
22
22
|
spec.require_paths = ["lib"]
|
23
23
|
|
24
|
-
spec.add_dependency "oj",
|
25
|
-
spec.add_dependency "virtus",
|
24
|
+
spec.add_dependency "oj", "~> 2.11"
|
25
|
+
spec.add_dependency "virtus", "~> 1.0"
|
26
26
|
|
27
|
-
spec.add_development_dependency "bundler", "~> 1.
|
28
|
-
spec.add_development_dependency "rake"
|
29
|
-
spec.add_development_dependency "rspec", "~> 3.
|
27
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
28
|
+
spec.add_development_dependency "rake", "~> 10.4"
|
29
|
+
spec.add_development_dependency "rspec", "~> 3.2"
|
30
30
|
spec.add_development_dependency "pry"
|
31
|
-
spec.add_development_dependency "
|
32
|
-
spec.add_development_dependency "bunny", "~> 1.4.1"
|
31
|
+
spec.add_development_dependency "bunny", "~> 1.7"
|
33
32
|
end
|
@@ -3,7 +3,6 @@ module Emque
|
|
3
3
|
class Configuration
|
4
4
|
attr_accessor :app_name
|
5
5
|
attr_accessor :publishing_adapter
|
6
|
-
attr_accessor :kafka_options
|
7
6
|
attr_accessor :rabbitmq_options
|
8
7
|
attr_accessor :error_handlers
|
9
8
|
attr_accessor :log_publish_message
|
@@ -15,8 +14,6 @@ module Emque
|
|
15
14
|
@error_handlers = []
|
16
15
|
@log_publish_message = false
|
17
16
|
@publish_messages = true
|
18
|
-
@kafka_options = { :seed_brokers => ["localhost:9092"],
|
19
|
-
:producer_options => {} }
|
20
17
|
@rabbitmq_options = { :url => "amqp://guest:guest@localhost:5672" }
|
21
18
|
end
|
22
19
|
end
|
@@ -33,10 +33,7 @@ module Emque
|
|
33
33
|
def publisher
|
34
34
|
return @publisher unless @publisher.nil?
|
35
35
|
|
36
|
-
if (configuration.publishing_adapter == :
|
37
|
-
require "emque/producing/publisher/kafka"
|
38
|
-
@publisher = Emque::Producing::Publisher::Kafka.new
|
39
|
-
elsif (configuration.publishing_adapter == :rabbitmq)
|
36
|
+
if (configuration.publishing_adapter == :rabbitmq)
|
40
37
|
require "emque/producing/publisher/rabbitmq"
|
41
38
|
@publisher = Emque::Producing::Publisher::RabbitMq.new
|
42
39
|
else
|
@@ -6,16 +6,10 @@ describe Emque::Producing::Configuration do
|
|
6
6
|
it "provides default values" do
|
7
7
|
expect(subject.app_name).to eq ""
|
8
8
|
expect(subject.error_handlers).to eq []
|
9
|
-
expect(subject.kafka_options[:seed_brokers]).to eq ["localhost:9092"]
|
10
9
|
end
|
11
10
|
|
12
11
|
it "allows app_name to be overwritten" do
|
13
12
|
subject.app_name = "my app"
|
14
13
|
expect(subject.app_name).to eq "my app"
|
15
14
|
end
|
16
|
-
|
17
|
-
it "allows seed_brokers to be overwritten" do
|
18
|
-
subject.kafka_options[:seed_brokers] = ["kafka1:9092", "kafka2:9092"]
|
19
|
-
expect(subject.kafka_options[:seed_brokers]).to eq ["kafka1:9092", "kafka2:9092"]
|
20
|
-
end
|
21
15
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: emque-producing
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 1.0.0.beta2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Emily Dobervich
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-
|
12
|
+
date: 2015-03-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: oj
|
@@ -17,70 +17,70 @@ dependencies:
|
|
17
17
|
requirements:
|
18
18
|
- - "~>"
|
19
19
|
- !ruby/object:Gem::Version
|
20
|
-
version: 2.
|
20
|
+
version: '2.11'
|
21
21
|
type: :runtime
|
22
22
|
prerelease: false
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
25
|
- - "~>"
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version: 2.
|
27
|
+
version: '2.11'
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
29
|
name: virtus
|
30
30
|
requirement: !ruby/object:Gem::Requirement
|
31
31
|
requirements:
|
32
32
|
- - "~>"
|
33
33
|
- !ruby/object:Gem::Version
|
34
|
-
version: 1.0
|
34
|
+
version: '1.0'
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
39
|
- - "~>"
|
40
40
|
- !ruby/object:Gem::Version
|
41
|
-
version: 1.0
|
41
|
+
version: '1.0'
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
name: bundler
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
45
45
|
requirements:
|
46
46
|
- - "~>"
|
47
47
|
- !ruby/object:Gem::Version
|
48
|
-
version: '1.
|
48
|
+
version: '1.7'
|
49
49
|
type: :development
|
50
50
|
prerelease: false
|
51
51
|
version_requirements: !ruby/object:Gem::Requirement
|
52
52
|
requirements:
|
53
53
|
- - "~>"
|
54
54
|
- !ruby/object:Gem::Version
|
55
|
-
version: '1.
|
55
|
+
version: '1.7'
|
56
56
|
- !ruby/object:Gem::Dependency
|
57
57
|
name: rake
|
58
58
|
requirement: !ruby/object:Gem::Requirement
|
59
59
|
requirements:
|
60
|
-
- - "
|
60
|
+
- - "~>"
|
61
61
|
- !ruby/object:Gem::Version
|
62
|
-
version: '
|
62
|
+
version: '10.4'
|
63
63
|
type: :development
|
64
64
|
prerelease: false
|
65
65
|
version_requirements: !ruby/object:Gem::Requirement
|
66
66
|
requirements:
|
67
|
-
- - "
|
67
|
+
- - "~>"
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version: '
|
69
|
+
version: '10.4'
|
70
70
|
- !ruby/object:Gem::Dependency
|
71
71
|
name: rspec
|
72
72
|
requirement: !ruby/object:Gem::Requirement
|
73
73
|
requirements:
|
74
74
|
- - "~>"
|
75
75
|
- !ruby/object:Gem::Version
|
76
|
-
version: '3.
|
76
|
+
version: '3.2'
|
77
77
|
type: :development
|
78
78
|
prerelease: false
|
79
79
|
version_requirements: !ruby/object:Gem::Requirement
|
80
80
|
requirements:
|
81
81
|
- - "~>"
|
82
82
|
- !ruby/object:Gem::Version
|
83
|
-
version: '3.
|
83
|
+
version: '3.2'
|
84
84
|
- !ruby/object:Gem::Dependency
|
85
85
|
name: pry
|
86
86
|
requirement: !ruby/object:Gem::Requirement
|
@@ -95,34 +95,20 @@ dependencies:
|
|
95
95
|
- - ">="
|
96
96
|
- !ruby/object:Gem::Version
|
97
97
|
version: '0'
|
98
|
-
- !ruby/object:Gem::Dependency
|
99
|
-
name: poseidon
|
100
|
-
requirement: !ruby/object:Gem::Requirement
|
101
|
-
requirements:
|
102
|
-
- - '='
|
103
|
-
- !ruby/object:Gem::Version
|
104
|
-
version: 0.0.4
|
105
|
-
type: :development
|
106
|
-
prerelease: false
|
107
|
-
version_requirements: !ruby/object:Gem::Requirement
|
108
|
-
requirements:
|
109
|
-
- - '='
|
110
|
-
- !ruby/object:Gem::Version
|
111
|
-
version: 0.0.4
|
112
98
|
- !ruby/object:Gem::Dependency
|
113
99
|
name: bunny
|
114
100
|
requirement: !ruby/object:Gem::Requirement
|
115
101
|
requirements:
|
116
102
|
- - "~>"
|
117
103
|
- !ruby/object:Gem::Version
|
118
|
-
version: 1.
|
104
|
+
version: '1.7'
|
119
105
|
type: :development
|
120
106
|
prerelease: false
|
121
107
|
version_requirements: !ruby/object:Gem::Requirement
|
122
108
|
requirements:
|
123
109
|
- - "~>"
|
124
110
|
- !ruby/object:Gem::Version
|
125
|
-
version: 1.
|
111
|
+
version: '1.7'
|
126
112
|
description: Define and send messages to a variety of message brokers
|
127
113
|
email:
|
128
114
|
- emily@teamsnap.com
|
@@ -133,7 +119,6 @@ extra_rdoc_files: []
|
|
133
119
|
files:
|
134
120
|
- ".gitignore"
|
135
121
|
- Gemfile
|
136
|
-
- Gemfile.lock
|
137
122
|
- LICENSE.txt
|
138
123
|
- README.md
|
139
124
|
- Rakefile
|
@@ -146,18 +131,13 @@ files:
|
|
146
131
|
- lib/emque/producing/message/message_with_changeset.rb
|
147
132
|
- lib/emque/producing/producing.rb
|
148
133
|
- lib/emque/producing/publisher/base.rb
|
149
|
-
- lib/emque/producing/publisher/kafka.rb
|
150
134
|
- lib/emque/producing/publisher/rabbitmq.rb
|
151
135
|
- lib/emque/producing/version.rb
|
152
|
-
- spec/bin/kafka-run-class.sh
|
153
|
-
- spec/kafka_spec_helper.rb
|
154
136
|
- spec/producing/configuration_spec.rb
|
155
137
|
- spec/producing/message/changes_payload_generator_spec.rb
|
156
138
|
- spec/producing/message/message_spec.rb
|
157
139
|
- spec/producing/producing_spec.rb
|
158
|
-
- spec/producing/publisher_spec.rb
|
159
140
|
- spec/spec_helper.rb
|
160
|
-
- spec/test_cluster.rb
|
161
141
|
homepage: ''
|
162
142
|
licenses:
|
163
143
|
- MIT
|
@@ -173,9 +153,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
173
153
|
version: 1.9.3
|
174
154
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
175
155
|
requirements:
|
176
|
-
- - "
|
156
|
+
- - ">"
|
177
157
|
- !ruby/object:Gem::Version
|
178
|
-
version:
|
158
|
+
version: 1.3.1
|
179
159
|
requirements: []
|
180
160
|
rubyforge_project:
|
181
161
|
rubygems_version: 2.2.2
|
@@ -183,12 +163,8 @@ signing_key:
|
|
183
163
|
specification_version: 4
|
184
164
|
summary: Define and send messages to a variety of message brokers
|
185
165
|
test_files:
|
186
|
-
- spec/bin/kafka-run-class.sh
|
187
|
-
- spec/kafka_spec_helper.rb
|
188
166
|
- spec/producing/configuration_spec.rb
|
189
167
|
- spec/producing/message/changes_payload_generator_spec.rb
|
190
168
|
- spec/producing/message/message_spec.rb
|
191
169
|
- spec/producing/producing_spec.rb
|
192
|
-
- spec/producing/publisher_spec.rb
|
193
170
|
- spec/spec_helper.rb
|
194
|
-
- spec/test_cluster.rb
|
data/Gemfile.lock
DELETED
@@ -1,64 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
emque-producing (0.0.3)
|
5
|
-
oj (~> 2.10.2)
|
6
|
-
virtus (~> 1.0.3)
|
7
|
-
|
8
|
-
GEM
|
9
|
-
remote: https://rubygems.org/
|
10
|
-
specs:
|
11
|
-
amq-protocol (1.9.2)
|
12
|
-
axiom-types (0.1.1)
|
13
|
-
descendants_tracker (~> 0.0.4)
|
14
|
-
ice_nine (~> 0.11.0)
|
15
|
-
thread_safe (~> 0.3, >= 0.3.1)
|
16
|
-
bunny (1.4.1)
|
17
|
-
amq-protocol (>= 1.9.2)
|
18
|
-
coderay (1.1.0)
|
19
|
-
coercible (1.0.0)
|
20
|
-
descendants_tracker (~> 0.0.1)
|
21
|
-
descendants_tracker (0.0.4)
|
22
|
-
thread_safe (~> 0.3, >= 0.3.1)
|
23
|
-
diff-lcs (1.2.5)
|
24
|
-
equalizer (0.0.9)
|
25
|
-
ice_nine (0.11.0)
|
26
|
-
method_source (0.8.2)
|
27
|
-
oj (2.10.2)
|
28
|
-
poseidon (0.0.4)
|
29
|
-
pry (0.9.12.4)
|
30
|
-
coderay (~> 1.0)
|
31
|
-
method_source (~> 0.8)
|
32
|
-
slop (~> 3.4)
|
33
|
-
rake (10.1.1)
|
34
|
-
rspec (3.1.0)
|
35
|
-
rspec-core (~> 3.1.0)
|
36
|
-
rspec-expectations (~> 3.1.0)
|
37
|
-
rspec-mocks (~> 3.1.0)
|
38
|
-
rspec-core (3.1.5)
|
39
|
-
rspec-support (~> 3.1.0)
|
40
|
-
rspec-expectations (3.1.2)
|
41
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
42
|
-
rspec-support (~> 3.1.0)
|
43
|
-
rspec-mocks (3.1.2)
|
44
|
-
rspec-support (~> 3.1.0)
|
45
|
-
rspec-support (3.1.1)
|
46
|
-
slop (3.4.7)
|
47
|
-
thread_safe (0.3.4)
|
48
|
-
virtus (1.0.3)
|
49
|
-
axiom-types (~> 0.1)
|
50
|
-
coercible (~> 1.0)
|
51
|
-
descendants_tracker (~> 0.0, >= 0.0.3)
|
52
|
-
equalizer (~> 0.0, >= 0.0.9)
|
53
|
-
|
54
|
-
PLATFORMS
|
55
|
-
ruby
|
56
|
-
|
57
|
-
DEPENDENCIES
|
58
|
-
bundler (~> 1.0)
|
59
|
-
bunny (~> 1.4.1)
|
60
|
-
emque-producing!
|
61
|
-
poseidon (= 0.0.4)
|
62
|
-
pry
|
63
|
-
rake
|
64
|
-
rspec (~> 3.1)
|
@@ -1,25 +0,0 @@
|
|
1
|
-
require "poseidon"
|
2
|
-
|
3
|
-
module Emque
|
4
|
-
module Producing
|
5
|
-
module Publisher
|
6
|
-
class Kafka < Emque::Producing::Publisher::Base
|
7
|
-
def initialize
|
8
|
-
@producer = Poseidon::Producer.new(
|
9
|
-
Emque::Producing.configuration.kafka_options[:seed_brokers],
|
10
|
-
"producer_#{host_name}_#{Process.pid}",
|
11
|
-
Emque::Producing.configuration.kafka_options[:producer_options])
|
12
|
-
end
|
13
|
-
|
14
|
-
def publish(topic, message_type, message, key = nil)
|
15
|
-
begin
|
16
|
-
msg = Poseidon::MessageToSend.new(topic, message, key)
|
17
|
-
@producer.send_messages([msg])
|
18
|
-
rescue => e
|
19
|
-
handle_error(e)
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
data/spec/bin/kafka-run-class.sh
DELETED
@@ -1,67 +0,0 @@
|
|
1
|
-
#!/bin/bash
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one or more
|
3
|
-
# contributor license agreements. See the NOTICE file distributed with
|
4
|
-
# this work for additional information regarding copyright ownership.
|
5
|
-
# The ASF licenses this file to You under the Apache License, Version 2.0
|
6
|
-
# (the "License"); you may not use this file except in compliance with
|
7
|
-
# the License. You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
-
# See the License for the specific language governing permissions and
|
15
|
-
# limitations under the License.
|
16
|
-
|
17
|
-
if [ $# -lt 1 ];
|
18
|
-
then
|
19
|
-
echo "USAGE: $0 classname [opts]"
|
20
|
-
exit 1
|
21
|
-
fi
|
22
|
-
|
23
|
-
if [-z "$SCALA_VERSION" ]; then
|
24
|
-
SCALA_VERSION=2.8.0
|
25
|
-
fi
|
26
|
-
|
27
|
-
# assume all dependencies have been packaged into one jar with sbt-assembly's task "assembly-package-dependency"
|
28
|
-
for file in $KAFKA_PATH/core/target/scala-$SCALA_VERSION/*.jar;
|
29
|
-
do
|
30
|
-
CLASSPATH=$CLASSPATH:$file
|
31
|
-
done
|
32
|
-
|
33
|
-
for file in $KAFKA_PATH/perf/target/scala-$SCALA_VERSION/kafka*.jar;
|
34
|
-
do
|
35
|
-
CLASSPATH=$CLASSPATH:$file
|
36
|
-
done
|
37
|
-
|
38
|
-
# classpath addition for release
|
39
|
-
for file in $KAFKA_PATH/libs/*.jar;
|
40
|
-
do
|
41
|
-
CLASSPATH=$CLASSPATH:$file
|
42
|
-
done
|
43
|
-
|
44
|
-
for file in $KAFKA_PATH/kafka*.jar;
|
45
|
-
do
|
46
|
-
CLASSPATH=$CLASSPATH:$file
|
47
|
-
done
|
48
|
-
|
49
|
-
if [ -z "$KAFKA_JMX_OPTS" ]; then
|
50
|
-
KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false "
|
51
|
-
fi
|
52
|
-
|
53
|
-
if [ -z "$KAFKA_OPTS" ]; then
|
54
|
-
KAFKA_OPTS="-Xmx512M -server -Dlog4j.configuration=file:$KAFKA_PATH/config/log4j.properties"
|
55
|
-
fi
|
56
|
-
|
57
|
-
if [ $JMX_PORT ]; then
|
58
|
-
KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT "
|
59
|
-
fi
|
60
|
-
|
61
|
-
if [ -z "$JAVA_HOME" ]; then
|
62
|
-
JAVA="java"
|
63
|
-
else
|
64
|
-
JAVA="$JAVA_HOME/bin/java"
|
65
|
-
fi
|
66
|
-
|
67
|
-
exec $JAVA $KAFKA_OPTS $KAFKA_JMX_OPTS -cp $CLASSPATH "$@"
|
data/spec/kafka_spec_helper.rb
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
ROOT_DIRECTORY = File.absolute_path(File.dirname(__FILE__) + "/../")
|
2
|
-
|
3
|
-
require "test_cluster"
|
4
|
-
require "spec_helper"
|
5
|
-
|
6
|
-
unless File.directory?(File.join(ROOT_DIRECTORY, "kafka_2.8.0-0.8.1"))
|
7
|
-
puts "\033[0;32m"
|
8
|
-
puts "*" * 83
|
9
|
-
puts "Downloading kafka"
|
10
|
-
puts "*" * 83
|
11
|
-
puts "\033[0;0m"
|
12
|
-
|
13
|
-
system(
|
14
|
-
"cd #{ROOT_DIRECTORY} && curl https://archive.apache.org/dist/kafka/0.8.1/kafka_2.8.0-0.8.1.tgz | tar xz"
|
15
|
-
)
|
16
|
-
end
|
17
|
-
|
18
|
-
ENV["KAFKA_PATH"] = File.join(ROOT_DIRECTORY, "kafka_2.8.0-0.8.1")
|
19
|
-
ENV["SCALA_VERSION"] = "2.8.0"
|
20
|
-
|
21
|
-
RSpec.configure do |config|
|
22
|
-
config.before(:suite) do
|
23
|
-
JavaRunner.remove_tmp
|
24
|
-
JavaRunner.set_kafka_path!
|
25
|
-
$tc = TestCluster.new
|
26
|
-
$tc.start
|
27
|
-
sleep 5
|
28
|
-
end
|
29
|
-
|
30
|
-
config.after(:suite) do
|
31
|
-
$tc.stop
|
32
|
-
end
|
33
|
-
end
|
@@ -1,20 +0,0 @@
|
|
1
|
-
require "spec_helper"
|
2
|
-
require "emque/producing/publisher/kafka"
|
3
|
-
|
4
|
-
describe Emque::Producing::Publisher do
|
5
|
-
describe "#publish" do
|
6
|
-
context "when error handler raises an exception" do
|
7
|
-
it "handles the exception" do
|
8
|
-
expect_any_instance_of(Poseidon::Producer). to receive(:send_messages).and_raise
|
9
|
-
Emque::Producing.configure do |c|
|
10
|
-
c.error_handlers << Proc.new {|ex,context|
|
11
|
-
raise "something"
|
12
|
-
}
|
13
|
-
end
|
14
|
-
Emque::Producing.logger = nil
|
15
|
-
publisher = Emque::Producing::Publisher::Kafka.new
|
16
|
-
publisher.publish("mytopic", "message.type", "mymessage")
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
data/spec/test_cluster.rb
DELETED
@@ -1,201 +0,0 @@
|
|
1
|
-
# from https://github.com/bpot/poseidon
|
2
|
-
|
3
|
-
require 'daemon_controller'
|
4
|
-
|
5
|
-
class TestCluster
|
6
|
-
attr_reader :broker, :zookeeper
|
7
|
-
def initialize
|
8
|
-
@zookeeper = ZookeeperRunner.new
|
9
|
-
@broker = BrokerRunner.new(0, 9092)
|
10
|
-
end
|
11
|
-
|
12
|
-
def start
|
13
|
-
@zookeeper.start
|
14
|
-
@broker.start
|
15
|
-
end
|
16
|
-
|
17
|
-
def stop
|
18
|
-
@zookeeper.stop
|
19
|
-
@broker.stop
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
class JavaRunner
|
24
|
-
def self.remove_tmp
|
25
|
-
FileUtils.rm_rf("#{ROOT_DIRECTORY}/tmp")
|
26
|
-
end
|
27
|
-
|
28
|
-
def self.set_kafka_path!
|
29
|
-
JavaRunner.kafka_path = File.join(ROOT_DIRECTORY, "kafka-0.8.0-src")
|
30
|
-
end
|
31
|
-
|
32
|
-
def self.kafka_path=(kafka_path)
|
33
|
-
@kafka_path = kafka_path
|
34
|
-
end
|
35
|
-
|
36
|
-
def self.kafka_path
|
37
|
-
@kafka_path
|
38
|
-
end
|
39
|
-
|
40
|
-
attr_reader :pid
|
41
|
-
def initialize(id, start_cmd, port, properties = {})
|
42
|
-
@id = id
|
43
|
-
@properties = properties
|
44
|
-
@pid = nil
|
45
|
-
@start_cmd = start_cmd
|
46
|
-
@port = port
|
47
|
-
end
|
48
|
-
|
49
|
-
def start
|
50
|
-
write_properties
|
51
|
-
run
|
52
|
-
end
|
53
|
-
|
54
|
-
def stop
|
55
|
-
daemon_controller.stop
|
56
|
-
end
|
57
|
-
|
58
|
-
def without_process
|
59
|
-
stop
|
60
|
-
begin
|
61
|
-
yield
|
62
|
-
ensure
|
63
|
-
start
|
64
|
-
sleep 5
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
private
|
69
|
-
|
70
|
-
def daemon_controller
|
71
|
-
@dc ||= DaemonController.new(
|
72
|
-
:identifier => @id,
|
73
|
-
:start_command => "#{@start_cmd} #{config_path} >>#{log_path} 2>&1 & echo $! > #{pid_path}",
|
74
|
-
:ping_command => [:tcp, '127.0.0.1', @port],
|
75
|
-
:pid_file => pid_path,
|
76
|
-
:log_file => log_path,
|
77
|
-
:start_timeout => 25
|
78
|
-
)
|
79
|
-
end
|
80
|
-
|
81
|
-
def run
|
82
|
-
FileUtils.mkdir_p(log_dir)
|
83
|
-
FileUtils.mkdir_p(pid_dir)
|
84
|
-
daemon_controller.start
|
85
|
-
end
|
86
|
-
|
87
|
-
def write_properties
|
88
|
-
FileUtils.mkdir_p(config_dir)
|
89
|
-
File.open(config_path, "w+") do |f|
|
90
|
-
@properties.each do |k,v|
|
91
|
-
f.puts "#{k}=#{v}"
|
92
|
-
end
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
|
-
def pid_path
|
97
|
-
"#{pid_dir}/#{@id}.pid"
|
98
|
-
end
|
99
|
-
|
100
|
-
def pid_dir
|
101
|
-
"#{file_path}/pid"
|
102
|
-
end
|
103
|
-
|
104
|
-
def log_path
|
105
|
-
"#{log_dir}/#{@id}.log"
|
106
|
-
end
|
107
|
-
|
108
|
-
def log_dir
|
109
|
-
"#{file_path}/log"
|
110
|
-
end
|
111
|
-
|
112
|
-
def config_path
|
113
|
-
"#{config_dir}/#{@id}.properties"
|
114
|
-
end
|
115
|
-
|
116
|
-
def config_dir
|
117
|
-
"#{file_path}/config"
|
118
|
-
end
|
119
|
-
|
120
|
-
def file_path
|
121
|
-
ROOT_DIRECTORY + "/tmp/"
|
122
|
-
end
|
123
|
-
end
|
124
|
-
|
125
|
-
class BrokerRunner
|
126
|
-
DEFAULT_PROPERTIES = {
|
127
|
-
"broker.id" => 0,
|
128
|
-
"port" => 9092,
|
129
|
-
"num.network.threads" => 2,
|
130
|
-
"num.io.threads" => 2,
|
131
|
-
"socket.send.buffer.bytes" => 1048576,
|
132
|
-
"socket.receive.buffer.bytes" => 1048576,
|
133
|
-
"socket.request.max.bytes" => 104857600,
|
134
|
-
"log.dir" => "#{ROOT_DIRECTORY}/tmp/kafka-logs",
|
135
|
-
"num.partitions" => 1,
|
136
|
-
"log.flush.interval.messages" => 10000,
|
137
|
-
"log.flush.interval.ms" => 1000,
|
138
|
-
"log.retention.hours" => 168,
|
139
|
-
"log.segment.bytes" => 536870912,
|
140
|
-
"log.cleanup.interval.mins" => 1,
|
141
|
-
"zookeeper.connect" => "localhost:2181",
|
142
|
-
"zookeeper.connection.timeout.ms" => 1000000,
|
143
|
-
"kafka.metrics.polling.interval.secs" => 5,
|
144
|
-
"kafka.metrics.reporters" => "kafka.metrics.KafkaCSVMetricsReporter",
|
145
|
-
"kafka.csv.metrics.dir" => "#{ROOT_DIRECTORY}/tmp/kafka_metrics",
|
146
|
-
"kafka.csv.metrics.reporter.enabled" => "false",
|
147
|
-
}
|
148
|
-
|
149
|
-
def initialize(id, port, partition_count = 1)
|
150
|
-
@id = id
|
151
|
-
@port = port
|
152
|
-
@jr = JavaRunner.new("broker_#{id}",
|
153
|
-
"#{ROOT_DIRECTORY}/spec/bin/kafka-run-class.sh kafka.Kafka",
|
154
|
-
port,
|
155
|
-
DEFAULT_PROPERTIES.merge(
|
156
|
-
"broker.id" => id,
|
157
|
-
"port" => port,
|
158
|
-
"log.dir" => "#{ROOT_DIRECTORY}/tmp/kafka-logs_#{id}",
|
159
|
-
"num.partitions" => partition_count
|
160
|
-
))
|
161
|
-
end
|
162
|
-
|
163
|
-
def pid
|
164
|
-
@jr.pid
|
165
|
-
end
|
166
|
-
|
167
|
-
def start
|
168
|
-
@jr.start
|
169
|
-
end
|
170
|
-
|
171
|
-
def stop
|
172
|
-
@jr.stop
|
173
|
-
end
|
174
|
-
|
175
|
-
def without_process
|
176
|
-
@jr.without_process { yield }
|
177
|
-
end
|
178
|
-
end
|
179
|
-
|
180
|
-
class ZookeeperRunner
|
181
|
-
def initialize
|
182
|
-
@jr = JavaRunner.new("zookeeper",
|
183
|
-
"#{ROOT_DIRECTORY}/spec/bin/kafka-run-class.sh org.apache.zookeeper.server.quorum.QuorumPeerMain",
|
184
|
-
2181,
|
185
|
-
:dataDir => "#{ROOT_DIRECTORY}/tmp/zookeeper",
|
186
|
-
:clientPort => 2181,
|
187
|
-
:maxClientCnxns => 0)
|
188
|
-
end
|
189
|
-
|
190
|
-
def pid
|
191
|
-
@jr.pid
|
192
|
-
end
|
193
|
-
|
194
|
-
def start
|
195
|
-
@jr.start
|
196
|
-
end
|
197
|
-
|
198
|
-
def stop
|
199
|
-
@jr.stop
|
200
|
-
end
|
201
|
-
end
|