emque-producing 0.0.3 → 1.0.0.beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -1
- data/README.md +10 -12
- data/emque-producing.gemspec +6 -7
- data/lib/emque/producing/configuration.rb +0 -3
- data/lib/emque/producing/producing.rb +1 -4
- data/lib/emque/producing/version.rb +1 -1
- data/spec/producing/configuration_spec.rb +0 -6
- metadata +18 -42
- data/Gemfile.lock +0 -64
- data/lib/emque/producing/publisher/kafka.rb +0 -25
- data/spec/bin/kafka-run-class.sh +0 -67
- data/spec/kafka_spec_helper.rb +0 -33
- data/spec/producing/publisher_spec.rb +0 -20
- data/spec/test_cluster.rb +0 -201
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b77c1aea181103402f745688487ef3de2b03741c
|
4
|
+
data.tar.gz: 97051666e37ea9f28bd626d038cb8c85f60121a3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a66f000eb81b489e4d655b475ddd6061c58a45190c17259aaa89f62224c35521a2f99ccee6fc8a2aceaf9ca7a11ffdfd815d2ceb909eb510df255f6ff906a51f
|
7
|
+
data.tar.gz: f9779c82e7f934dce063a0143410133c94e48325fb9a4376238c1617c5a8dbac78dd5c27f14bb2a7ec37b9d70896b3895aa70fd5b77627554d5d5a2e444bf2b8
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -4,8 +4,7 @@ teamsnap/emque-producing](https://www.codeship.io/projects/2ca7fd90-1785-0132-5f
|
|
4
4
|
# Emque Producing
|
5
5
|
|
6
6
|
Define and send messages with Ruby to a variety of [message brokers](http://en.wikipedia.org/wiki/Message_broker).
|
7
|
-
|
8
|
-
[Kafka](http://kafka.apache.org/).
|
7
|
+
The only currently supported message broker is [RabbitMQ](https://www.rabbitmq.com)
|
9
8
|
|
10
9
|
This is a library that pairs nicely with [Emque
|
11
10
|
Consuming](https://www.github.com/teamsnap/emque-consuming), a framework for
|
@@ -19,12 +18,6 @@ Add these lines to your application's Gemfile, depending on your message broker:
|
|
19
18
|
gem "emque-producing"
|
20
19
|
gem "bunny", "~> 1.4.1"
|
21
20
|
|
22
|
-
or
|
23
|
-
|
24
|
-
# for Kafka, poseidon is used
|
25
|
-
gem "emque-producing"
|
26
|
-
gem "poseidon", "0.0.4"
|
27
|
-
|
28
21
|
And then execute:
|
29
22
|
|
30
23
|
$ bundle
|
@@ -41,8 +34,6 @@ Or install it yourself as:
|
|
41
34
|
c.app_name = "app"
|
42
35
|
c.publishing_adapter = :rabbitmq
|
43
36
|
c.rabbitmq_options = { :url => "amqp://guest:guest@localhost:5672" }
|
44
|
-
#c.kafka_options = { :seed_brokers => [localhost:9092],
|
45
|
-
# :producer_options => {} }
|
46
37
|
c.error_handlers << Proc.new {|ex,context|
|
47
38
|
# notify/log
|
48
39
|
}
|
@@ -62,13 +53,15 @@ Or install it yourself as:
|
|
62
53
|
message = MyMessage.new({:first_property => 1, :another_property => "another"})
|
63
54
|
message.publish
|
64
55
|
|
56
|
+
For a more thorough guide to creating new messages and/or message producing
|
57
|
+
applications, [please read the wiki
|
58
|
+
entry](https://github.com/teamsnap/emque-producing/wiki/Creating-New-Producing-Applications)
|
59
|
+
|
65
60
|
## Requirements
|
66
61
|
|
67
62
|
* Ruby 1.9.3 or higher
|
68
63
|
* RabbitMQ 3.x
|
69
64
|
* Bunny 1.4.x
|
70
|
-
* Kafka 0.8.1
|
71
|
-
* Poseidon 0.0.4
|
72
65
|
|
73
66
|
## Tests
|
74
67
|
|
@@ -96,3 +89,8 @@ adapter which may be convenient in the case of CI environments.
|
|
96
89
|
3. Commit your changes (`git commit -am 'Add some feature'`)
|
97
90
|
4. Push to the branch (`git push origin my-new-feature`)
|
98
91
|
5. Create new Pull Request
|
92
|
+
|
93
|
+
## Roadmap
|
94
|
+
|
95
|
+
Kafka would make for a good adapter to be added to emque-producing. Anyone
|
96
|
+
wishing to submit a PR can use the RabbitMQ adapter as a model.
|
data/emque-producing.gemspec
CHANGED
@@ -21,13 +21,12 @@ Gem::Specification.new do |spec|
|
|
21
21
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
22
22
|
spec.require_paths = ["lib"]
|
23
23
|
|
24
|
-
spec.add_dependency "oj",
|
25
|
-
spec.add_dependency "virtus",
|
24
|
+
spec.add_dependency "oj", "~> 2.11"
|
25
|
+
spec.add_dependency "virtus", "~> 1.0"
|
26
26
|
|
27
|
-
spec.add_development_dependency "bundler", "~> 1.
|
28
|
-
spec.add_development_dependency "rake"
|
29
|
-
spec.add_development_dependency "rspec", "~> 3.
|
27
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
28
|
+
spec.add_development_dependency "rake", "~> 10.4"
|
29
|
+
spec.add_development_dependency "rspec", "~> 3.2"
|
30
30
|
spec.add_development_dependency "pry"
|
31
|
-
spec.add_development_dependency "
|
32
|
-
spec.add_development_dependency "bunny", "~> 1.4.1"
|
31
|
+
spec.add_development_dependency "bunny", "~> 1.7"
|
33
32
|
end
|
@@ -3,7 +3,6 @@ module Emque
|
|
3
3
|
class Configuration
|
4
4
|
attr_accessor :app_name
|
5
5
|
attr_accessor :publishing_adapter
|
6
|
-
attr_accessor :kafka_options
|
7
6
|
attr_accessor :rabbitmq_options
|
8
7
|
attr_accessor :error_handlers
|
9
8
|
attr_accessor :log_publish_message
|
@@ -15,8 +14,6 @@ module Emque
|
|
15
14
|
@error_handlers = []
|
16
15
|
@log_publish_message = false
|
17
16
|
@publish_messages = true
|
18
|
-
@kafka_options = { :seed_brokers => ["localhost:9092"],
|
19
|
-
:producer_options => {} }
|
20
17
|
@rabbitmq_options = { :url => "amqp://guest:guest@localhost:5672" }
|
21
18
|
end
|
22
19
|
end
|
@@ -33,10 +33,7 @@ module Emque
|
|
33
33
|
def publisher
|
34
34
|
return @publisher unless @publisher.nil?
|
35
35
|
|
36
|
-
if (configuration.publishing_adapter == :
|
37
|
-
require "emque/producing/publisher/kafka"
|
38
|
-
@publisher = Emque::Producing::Publisher::Kafka.new
|
39
|
-
elsif (configuration.publishing_adapter == :rabbitmq)
|
36
|
+
if (configuration.publishing_adapter == :rabbitmq)
|
40
37
|
require "emque/producing/publisher/rabbitmq"
|
41
38
|
@publisher = Emque::Producing::Publisher::RabbitMq.new
|
42
39
|
else
|
@@ -6,16 +6,10 @@ describe Emque::Producing::Configuration do
|
|
6
6
|
it "provides default values" do
|
7
7
|
expect(subject.app_name).to eq ""
|
8
8
|
expect(subject.error_handlers).to eq []
|
9
|
-
expect(subject.kafka_options[:seed_brokers]).to eq ["localhost:9092"]
|
10
9
|
end
|
11
10
|
|
12
11
|
it "allows app_name to be overwritten" do
|
13
12
|
subject.app_name = "my app"
|
14
13
|
expect(subject.app_name).to eq "my app"
|
15
14
|
end
|
16
|
-
|
17
|
-
it "allows seed_brokers to be overwritten" do
|
18
|
-
subject.kafka_options[:seed_brokers] = ["kafka1:9092", "kafka2:9092"]
|
19
|
-
expect(subject.kafka_options[:seed_brokers]).to eq ["kafka1:9092", "kafka2:9092"]
|
20
|
-
end
|
21
15
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: emque-producing
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 1.0.0.beta2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Emily Dobervich
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-
|
12
|
+
date: 2015-03-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: oj
|
@@ -17,70 +17,70 @@ dependencies:
|
|
17
17
|
requirements:
|
18
18
|
- - "~>"
|
19
19
|
- !ruby/object:Gem::Version
|
20
|
-
version: 2.
|
20
|
+
version: '2.11'
|
21
21
|
type: :runtime
|
22
22
|
prerelease: false
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
25
|
- - "~>"
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version: 2.
|
27
|
+
version: '2.11'
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
29
|
name: virtus
|
30
30
|
requirement: !ruby/object:Gem::Requirement
|
31
31
|
requirements:
|
32
32
|
- - "~>"
|
33
33
|
- !ruby/object:Gem::Version
|
34
|
-
version: 1.0
|
34
|
+
version: '1.0'
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
39
|
- - "~>"
|
40
40
|
- !ruby/object:Gem::Version
|
41
|
-
version: 1.0
|
41
|
+
version: '1.0'
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
name: bundler
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
45
45
|
requirements:
|
46
46
|
- - "~>"
|
47
47
|
- !ruby/object:Gem::Version
|
48
|
-
version: '1.
|
48
|
+
version: '1.7'
|
49
49
|
type: :development
|
50
50
|
prerelease: false
|
51
51
|
version_requirements: !ruby/object:Gem::Requirement
|
52
52
|
requirements:
|
53
53
|
- - "~>"
|
54
54
|
- !ruby/object:Gem::Version
|
55
|
-
version: '1.
|
55
|
+
version: '1.7'
|
56
56
|
- !ruby/object:Gem::Dependency
|
57
57
|
name: rake
|
58
58
|
requirement: !ruby/object:Gem::Requirement
|
59
59
|
requirements:
|
60
|
-
- - "
|
60
|
+
- - "~>"
|
61
61
|
- !ruby/object:Gem::Version
|
62
|
-
version: '
|
62
|
+
version: '10.4'
|
63
63
|
type: :development
|
64
64
|
prerelease: false
|
65
65
|
version_requirements: !ruby/object:Gem::Requirement
|
66
66
|
requirements:
|
67
|
-
- - "
|
67
|
+
- - "~>"
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version: '
|
69
|
+
version: '10.4'
|
70
70
|
- !ruby/object:Gem::Dependency
|
71
71
|
name: rspec
|
72
72
|
requirement: !ruby/object:Gem::Requirement
|
73
73
|
requirements:
|
74
74
|
- - "~>"
|
75
75
|
- !ruby/object:Gem::Version
|
76
|
-
version: '3.
|
76
|
+
version: '3.2'
|
77
77
|
type: :development
|
78
78
|
prerelease: false
|
79
79
|
version_requirements: !ruby/object:Gem::Requirement
|
80
80
|
requirements:
|
81
81
|
- - "~>"
|
82
82
|
- !ruby/object:Gem::Version
|
83
|
-
version: '3.
|
83
|
+
version: '3.2'
|
84
84
|
- !ruby/object:Gem::Dependency
|
85
85
|
name: pry
|
86
86
|
requirement: !ruby/object:Gem::Requirement
|
@@ -95,34 +95,20 @@ dependencies:
|
|
95
95
|
- - ">="
|
96
96
|
- !ruby/object:Gem::Version
|
97
97
|
version: '0'
|
98
|
-
- !ruby/object:Gem::Dependency
|
99
|
-
name: poseidon
|
100
|
-
requirement: !ruby/object:Gem::Requirement
|
101
|
-
requirements:
|
102
|
-
- - '='
|
103
|
-
- !ruby/object:Gem::Version
|
104
|
-
version: 0.0.4
|
105
|
-
type: :development
|
106
|
-
prerelease: false
|
107
|
-
version_requirements: !ruby/object:Gem::Requirement
|
108
|
-
requirements:
|
109
|
-
- - '='
|
110
|
-
- !ruby/object:Gem::Version
|
111
|
-
version: 0.0.4
|
112
98
|
- !ruby/object:Gem::Dependency
|
113
99
|
name: bunny
|
114
100
|
requirement: !ruby/object:Gem::Requirement
|
115
101
|
requirements:
|
116
102
|
- - "~>"
|
117
103
|
- !ruby/object:Gem::Version
|
118
|
-
version: 1.
|
104
|
+
version: '1.7'
|
119
105
|
type: :development
|
120
106
|
prerelease: false
|
121
107
|
version_requirements: !ruby/object:Gem::Requirement
|
122
108
|
requirements:
|
123
109
|
- - "~>"
|
124
110
|
- !ruby/object:Gem::Version
|
125
|
-
version: 1.
|
111
|
+
version: '1.7'
|
126
112
|
description: Define and send messages to a variety of message brokers
|
127
113
|
email:
|
128
114
|
- emily@teamsnap.com
|
@@ -133,7 +119,6 @@ extra_rdoc_files: []
|
|
133
119
|
files:
|
134
120
|
- ".gitignore"
|
135
121
|
- Gemfile
|
136
|
-
- Gemfile.lock
|
137
122
|
- LICENSE.txt
|
138
123
|
- README.md
|
139
124
|
- Rakefile
|
@@ -146,18 +131,13 @@ files:
|
|
146
131
|
- lib/emque/producing/message/message_with_changeset.rb
|
147
132
|
- lib/emque/producing/producing.rb
|
148
133
|
- lib/emque/producing/publisher/base.rb
|
149
|
-
- lib/emque/producing/publisher/kafka.rb
|
150
134
|
- lib/emque/producing/publisher/rabbitmq.rb
|
151
135
|
- lib/emque/producing/version.rb
|
152
|
-
- spec/bin/kafka-run-class.sh
|
153
|
-
- spec/kafka_spec_helper.rb
|
154
136
|
- spec/producing/configuration_spec.rb
|
155
137
|
- spec/producing/message/changes_payload_generator_spec.rb
|
156
138
|
- spec/producing/message/message_spec.rb
|
157
139
|
- spec/producing/producing_spec.rb
|
158
|
-
- spec/producing/publisher_spec.rb
|
159
140
|
- spec/spec_helper.rb
|
160
|
-
- spec/test_cluster.rb
|
161
141
|
homepage: ''
|
162
142
|
licenses:
|
163
143
|
- MIT
|
@@ -173,9 +153,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
173
153
|
version: 1.9.3
|
174
154
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
175
155
|
requirements:
|
176
|
-
- - "
|
156
|
+
- - ">"
|
177
157
|
- !ruby/object:Gem::Version
|
178
|
-
version:
|
158
|
+
version: 1.3.1
|
179
159
|
requirements: []
|
180
160
|
rubyforge_project:
|
181
161
|
rubygems_version: 2.2.2
|
@@ -183,12 +163,8 @@ signing_key:
|
|
183
163
|
specification_version: 4
|
184
164
|
summary: Define and send messages to a variety of message brokers
|
185
165
|
test_files:
|
186
|
-
- spec/bin/kafka-run-class.sh
|
187
|
-
- spec/kafka_spec_helper.rb
|
188
166
|
- spec/producing/configuration_spec.rb
|
189
167
|
- spec/producing/message/changes_payload_generator_spec.rb
|
190
168
|
- spec/producing/message/message_spec.rb
|
191
169
|
- spec/producing/producing_spec.rb
|
192
|
-
- spec/producing/publisher_spec.rb
|
193
170
|
- spec/spec_helper.rb
|
194
|
-
- spec/test_cluster.rb
|
data/Gemfile.lock
DELETED
@@ -1,64 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
emque-producing (0.0.3)
|
5
|
-
oj (~> 2.10.2)
|
6
|
-
virtus (~> 1.0.3)
|
7
|
-
|
8
|
-
GEM
|
9
|
-
remote: https://rubygems.org/
|
10
|
-
specs:
|
11
|
-
amq-protocol (1.9.2)
|
12
|
-
axiom-types (0.1.1)
|
13
|
-
descendants_tracker (~> 0.0.4)
|
14
|
-
ice_nine (~> 0.11.0)
|
15
|
-
thread_safe (~> 0.3, >= 0.3.1)
|
16
|
-
bunny (1.4.1)
|
17
|
-
amq-protocol (>= 1.9.2)
|
18
|
-
coderay (1.1.0)
|
19
|
-
coercible (1.0.0)
|
20
|
-
descendants_tracker (~> 0.0.1)
|
21
|
-
descendants_tracker (0.0.4)
|
22
|
-
thread_safe (~> 0.3, >= 0.3.1)
|
23
|
-
diff-lcs (1.2.5)
|
24
|
-
equalizer (0.0.9)
|
25
|
-
ice_nine (0.11.0)
|
26
|
-
method_source (0.8.2)
|
27
|
-
oj (2.10.2)
|
28
|
-
poseidon (0.0.4)
|
29
|
-
pry (0.9.12.4)
|
30
|
-
coderay (~> 1.0)
|
31
|
-
method_source (~> 0.8)
|
32
|
-
slop (~> 3.4)
|
33
|
-
rake (10.1.1)
|
34
|
-
rspec (3.1.0)
|
35
|
-
rspec-core (~> 3.1.0)
|
36
|
-
rspec-expectations (~> 3.1.0)
|
37
|
-
rspec-mocks (~> 3.1.0)
|
38
|
-
rspec-core (3.1.5)
|
39
|
-
rspec-support (~> 3.1.0)
|
40
|
-
rspec-expectations (3.1.2)
|
41
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
42
|
-
rspec-support (~> 3.1.0)
|
43
|
-
rspec-mocks (3.1.2)
|
44
|
-
rspec-support (~> 3.1.0)
|
45
|
-
rspec-support (3.1.1)
|
46
|
-
slop (3.4.7)
|
47
|
-
thread_safe (0.3.4)
|
48
|
-
virtus (1.0.3)
|
49
|
-
axiom-types (~> 0.1)
|
50
|
-
coercible (~> 1.0)
|
51
|
-
descendants_tracker (~> 0.0, >= 0.0.3)
|
52
|
-
equalizer (~> 0.0, >= 0.0.9)
|
53
|
-
|
54
|
-
PLATFORMS
|
55
|
-
ruby
|
56
|
-
|
57
|
-
DEPENDENCIES
|
58
|
-
bundler (~> 1.0)
|
59
|
-
bunny (~> 1.4.1)
|
60
|
-
emque-producing!
|
61
|
-
poseidon (= 0.0.4)
|
62
|
-
pry
|
63
|
-
rake
|
64
|
-
rspec (~> 3.1)
|
@@ -1,25 +0,0 @@
|
|
1
|
-
require "poseidon"
|
2
|
-
|
3
|
-
module Emque
|
4
|
-
module Producing
|
5
|
-
module Publisher
|
6
|
-
class Kafka < Emque::Producing::Publisher::Base
|
7
|
-
def initialize
|
8
|
-
@producer = Poseidon::Producer.new(
|
9
|
-
Emque::Producing.configuration.kafka_options[:seed_brokers],
|
10
|
-
"producer_#{host_name}_#{Process.pid}",
|
11
|
-
Emque::Producing.configuration.kafka_options[:producer_options])
|
12
|
-
end
|
13
|
-
|
14
|
-
def publish(topic, message_type, message, key = nil)
|
15
|
-
begin
|
16
|
-
msg = Poseidon::MessageToSend.new(topic, message, key)
|
17
|
-
@producer.send_messages([msg])
|
18
|
-
rescue => e
|
19
|
-
handle_error(e)
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
data/spec/bin/kafka-run-class.sh
DELETED
@@ -1,67 +0,0 @@
|
|
1
|
-
#!/bin/bash
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one or more
|
3
|
-
# contributor license agreements. See the NOTICE file distributed with
|
4
|
-
# this work for additional information regarding copyright ownership.
|
5
|
-
# The ASF licenses this file to You under the Apache License, Version 2.0
|
6
|
-
# (the "License"); you may not use this file except in compliance with
|
7
|
-
# the License. You may obtain a copy of the License at
|
8
|
-
#
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
-
#
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
-
# See the License for the specific language governing permissions and
|
15
|
-
# limitations under the License.
|
16
|
-
|
17
|
-
if [ $# -lt 1 ];
|
18
|
-
then
|
19
|
-
echo "USAGE: $0 classname [opts]"
|
20
|
-
exit 1
|
21
|
-
fi
|
22
|
-
|
23
|
-
if [-z "$SCALA_VERSION" ]; then
|
24
|
-
SCALA_VERSION=2.8.0
|
25
|
-
fi
|
26
|
-
|
27
|
-
# assume all dependencies have been packaged into one jar with sbt-assembly's task "assembly-package-dependency"
|
28
|
-
for file in $KAFKA_PATH/core/target/scala-$SCALA_VERSION/*.jar;
|
29
|
-
do
|
30
|
-
CLASSPATH=$CLASSPATH:$file
|
31
|
-
done
|
32
|
-
|
33
|
-
for file in $KAFKA_PATH/perf/target/scala-$SCALA_VERSION/kafka*.jar;
|
34
|
-
do
|
35
|
-
CLASSPATH=$CLASSPATH:$file
|
36
|
-
done
|
37
|
-
|
38
|
-
# classpath addition for release
|
39
|
-
for file in $KAFKA_PATH/libs/*.jar;
|
40
|
-
do
|
41
|
-
CLASSPATH=$CLASSPATH:$file
|
42
|
-
done
|
43
|
-
|
44
|
-
for file in $KAFKA_PATH/kafka*.jar;
|
45
|
-
do
|
46
|
-
CLASSPATH=$CLASSPATH:$file
|
47
|
-
done
|
48
|
-
|
49
|
-
if [ -z "$KAFKA_JMX_OPTS" ]; then
|
50
|
-
KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false "
|
51
|
-
fi
|
52
|
-
|
53
|
-
if [ -z "$KAFKA_OPTS" ]; then
|
54
|
-
KAFKA_OPTS="-Xmx512M -server -Dlog4j.configuration=file:$KAFKA_PATH/config/log4j.properties"
|
55
|
-
fi
|
56
|
-
|
57
|
-
if [ $JMX_PORT ]; then
|
58
|
-
KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT "
|
59
|
-
fi
|
60
|
-
|
61
|
-
if [ -z "$JAVA_HOME" ]; then
|
62
|
-
JAVA="java"
|
63
|
-
else
|
64
|
-
JAVA="$JAVA_HOME/bin/java"
|
65
|
-
fi
|
66
|
-
|
67
|
-
exec $JAVA $KAFKA_OPTS $KAFKA_JMX_OPTS -cp $CLASSPATH "$@"
|
data/spec/kafka_spec_helper.rb
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
ROOT_DIRECTORY = File.absolute_path(File.dirname(__FILE__) + "/../")
|
2
|
-
|
3
|
-
require "test_cluster"
|
4
|
-
require "spec_helper"
|
5
|
-
|
6
|
-
unless File.directory?(File.join(ROOT_DIRECTORY, "kafka_2.8.0-0.8.1"))
|
7
|
-
puts "\033[0;32m"
|
8
|
-
puts "*" * 83
|
9
|
-
puts "Downloading kafka"
|
10
|
-
puts "*" * 83
|
11
|
-
puts "\033[0;0m"
|
12
|
-
|
13
|
-
system(
|
14
|
-
"cd #{ROOT_DIRECTORY} && curl https://archive.apache.org/dist/kafka/0.8.1/kafka_2.8.0-0.8.1.tgz | tar xz"
|
15
|
-
)
|
16
|
-
end
|
17
|
-
|
18
|
-
ENV["KAFKA_PATH"] = File.join(ROOT_DIRECTORY, "kafka_2.8.0-0.8.1")
|
19
|
-
ENV["SCALA_VERSION"] = "2.8.0"
|
20
|
-
|
21
|
-
RSpec.configure do |config|
|
22
|
-
config.before(:suite) do
|
23
|
-
JavaRunner.remove_tmp
|
24
|
-
JavaRunner.set_kafka_path!
|
25
|
-
$tc = TestCluster.new
|
26
|
-
$tc.start
|
27
|
-
sleep 5
|
28
|
-
end
|
29
|
-
|
30
|
-
config.after(:suite) do
|
31
|
-
$tc.stop
|
32
|
-
end
|
33
|
-
end
|
@@ -1,20 +0,0 @@
|
|
1
|
-
require "spec_helper"
|
2
|
-
require "emque/producing/publisher/kafka"
|
3
|
-
|
4
|
-
describe Emque::Producing::Publisher do
|
5
|
-
describe "#publish" do
|
6
|
-
context "when error handler raises an exception" do
|
7
|
-
it "handles the exception" do
|
8
|
-
expect_any_instance_of(Poseidon::Producer). to receive(:send_messages).and_raise
|
9
|
-
Emque::Producing.configure do |c|
|
10
|
-
c.error_handlers << Proc.new {|ex,context|
|
11
|
-
raise "something"
|
12
|
-
}
|
13
|
-
end
|
14
|
-
Emque::Producing.logger = nil
|
15
|
-
publisher = Emque::Producing::Publisher::Kafka.new
|
16
|
-
publisher.publish("mytopic", "message.type", "mymessage")
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
data/spec/test_cluster.rb
DELETED
@@ -1,201 +0,0 @@
|
|
1
|
-
# from https://github.com/bpot/poseidon
|
2
|
-
|
3
|
-
require 'daemon_controller'
|
4
|
-
|
5
|
-
class TestCluster
|
6
|
-
attr_reader :broker, :zookeeper
|
7
|
-
def initialize
|
8
|
-
@zookeeper = ZookeeperRunner.new
|
9
|
-
@broker = BrokerRunner.new(0, 9092)
|
10
|
-
end
|
11
|
-
|
12
|
-
def start
|
13
|
-
@zookeeper.start
|
14
|
-
@broker.start
|
15
|
-
end
|
16
|
-
|
17
|
-
def stop
|
18
|
-
@zookeeper.stop
|
19
|
-
@broker.stop
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
class JavaRunner
|
24
|
-
def self.remove_tmp
|
25
|
-
FileUtils.rm_rf("#{ROOT_DIRECTORY}/tmp")
|
26
|
-
end
|
27
|
-
|
28
|
-
def self.set_kafka_path!
|
29
|
-
JavaRunner.kafka_path = File.join(ROOT_DIRECTORY, "kafka-0.8.0-src")
|
30
|
-
end
|
31
|
-
|
32
|
-
def self.kafka_path=(kafka_path)
|
33
|
-
@kafka_path = kafka_path
|
34
|
-
end
|
35
|
-
|
36
|
-
def self.kafka_path
|
37
|
-
@kafka_path
|
38
|
-
end
|
39
|
-
|
40
|
-
attr_reader :pid
|
41
|
-
def initialize(id, start_cmd, port, properties = {})
|
42
|
-
@id = id
|
43
|
-
@properties = properties
|
44
|
-
@pid = nil
|
45
|
-
@start_cmd = start_cmd
|
46
|
-
@port = port
|
47
|
-
end
|
48
|
-
|
49
|
-
def start
|
50
|
-
write_properties
|
51
|
-
run
|
52
|
-
end
|
53
|
-
|
54
|
-
def stop
|
55
|
-
daemon_controller.stop
|
56
|
-
end
|
57
|
-
|
58
|
-
def without_process
|
59
|
-
stop
|
60
|
-
begin
|
61
|
-
yield
|
62
|
-
ensure
|
63
|
-
start
|
64
|
-
sleep 5
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
private
|
69
|
-
|
70
|
-
def daemon_controller
|
71
|
-
@dc ||= DaemonController.new(
|
72
|
-
:identifier => @id,
|
73
|
-
:start_command => "#{@start_cmd} #{config_path} >>#{log_path} 2>&1 & echo $! > #{pid_path}",
|
74
|
-
:ping_command => [:tcp, '127.0.0.1', @port],
|
75
|
-
:pid_file => pid_path,
|
76
|
-
:log_file => log_path,
|
77
|
-
:start_timeout => 25
|
78
|
-
)
|
79
|
-
end
|
80
|
-
|
81
|
-
def run
|
82
|
-
FileUtils.mkdir_p(log_dir)
|
83
|
-
FileUtils.mkdir_p(pid_dir)
|
84
|
-
daemon_controller.start
|
85
|
-
end
|
86
|
-
|
87
|
-
def write_properties
|
88
|
-
FileUtils.mkdir_p(config_dir)
|
89
|
-
File.open(config_path, "w+") do |f|
|
90
|
-
@properties.each do |k,v|
|
91
|
-
f.puts "#{k}=#{v}"
|
92
|
-
end
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
|
-
def pid_path
|
97
|
-
"#{pid_dir}/#{@id}.pid"
|
98
|
-
end
|
99
|
-
|
100
|
-
def pid_dir
|
101
|
-
"#{file_path}/pid"
|
102
|
-
end
|
103
|
-
|
104
|
-
def log_path
|
105
|
-
"#{log_dir}/#{@id}.log"
|
106
|
-
end
|
107
|
-
|
108
|
-
def log_dir
|
109
|
-
"#{file_path}/log"
|
110
|
-
end
|
111
|
-
|
112
|
-
def config_path
|
113
|
-
"#{config_dir}/#{@id}.properties"
|
114
|
-
end
|
115
|
-
|
116
|
-
def config_dir
|
117
|
-
"#{file_path}/config"
|
118
|
-
end
|
119
|
-
|
120
|
-
def file_path
|
121
|
-
ROOT_DIRECTORY + "/tmp/"
|
122
|
-
end
|
123
|
-
end
|
124
|
-
|
125
|
-
class BrokerRunner
|
126
|
-
DEFAULT_PROPERTIES = {
|
127
|
-
"broker.id" => 0,
|
128
|
-
"port" => 9092,
|
129
|
-
"num.network.threads" => 2,
|
130
|
-
"num.io.threads" => 2,
|
131
|
-
"socket.send.buffer.bytes" => 1048576,
|
132
|
-
"socket.receive.buffer.bytes" => 1048576,
|
133
|
-
"socket.request.max.bytes" => 104857600,
|
134
|
-
"log.dir" => "#{ROOT_DIRECTORY}/tmp/kafka-logs",
|
135
|
-
"num.partitions" => 1,
|
136
|
-
"log.flush.interval.messages" => 10000,
|
137
|
-
"log.flush.interval.ms" => 1000,
|
138
|
-
"log.retention.hours" => 168,
|
139
|
-
"log.segment.bytes" => 536870912,
|
140
|
-
"log.cleanup.interval.mins" => 1,
|
141
|
-
"zookeeper.connect" => "localhost:2181",
|
142
|
-
"zookeeper.connection.timeout.ms" => 1000000,
|
143
|
-
"kafka.metrics.polling.interval.secs" => 5,
|
144
|
-
"kafka.metrics.reporters" => "kafka.metrics.KafkaCSVMetricsReporter",
|
145
|
-
"kafka.csv.metrics.dir" => "#{ROOT_DIRECTORY}/tmp/kafka_metrics",
|
146
|
-
"kafka.csv.metrics.reporter.enabled" => "false",
|
147
|
-
}
|
148
|
-
|
149
|
-
def initialize(id, port, partition_count = 1)
|
150
|
-
@id = id
|
151
|
-
@port = port
|
152
|
-
@jr = JavaRunner.new("broker_#{id}",
|
153
|
-
"#{ROOT_DIRECTORY}/spec/bin/kafka-run-class.sh kafka.Kafka",
|
154
|
-
port,
|
155
|
-
DEFAULT_PROPERTIES.merge(
|
156
|
-
"broker.id" => id,
|
157
|
-
"port" => port,
|
158
|
-
"log.dir" => "#{ROOT_DIRECTORY}/tmp/kafka-logs_#{id}",
|
159
|
-
"num.partitions" => partition_count
|
160
|
-
))
|
161
|
-
end
|
162
|
-
|
163
|
-
def pid
|
164
|
-
@jr.pid
|
165
|
-
end
|
166
|
-
|
167
|
-
def start
|
168
|
-
@jr.start
|
169
|
-
end
|
170
|
-
|
171
|
-
def stop
|
172
|
-
@jr.stop
|
173
|
-
end
|
174
|
-
|
175
|
-
def without_process
|
176
|
-
@jr.without_process { yield }
|
177
|
-
end
|
178
|
-
end
|
179
|
-
|
180
|
-
class ZookeeperRunner
|
181
|
-
def initialize
|
182
|
-
@jr = JavaRunner.new("zookeeper",
|
183
|
-
"#{ROOT_DIRECTORY}/spec/bin/kafka-run-class.sh org.apache.zookeeper.server.quorum.QuorumPeerMain",
|
184
|
-
2181,
|
185
|
-
:dataDir => "#{ROOT_DIRECTORY}/tmp/zookeeper",
|
186
|
-
:clientPort => 2181,
|
187
|
-
:maxClientCnxns => 0)
|
188
|
-
end
|
189
|
-
|
190
|
-
def pid
|
191
|
-
@jr.pid
|
192
|
-
end
|
193
|
-
|
194
|
-
def start
|
195
|
-
@jr.start
|
196
|
-
end
|
197
|
-
|
198
|
-
def stop
|
199
|
-
@jr.stop
|
200
|
-
end
|
201
|
-
end
|