turbine 1.0.0.pre → 1.0.0.pre2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -1
- data/.travis.yml +7 -1
- data/README.md +9 -2
- data/lib/turbine.rb +2 -0
- data/lib/turbine/consumer/kafka.rb +6 -2
- data/lib/turbine/kafka_helper.rb +87 -0
- data/lib/turbine/processor.rb +14 -9
- data/lib/turbine/rake_tasks.rb +55 -0
- data/lib/turbine/version.rb +1 -1
- data/spec/turbine/consumer/kafka_spec.rb +4 -4
- data/spec/turbine/processor_spec.rb +7 -2
- data/tasks/kafka.rake +1 -55
- data/turbine.gemspec +1 -1
- metadata +58 -57
- data/lib/turbine/rspec/kafka_helper.rb +0 -85
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a7a2a4bba52161cd063ecf520df92f103c8bffd0
|
4
|
+
data.tar.gz: 3eb0563a6418e04eb860fe8c902ef0fa2fd4e20a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: db3ec24b191a0fec4d9e6c52421857a6026c326e9b1094d184e7f60a4c10e2eb8630bfaf271c3a66334bd7fb8b21a7e88f208eeba7c9eef9a854f5b9e9c2faa3
|
7
|
+
data.tar.gz: d91d4f80a74401ce66b777a9f7cb0ef1bf71b03a9c05eb4c4278167a023bafd80a5b414b7f29a1cafc1f2e92d455f69f457888234686471cd8b54dd92b6f8c0d
|
data/.rubocop.yml
CHANGED
data/.travis.yml
CHANGED
@@ -1,6 +1,11 @@
|
|
1
1
|
language: ruby
|
2
2
|
install: gem install bundler
|
3
3
|
script: "bundle && bundle exec rake ci"
|
4
|
+
|
5
|
+
env:
|
6
|
+
global:
|
7
|
+
- JRUBY_OPTS="--server -J-Xms1500m -J-Xmx1500m -J-XX:+UseConcMarkSweepGC -J-XX:-UseGCOverheadLimit -J-XX:+CMSClassUnloadingEnabled"
|
8
|
+
|
4
9
|
rvm:
|
5
10
|
- 2.2.2
|
6
11
|
- jruby
|
@@ -8,6 +13,7 @@ rvm:
|
|
8
13
|
- rbx-2
|
9
14
|
|
10
15
|
matrix:
|
16
|
+
fast_finish: true
|
11
17
|
allow_failures:
|
12
|
-
-
|
18
|
+
- rvm: jruby-head
|
13
19
|
- rvm: rbx-2
|
data/README.md
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
![Turbine](https://raw.githubusercontent.com/tarcieri/turbine/master/turbine.png)
|
2
2
|
=======
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/turbine.svg)](http://rubygems.org/gems/turbine)
|
3
4
|
[![Build Status](https://travis-ci.org/tarcieri/turbine.svg)](https://travis-ci.org/tarcieri/turbine)
|
4
5
|
[![Code Climate](https://codeclimate.com/github/tarcieri/turbine/badges/gpa.svg)](https://codeclimate.com/github/tarcieri/turbine)
|
5
6
|
[![Coverage Status](https://coveralls.io/repos/tarcieri/turbine/badge.svg)](https://coveralls.io/r/tarcieri/turbine)
|
6
7
|
|
7
8
|
Fault-tolerant multithreaded stream processing for Ruby.
|
8
9
|
|
9
|
-
Turbine is a
|
10
|
+
Turbine is a performance-oriented stream processing library built on Zookeeper.
|
10
11
|
It presently supports Kafka as a message queue, but is designed to be pluggable
|
11
12
|
in order to potentially support other message queues in the future.
|
12
13
|
|
@@ -33,6 +34,12 @@ Or install it yourself as:
|
|
33
34
|
|
34
35
|
$ gem install turbine
|
35
36
|
|
37
|
+
## Getting Started
|
38
|
+
|
39
|
+
Please see the [Quickstart](https://github.com/tarcieri/turbine/wiki/Quickstart)
|
40
|
+
on the Wiki for a quick tutorial on writing your first stream processing job
|
41
|
+
using Turbine.
|
42
|
+
|
36
43
|
## Usage
|
37
44
|
|
38
45
|
Turbine presently supports stream processing from the Kafka message queue
|
@@ -59,7 +66,7 @@ processor.process(consumer) do |msg|
|
|
59
66
|
end
|
60
67
|
```
|
61
68
|
|
62
|
-
## Error
|
69
|
+
## Error Handling
|
63
70
|
|
64
71
|
By default, Turbine prints exceptions that occur during message processing to STDERR.
|
65
72
|
Chances are, you'd probably rather log them to an exception logging service.
|
data/lib/turbine.rb
CHANGED
@@ -11,8 +11,12 @@ module Turbine
|
|
11
11
|
def fetch
|
12
12
|
batch = nil
|
13
13
|
|
14
|
-
|
15
|
-
|
14
|
+
begin
|
15
|
+
@consumer.fetch commit: false do |partition, messages|
|
16
|
+
batch = Batch.new(messages, partition)
|
17
|
+
end
|
18
|
+
rescue Poseidon::Connection::ConnectionFailedError => ex
|
19
|
+
raise ConnectionError, ex.to_s, ex.backtrace
|
16
20
|
end
|
17
21
|
|
18
22
|
batch
|
@@ -0,0 +1,87 @@
|
|
1
|
+
require "open3"
|
2
|
+
require "poseidon"
|
3
|
+
|
4
|
+
module Turbine
|
5
|
+
# Helper functions for integration testing with Kafka
|
6
|
+
module KafkaHelper
|
7
|
+
extend self
|
8
|
+
|
9
|
+
ZOOKEEPER_ADDR = "localhost:2181"
|
10
|
+
KAFKA_ADDR = "localhost:9092"
|
11
|
+
|
12
|
+
def delete_topic(topic)
|
13
|
+
log "*** Deleting Kafka topic: #{topic}"
|
14
|
+
|
15
|
+
topic_command :delete, topic: topic
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_topic(topic)
|
19
|
+
log "*** Creating Kafka topic: #{topic}"
|
20
|
+
|
21
|
+
required_topic_command :create,
|
22
|
+
"replication-factor" => 1,
|
23
|
+
"partitions" => 1,
|
24
|
+
"topic" => topic
|
25
|
+
end
|
26
|
+
|
27
|
+
def list_topics
|
28
|
+
topic_command(:list).split("\n")
|
29
|
+
end
|
30
|
+
|
31
|
+
def topic_exists?(topic)
|
32
|
+
list_topics.include?(topic)
|
33
|
+
end
|
34
|
+
|
35
|
+
def fill_topic(topic, n = 100_000)
|
36
|
+
fail ArgumentError, "min messages is 1000" if n < 1000
|
37
|
+
|
38
|
+
producer = Poseidon::Producer.new([KAFKA_ADDR], "my_test_producer", type: :sync)
|
39
|
+
|
40
|
+
log "*** Filling topic with #{n} messages: #{topic}"
|
41
|
+
|
42
|
+
(n / 1000).times do |i|
|
43
|
+
messages = []
|
44
|
+
|
45
|
+
1000.times do |j|
|
46
|
+
n = (i * 1000 + j)
|
47
|
+
messages << Poseidon::MessageToSend.new(topic, n.to_s)
|
48
|
+
end
|
49
|
+
|
50
|
+
producer.send_messages(messages)
|
51
|
+
end
|
52
|
+
ensure
|
53
|
+
producer.close if producer
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def kafka_path
|
59
|
+
ENV["KAFKA_PATH"] || File.expand_path("../../../kafka", __FILE__)
|
60
|
+
end
|
61
|
+
|
62
|
+
def kafka_topics_bin_path
|
63
|
+
"#{kafka_path}/bin/kafka-topics.sh"
|
64
|
+
end
|
65
|
+
|
66
|
+
def kafka_args(args = {})
|
67
|
+
{ zookeeper: ZOOKEEPER_ADDR }.merge(args).map { |k, v| "--#{k} #{v}" }.join(" ")
|
68
|
+
end
|
69
|
+
|
70
|
+
def topic_command(command, args = {})
|
71
|
+
cmd = "#{kafka_topics_bin_path} --#{command} #{kafka_args(args)}"
|
72
|
+
stdout_str, _stderr_str, status = Open3.capture3(cmd)
|
73
|
+
return unless status.success?
|
74
|
+
stdout_str
|
75
|
+
end
|
76
|
+
|
77
|
+
def required_topic_command(command, args = {})
|
78
|
+
result = topic_command(command, args)
|
79
|
+
fail "Kafka command failed!" unless result
|
80
|
+
true
|
81
|
+
end
|
82
|
+
|
83
|
+
def log(message)
|
84
|
+
STDERR.puts(message)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
data/lib/turbine/processor.rb
CHANGED
@@ -2,23 +2,22 @@ module Turbine
|
|
2
2
|
# Multithreaded message processor
|
3
3
|
class Processor
|
4
4
|
# How long to sleep when busy waiting for the queue to empty
|
5
|
-
BUSY_WAIT_INTERVAL
|
5
|
+
BUSY_WAIT_INTERVAL = 0.0001
|
6
|
+
DEFAULT_DRAIN_TIMEOUT = 10
|
6
7
|
|
7
8
|
def initialize(*args)
|
8
9
|
@running = Concurrent::AtomicBoolean.new
|
9
10
|
@pool = Concurrent::ThreadPoolExecutor.new(*args)
|
10
11
|
@completed_count = Concurrent::AtomicFixnum.new
|
11
12
|
@pending = []
|
12
|
-
@error_handler =
|
13
|
-
STDERR.puts "*** Error processing message: #{ex.class} #{ex}\n#{ex.backtrace.join("\n")}"
|
14
|
-
end
|
13
|
+
@error_handler = method(:default_error_handler)
|
15
14
|
end
|
16
15
|
|
17
16
|
def process(consumer, &block)
|
18
17
|
fail ArgumentError, "no block given" unless block
|
19
18
|
processor_method = method(:process_batch)
|
20
|
-
|
21
19
|
@running.value = true
|
20
|
+
|
22
21
|
while @running.value && (batch = consumer.fetch)
|
23
22
|
enqueue_batch(batch)
|
24
23
|
|
@@ -31,6 +30,9 @@ module Turbine
|
|
31
30
|
|
32
31
|
commit_completions(consumer)
|
33
32
|
end
|
33
|
+
ensure
|
34
|
+
drain(DEFAULT_DRAIN_TIMEOUT)
|
35
|
+
commit_completions(consumer)
|
34
36
|
end
|
35
37
|
|
36
38
|
def stop
|
@@ -77,7 +79,7 @@ module Turbine
|
|
77
79
|
|
78
80
|
def process_batch(batch, block)
|
79
81
|
for index in (0...batch.size)
|
80
|
-
msg = batch[index]
|
82
|
+
msg = batch[index].value
|
81
83
|
|
82
84
|
begin
|
83
85
|
block.call(msg)
|
@@ -91,12 +93,15 @@ module Turbine
|
|
91
93
|
batch.complete
|
92
94
|
end
|
93
95
|
|
96
|
+
# We exceeded the pool's queue, so busy-wait and retry
|
97
|
+
# TODO: more intelligent busy-waiting strategy
|
94
98
|
def busy_wait(consumer)
|
95
99
|
commit_completions(consumer)
|
96
|
-
|
97
|
-
# We exceeded the pool's queue, so busy-wait and retry
|
98
|
-
# TODO: more intelligent busy-waiting strategy
|
99
100
|
sleep BUSY_WAIT_INTERVAL
|
100
101
|
end
|
102
|
+
|
103
|
+
def default_error_handler(ex, _msg)
|
104
|
+
STDERR.puts "*** Error processing message: #{ex.class} #{ex}\n#{ex.backtrace.join("\n")}"
|
105
|
+
end
|
101
106
|
end
|
102
107
|
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require "rake/clean"
|
2
|
+
require "colorize"
|
3
|
+
require "socket"
|
4
|
+
require "timeout"
|
5
|
+
|
6
|
+
KAFKA_PORT = 9092
|
7
|
+
START_TIMEOUT = 10
|
8
|
+
|
9
|
+
namespace :kafka do
|
10
|
+
KAFKA_VERSION = "0.8.2.1"
|
11
|
+
KAFKA_TARBALL = "kafka_2.10-#{KAFKA_VERSION}.tgz"
|
12
|
+
|
13
|
+
task download: "tmp/#{KAFKA_TARBALL}"
|
14
|
+
directory "tmp"
|
15
|
+
|
16
|
+
file "tmp/#{KAFKA_TARBALL}" => "tmp" do
|
17
|
+
puts "#{'***'.blue} #{'Downloading Kafka'.light_white}"
|
18
|
+
url = "https://www.apache.org/dist/kafka/#{KAFKA_VERSION}/kafka_2.10-#{KAFKA_VERSION}.tgz"
|
19
|
+
sh "curl #{url} -o tmp/#{KAFKA_TARBALL}"
|
20
|
+
end
|
21
|
+
|
22
|
+
task install: :download do
|
23
|
+
puts "#{'***'.blue} #{'Unpacking Kafka'.light_white}"
|
24
|
+
|
25
|
+
rm_rf "kafka" if File.exist? "kafka"
|
26
|
+
sh "tar -zxf tmp/#{KAFKA_TARBALL}"
|
27
|
+
mv "kafka_2.10-#{KAFKA_VERSION}", "kafka"
|
28
|
+
end
|
29
|
+
|
30
|
+
task start: %w(kafka zookeeper:start) do
|
31
|
+
puts "#{'***'.blue} #{'Starting Kafka'.light_white}"
|
32
|
+
sh "cd kafka && bin/kafka-server-start.sh config/server.properties &"
|
33
|
+
|
34
|
+
Timeout.timeout(START_TIMEOUT) do
|
35
|
+
begin
|
36
|
+
socket = TCPSocket.open("localhost", 9092)
|
37
|
+
rescue Errno::ECONNREFUSED
|
38
|
+
sleep 0.01
|
39
|
+
retry
|
40
|
+
end
|
41
|
+
|
42
|
+
socket.close
|
43
|
+
end
|
44
|
+
|
45
|
+
# Give Kafka some time to finish printing startup messages
|
46
|
+
sleep 0.5
|
47
|
+
puts "#{'***'.blue} #{'Kafka started!'.light_white}"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
file "kafka" do
|
52
|
+
Rake::Task["kafka:install"].invoke
|
53
|
+
end
|
54
|
+
|
55
|
+
CLEAN.include "tmp", "kafka"
|
data/lib/turbine/version.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require "spec_helper"
|
2
2
|
require "turbine/consumer/kafka"
|
3
|
-
require "turbine/
|
3
|
+
require "turbine/kafka_helper"
|
4
4
|
require "benchmark"
|
5
5
|
|
6
6
|
RSpec.describe Turbine::Consumer::Kafka do
|
@@ -27,12 +27,12 @@ RSpec.describe Turbine::Consumer::Kafka do
|
|
27
27
|
timestamp = Time.now.strftime("%Y%m%d%H%M%S%L")
|
28
28
|
|
29
29
|
@example_topic = "turbike-kafka-specs-#{timestamp}"
|
30
|
-
KafkaHelper.create_topic(@example_topic)
|
31
|
-
KafkaHelper.fill_topic(@example_topic, MESSAGE_COUNT)
|
30
|
+
Turbine::KafkaHelper.create_topic(@example_topic)
|
31
|
+
Turbine::KafkaHelper.fill_topic(@example_topic, MESSAGE_COUNT)
|
32
32
|
end
|
33
33
|
|
34
34
|
after :all do
|
35
|
-
KafkaHelper.delete_topic(@example_topic)
|
35
|
+
Turbine::KafkaHelper.delete_topic(@example_topic)
|
36
36
|
end
|
37
37
|
|
38
38
|
it "fetches batches of messages" do
|
@@ -6,11 +6,16 @@ RSpec.describe Turbine::Processor do
|
|
6
6
|
QUEUE_SIZE = 100
|
7
7
|
|
8
8
|
let(:example_batch_size) { 100 }
|
9
|
-
let(:example_elements) { (0...example_batch_size).to_a }
|
10
9
|
let(:example_partition) { 0 }
|
11
10
|
let(:example_batch_count) { 1000 }
|
12
11
|
let(:example_message_count) { example_batch_size * example_batch_count }
|
13
12
|
|
13
|
+
let(:example_elements) do
|
14
|
+
example_batch_size.times.map do |n|
|
15
|
+
double(:message, value: n)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
14
19
|
let(:example_batches) do
|
15
20
|
Array.new(example_batch_count).fill do
|
16
21
|
Turbine::Batch.new(example_elements, example_partition)
|
@@ -49,7 +54,7 @@ RSpec.describe Turbine::Processor do
|
|
49
54
|
|
50
55
|
it "tolerates processing errors gracefully" do
|
51
56
|
# Check the default handler is printing to STDERR
|
52
|
-
expect(STDERR).to receive(:puts).
|
57
|
+
expect(STDERR).to receive(:puts).at_most(example_message_count).times
|
53
58
|
|
54
59
|
example_processor.process(mock_consumer) do |_msg, _ex|
|
55
60
|
fail "uhoh!"
|
data/tasks/kafka.rake
CHANGED
@@ -1,55 +1 @@
|
|
1
|
-
require "
|
2
|
-
require "colorize"
|
3
|
-
require "socket"
|
4
|
-
require "timeout"
|
5
|
-
|
6
|
-
KAFKA_PORT = 9092
|
7
|
-
START_TIMEOUT = 5
|
8
|
-
|
9
|
-
namespace :kafka do
|
10
|
-
KAFKA_VERSION = "0.8.2.1"
|
11
|
-
KAFKA_TARBALL = "kafka_2.10-#{KAFKA_VERSION}.tgz"
|
12
|
-
|
13
|
-
task download: "tmp/#{KAFKA_TARBALL}"
|
14
|
-
directory "tmp"
|
15
|
-
|
16
|
-
file "tmp/#{KAFKA_TARBALL}" => "tmp" do
|
17
|
-
puts "#{'***'.blue} #{'Downloading Kafka'.light_white}"
|
18
|
-
url = "https://www.apache.org/dist/kafka/#{KAFKA_VERSION}/kafka_2.10-#{KAFKA_VERSION}.tgz"
|
19
|
-
sh "curl #{url} -o tmp/#{KAFKA_TARBALL}"
|
20
|
-
end
|
21
|
-
|
22
|
-
task install: :download do
|
23
|
-
puts "#{'***'.blue} #{'Unpacking Kafka'.light_white}"
|
24
|
-
|
25
|
-
rm_rf "kafka" if File.exist? "kafka"
|
26
|
-
sh "tar -zxf tmp/#{KAFKA_TARBALL}"
|
27
|
-
mv "kafka_2.10-#{KAFKA_VERSION}", "kafka"
|
28
|
-
end
|
29
|
-
|
30
|
-
task start: %w(kafka zookeeper:start) do
|
31
|
-
puts "#{'***'.blue} #{'Starting Kafka'.light_white}"
|
32
|
-
sh "cd kafka && bin/kafka-server-start.sh config/server.properties &"
|
33
|
-
|
34
|
-
Timeout.timeout(START_TIMEOUT) do
|
35
|
-
begin
|
36
|
-
socket = TCPSocket.open("localhost", 9092)
|
37
|
-
rescue Errno::ECONNREFUSED
|
38
|
-
sleep 0.01
|
39
|
-
retry
|
40
|
-
end
|
41
|
-
|
42
|
-
socket.close
|
43
|
-
end
|
44
|
-
|
45
|
-
# Give Kafka some time to finish printing startup messages
|
46
|
-
sleep 0.5
|
47
|
-
puts "#{'***'.blue} #{'Kafka started!'.light_white}"
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
file "kafka" do
|
52
|
-
Rake::Task["kafka:install"].invoke
|
53
|
-
end
|
54
|
-
|
55
|
-
CLEAN.include "tmp", "kafka"
|
1
|
+
require "turbine/rake_tasks"
|
data/turbine.gemspec
CHANGED
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
10
10
|
spec.email = ["bascule@gmail.com"]
|
11
11
|
|
12
12
|
spec.summary = "Fault-tolerant multithreaded stream processing for Ruby"
|
13
|
-
spec.description = "
|
13
|
+
spec.description = "Performance-oriented stream processing built on Kafka and Zookeeper"
|
14
14
|
spec.homepage = "https://github.com/tarcieri/turbine"
|
15
15
|
spec.license = "MIT"
|
16
16
|
|
metadata
CHANGED
@@ -1,124 +1,124 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: turbine
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.pre2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tony Arcieri
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-05-
|
11
|
+
date: 2015-05-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: zk
|
15
|
-
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
|
-
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirement: !ruby/object:Gem::Requirement
|
23
21
|
requirements:
|
24
|
-
- -
|
22
|
+
- - '>='
|
25
23
|
- !ruby/object:Gem::Version
|
26
24
|
version: '0'
|
25
|
+
prerelease: false
|
26
|
+
type: :runtime
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: poseidon
|
29
|
-
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - '>='
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: 0.0.5
|
34
|
-
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
37
35
|
requirements:
|
38
|
-
- -
|
36
|
+
- - '>='
|
39
37
|
- !ruby/object:Gem::Version
|
40
38
|
version: 0.0.5
|
39
|
+
prerelease: false
|
40
|
+
type: :runtime
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: poseidon_cluster
|
43
|
-
|
43
|
+
version_requirements: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - '>='
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: 0.3.0
|
48
|
-
|
49
|
-
prerelease: false
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
51
49
|
requirements:
|
52
|
-
- -
|
50
|
+
- - '>='
|
53
51
|
- !ruby/object:Gem::Version
|
54
52
|
version: 0.3.0
|
53
|
+
prerelease: false
|
54
|
+
type: :runtime
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: concurrent-ruby
|
57
|
-
|
57
|
+
version_requirements: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - '>='
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
|
-
|
63
|
-
prerelease: false
|
64
|
-
version_requirements: !ruby/object:Gem::Requirement
|
62
|
+
requirement: !ruby/object:Gem::Requirement
|
65
63
|
requirements:
|
66
|
-
- -
|
64
|
+
- - '>='
|
67
65
|
- !ruby/object:Gem::Version
|
68
66
|
version: '0'
|
67
|
+
prerelease: false
|
68
|
+
type: :runtime
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: bundler
|
71
|
-
|
71
|
+
version_requirements: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ~>
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '1.9'
|
76
|
-
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
76
|
+
requirement: !ruby/object:Gem::Requirement
|
79
77
|
requirements:
|
80
|
-
- -
|
78
|
+
- - ~>
|
81
79
|
- !ruby/object:Gem::Version
|
82
80
|
version: '1.9'
|
81
|
+
prerelease: false
|
82
|
+
type: :development
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: rake
|
85
|
-
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- -
|
87
|
+
- - ~>
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: '10.0'
|
90
|
-
|
91
|
-
prerelease: false
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
90
|
+
requirement: !ruby/object:Gem::Requirement
|
93
91
|
requirements:
|
94
|
-
- -
|
92
|
+
- - ~>
|
95
93
|
- !ruby/object:Gem::Version
|
96
94
|
version: '10.0'
|
95
|
+
prerelease: false
|
96
|
+
type: :development
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: rspec
|
99
|
-
|
99
|
+
version_requirements: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- -
|
101
|
+
- - ~>
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '3.2'
|
104
|
-
|
105
|
-
prerelease: false
|
106
|
-
version_requirements: !ruby/object:Gem::Requirement
|
104
|
+
requirement: !ruby/object:Gem::Requirement
|
107
105
|
requirements:
|
108
|
-
- -
|
106
|
+
- - ~>
|
109
107
|
- !ruby/object:Gem::Version
|
110
108
|
version: '3.2'
|
111
|
-
|
109
|
+
prerelease: false
|
110
|
+
type: :development
|
111
|
+
description: Performance-oriented stream processing built on Kafka and Zookeeper
|
112
112
|
email:
|
113
113
|
- bascule@gmail.com
|
114
114
|
executables: []
|
115
115
|
extensions: []
|
116
116
|
extra_rdoc_files: []
|
117
117
|
files:
|
118
|
-
-
|
119
|
-
-
|
120
|
-
-
|
121
|
-
-
|
118
|
+
- .gitignore
|
119
|
+
- .rspec
|
120
|
+
- .rubocop.yml
|
121
|
+
- .travis.yml
|
122
122
|
- CODE_OF_CONDUCT.md
|
123
123
|
- Gemfile
|
124
124
|
- Guardfile
|
@@ -131,8 +131,9 @@ files:
|
|
131
131
|
- lib/turbine/batch.rb
|
132
132
|
- lib/turbine/consumer.rb
|
133
133
|
- lib/turbine/consumer/kafka.rb
|
134
|
+
- lib/turbine/kafka_helper.rb
|
134
135
|
- lib/turbine/processor.rb
|
135
|
-
- lib/turbine/
|
136
|
+
- lib/turbine/rake_tasks.rb
|
136
137
|
- lib/turbine/version.rb
|
137
138
|
- spec/spec_helper.rb
|
138
139
|
- spec/turbine/batch_spec.rb
|
@@ -149,24 +150,24 @@ homepage: https://github.com/tarcieri/turbine
|
|
149
150
|
licenses:
|
150
151
|
- MIT
|
151
152
|
metadata: {}
|
152
|
-
post_install_message:
|
153
|
+
post_install_message:
|
153
154
|
rdoc_options: []
|
154
155
|
require_paths:
|
155
156
|
- lib
|
156
157
|
required_ruby_version: !ruby/object:Gem::Requirement
|
157
158
|
requirements:
|
158
|
-
- -
|
159
|
+
- - '>='
|
159
160
|
- !ruby/object:Gem::Version
|
160
161
|
version: '0'
|
161
162
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
162
163
|
requirements:
|
163
|
-
- -
|
164
|
+
- - '>'
|
164
165
|
- !ruby/object:Gem::Version
|
165
166
|
version: 1.3.1
|
166
167
|
requirements: []
|
167
|
-
rubyforge_project:
|
168
|
-
rubygems_version: 2.4.
|
169
|
-
signing_key:
|
168
|
+
rubyforge_project:
|
169
|
+
rubygems_version: 2.4.5
|
170
|
+
signing_key:
|
170
171
|
specification_version: 4
|
171
172
|
summary: Fault-tolerant multithreaded stream processing for Ruby
|
172
173
|
test_files: []
|
@@ -1,85 +0,0 @@
|
|
1
|
-
require "open3"
|
2
|
-
require "poseidon"
|
3
|
-
|
4
|
-
# Helper functions for integration testing with Kafka
|
5
|
-
module KafkaHelper
|
6
|
-
extend self
|
7
|
-
|
8
|
-
ZOOKEEPER_ADDR = "localhost:2181"
|
9
|
-
KAFKA_ADDR = "localhost:9092"
|
10
|
-
|
11
|
-
def delete_topic(topic)
|
12
|
-
log "*** Deleting Kafka topic: #{topic}"
|
13
|
-
|
14
|
-
topic_command :delete, topic: topic
|
15
|
-
end
|
16
|
-
|
17
|
-
def create_topic(topic)
|
18
|
-
log "*** Creating Kafka topic: #{topic}"
|
19
|
-
|
20
|
-
required_topic_command :create,
|
21
|
-
"replication-factor" => 1,
|
22
|
-
"partitions" => 1,
|
23
|
-
"topic" => topic
|
24
|
-
end
|
25
|
-
|
26
|
-
def list_topics
|
27
|
-
topic_command(:list).split("\n")
|
28
|
-
end
|
29
|
-
|
30
|
-
def topic_exists?(topic)
|
31
|
-
list_topics.include?(topic)
|
32
|
-
end
|
33
|
-
|
34
|
-
def fill_topic(topic, n = 100_000)
|
35
|
-
fail ArgumentError, "min messages is 1000" if n < 1000
|
36
|
-
|
37
|
-
producer = Poseidon::Producer.new([KAFKA_ADDR], "my_test_producer", type: :sync)
|
38
|
-
|
39
|
-
log "*** Filling topic with #{n} messages: #{topic}"
|
40
|
-
|
41
|
-
(n / 1000).times do |i|
|
42
|
-
messages = []
|
43
|
-
|
44
|
-
1000.times do |j|
|
45
|
-
n = (i * 1000 + j)
|
46
|
-
messages << Poseidon::MessageToSend.new(topic, n.to_s)
|
47
|
-
end
|
48
|
-
|
49
|
-
producer.send_messages(messages)
|
50
|
-
end
|
51
|
-
ensure
|
52
|
-
producer.close if producer
|
53
|
-
end
|
54
|
-
|
55
|
-
private
|
56
|
-
|
57
|
-
def kafka_path
|
58
|
-
File.expand_path("../../../../kafka", __FILE__)
|
59
|
-
end
|
60
|
-
|
61
|
-
def kafka_topics_bin_path
|
62
|
-
"#{kafka_path}/bin/kafka-topics.sh"
|
63
|
-
end
|
64
|
-
|
65
|
-
def kafka_args(args = {})
|
66
|
-
{ zookeeper: ZOOKEEPER_ADDR }.merge(args).map { |k, v| "--#{k} #{v}" }.join(" ")
|
67
|
-
end
|
68
|
-
|
69
|
-
def topic_command(command, args = {})
|
70
|
-
cmd = "#{kafka_topics_bin_path} --#{command} #{kafka_args(args)}"
|
71
|
-
stdout_str, _stderr_str, status = Open3.capture3(cmd)
|
72
|
-
return unless status.success?
|
73
|
-
stdout_str
|
74
|
-
end
|
75
|
-
|
76
|
-
def required_topic_command(command, args = {})
|
77
|
-
result = topic_command(command, args)
|
78
|
-
fail "Kafka command failed!" unless result
|
79
|
-
true
|
80
|
-
end
|
81
|
-
|
82
|
-
def log(message)
|
83
|
-
STDERR.puts(message)
|
84
|
-
end
|
85
|
-
end
|