turbine 1.0.0.pre

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 06c2b81075e5071fcc7ca1eaaab8c1e7edaaf787
+  data.tar.gz: 34a741dd15ec2e4a24d6e9b89a7710d222c4b758
+SHA512:
+  metadata.gz: 24e0625293f5306c2f22615e5d6d2822536427bda3c811130de5fff9f60faf05b2348f6554fdfe7db58c3b163bba428cc4e0e2383f854a471d0d61fe12075f98
+  data.tar.gz: cbec4a21aa14a47fd9878fbe48e12d60aa20a910828532a37f70b0ee69bf9fc565d2fe8d2ab56827dcb6746812d6f10e9b33eb6143646aad9b9c08e6093251cf

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/zookeeper/
+/kafka/

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format documentation
+--order random

data/.rubocop.yml

@@ -0,0 +1,15 @@
+LineLength:
+  Max: 100
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/For:
+  Enabled: false # for: For speed!
+
+Style/ModuleFunction:
+  Enabled: false
+
+Metrics/MethodLength:
+  CountComments: false
+  Max: 15

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+install: gem install bundler
+script: "bundle && bundle exec rake ci"
+rvm:
+  - 2.2.2
+  - jruby
+  - jruby-head
+  - rbx-2
+
+matrix:
+  allow_failures:
+    - rvn: jruby-head
+    - rvm: rbx-2

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http:contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile

@@ -0,0 +1,15 @@
+source "https://rubygems.org"
+
+gem "colorize"
+gem "poseidon_cluster", github: "bsm/poseidon_cluster"
+
+group :development, :test do
+  gem "guard-rspec"
+  gem "rubocop"
+end
+
+group :test do
+  gem "coveralls", require: false
+end
+
+gemspec

data/Guardfile

@@ -0,0 +1,8 @@
+directories %w(lib spec)
+clearing :on
+
+guard :rspec, cmd: "bundle exec rspec" do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch("spec/spec_helper.rb")  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Tony Arcieri
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,109 @@
+![Turbine](https://raw.githubusercontent.com/tarcieri/turbine/master/turbine.png)
+=======
+[![Build Status](https://travis-ci.org/tarcieri/turbine.svg)](https://travis-ci.org/tarcieri/turbine)
+[![Code Climate](https://codeclimate.com/github/tarcieri/turbine/badges/gpa.svg)](https://codeclimate.com/github/tarcieri/turbine)
+[![Coverage Status](https://coveralls.io/repos/tarcieri/turbine/badge.svg)](https://coveralls.io/r/tarcieri/turbine)
+
+Fault-tolerant multithreaded stream processing for Ruby.
+
+Turbine is a perforance-oriented stream processing library built on Zookeeper.
+It presently supports Kafka as a message queue, but is designed to be pluggable
+in order to potentially support other message queues in the future.
+
+Turbine is not a job queue and is missing most of the features you'd expect
+from a job queue by design. Turbine is designed to be small, simple, and fast.
+
+## Installation
+
+Add these lines to your application's Gemfile:
+
+**NOTE:** Turbine relies on an unreleased version (0.3.0) of poseidon_cluster,
+so you will also need to use the version off of GitHub for now.
+
+```ruby
+gem "turbine"
+gem "poseidon_cluster", github: "bsm/poseidon_cluster"
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install turbine
+
+## Usage
+
+Turbine presently supports stream processing from the Kafka message queue
+using the [poseidon_cluster](https://github.com/bsm/poseidon_cluster) gem,
+which implements self-rebalancing Consumer Groups.
+
+To create a new Kafka consumer for a topic, do the following:
+
+```ruby
+require "turbine"
+require "turbine/consumer/kafka"
+
+consumer = Turbine::Consumer::Kafka.new(
+  "my-group",                               # Group name
+  ["kafka1.host:9092", "kafka2.host:9092"], # Kafka brokers
+  ["zk1.host:2181",    "zk2.host:2181"],    # Zookeeper hosts
+  "my-topic"                                # Topic name
+)
+
+processor = Turbine::Processor.new(min_threads: 5, max_threads: 5, max_queue: 1000)
+
+processor.process(consumer) do |msg|
+   ...
+end
+```
+
+## Error handling
+
+By default, Turbine prints exceptions that occur during message processing to STDERR.
+Chances are, you'd probably rather log them to an exception logging service.
+
+After creating a processor object, you can configure a custom exception handler so
+you can log these exceptions to a more appropriate place than STDERR:
+
+```ruby
+processor = Turbine::Processor.new(min_threads: 5, max_threads: 5, max_queue: 1000)
+
+processor.error_handler do |ex, _msg|
+  MyBugHandler.notify_or_ignore(ex)
+end
+
+processor.process(consumer) do |msg|
+   ...
+end
+```
+
+## Semantics (PLEASE READ)
+
+Turbine automatically reschedules processing of messages in the stream in the event of faults or rebalancing of resources. Because of this, the same message may be received multiple times. Stream processing jobs written in Turbine MUST account for this.
+
+An example of where things could go wrong is a "counter" job. Imagine we look for a particular event and increment a counter in statsd/memcached/redis. This will not give accurate numbers, because message replays will spuriously increment the counter.
+
+The contract of Turbine is as follows:
+
+* Turbine messages are guaranteed to be delivered AT LEAST once but Turbine MAY replay the same message many times
+* Because of this, stream processing jobs written in Turbine MUST be idempotent (i.e. repeat processing of the same message is gracefully tolerated)
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+* Fork this repository on github
+* Make your changes and send us a pull request
+* If we like them we'll merge them
+* If we've accepted a patch, feel free to ask for commit access
+
+## License
+
+Copyright (c) 2015 Tony Arcieri. Distributed under the MIT License. See
+LICENSE.txt for further details.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+
+Dir["tasks/**/*.rake"].each { |task| load task }
+
+task default: %w(spec rubocop)
+task ci:      %w(kafka:start spec rubocop)

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "turbine"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+rake zookeeper:install kafka:install

data/lib/turbine/batch.rb

@@ -0,0 +1,28 @@
+module Turbine
+  # Batches of messages to be processed
+  class Batch
+    attr_reader :messages, :partition
+
+    def initialize(messages, partition)
+      @messages  = messages.freeze
+      @partition = partition
+      @completed = Concurrent::AtomicBoolean.new
+    end
+
+    def complete
+      @completed.value = true
+    end
+
+    def completed?
+      @completed.value
+    end
+
+    def [](n)
+      @messages.at(n)
+    end
+
+    def size
+      @messages.size
+    end
+  end
+end

data/lib/turbine/consumer/kafka.rb

@@ -0,0 +1,31 @@
+require "poseidon_cluster"
+
+module Turbine
+  module Consumer
+    # Turbine consumer for the Kafka message queue
+    class Kafka
+      def initialize(*args)
+        @consumer = Poseidon::ConsumerGroup.new(*args)
+      end
+
+      def fetch
+        batch = nil
+
+        @consumer.fetch commit: false do |partition, messages|
+          batch = Batch.new(messages, partition)
+        end
+
+        batch
+      end
+
+      def commit(batch)
+        return if batch.messages.empty?
+        @consumer.commit batch.partition, batch.messages.last.offset + 1
+      end
+
+      def close
+        @consumer.close
+      end
+    end
+  end
+end

data/lib/turbine/consumer.rb

@@ -0,0 +1,5 @@
+module Turbine
+  # Module containing all Turbine consumers
+  module Consumer
+  end
+end

data/lib/turbine/processor.rb

@@ -0,0 +1,102 @@
+module Turbine
+  # Multithreaded message processor
+  class Processor
+    # How long to sleep when busy waiting for the queue to empty
+    BUSY_WAIT_INTERVAL = 0.0001
+
+    def initialize(*args)
+      @running = Concurrent::AtomicBoolean.new
+      @pool = Concurrent::ThreadPoolExecutor.new(*args)
+      @completed_count = Concurrent::AtomicFixnum.new
+      @pending = []
+      @error_handler = proc do |ex|
+        STDERR.puts "*** Error processing message: #{ex.class} #{ex}\n#{ex.backtrace.join("\n")}"
+      end
+    end
+
+    def process(consumer, &block)
+      fail ArgumentError, "no block given" unless block
+      processor_method = method(:process_batch)
+
+      @running.value = true
+      while @running.value && (batch = consumer.fetch)
+        enqueue_batch(batch)
+
+        begin
+          @pool.post(batch, block, &processor_method)
+        rescue Concurrent::RejectedExecutionError
+          busy_wait(consumer)
+          retry
+        end
+
+        commit_completions(consumer)
+      end
+    end
+
+    def stop
+      @running.value = false
+    end
+
+    def drain(timeout = nil)
+      stop
+      @pool.shutdown
+      @pool.wait_for_termination(timeout)
+    end
+
+    def completed_count
+      @completed_count.value
+    end
+
+    def running?
+      @running.value
+    end
+
+    def error_handler(&block)
+      @error_handler = block
+    end
+
+    private
+
+    def enqueue_batch(batch)
+      partition = @pending[batch.partition] ||= []
+      partition << batch
+    end
+
+    def commit_completions(consumer)
+      for partition in @pending
+        next unless partition
+
+        last_completed_batch = nil
+        while (batch = partition.first) && batch.completed?
+          last_completed_batch = partition.shift
+        end
+
+        consumer.commit(last_completed_batch) if last_completed_batch
+      end
+    end
+
+    def process_batch(batch, block)
+      for index in (0...batch.size)
+        msg = batch[index]
+
+        begin
+          block.call(msg)
+        rescue => ex
+          @error_handler.call(ex, msg)
+        end
+
+        @completed_count.increment
+      end
+
+      batch.complete
+    end
+
+    def busy_wait(consumer)
+      commit_completions(consumer)
+
+      # We exceeded the pool's queue, so busy-wait and retry
+      # TODO: more intelligent busy-waiting strategy
+      sleep BUSY_WAIT_INTERVAL
+    end
+  end
+end

data/lib/turbine/rspec/kafka_helper.rb

@@ -0,0 +1,85 @@
+require "open3"
+require "poseidon"
+
+# Helper functions for integration testing with Kafka
+module KafkaHelper
+  extend self
+
+  ZOOKEEPER_ADDR = "localhost:2181"
+  KAFKA_ADDR     = "localhost:9092"
+
+  def delete_topic(topic)
+    log "*** Deleting Kafka topic: #{topic}"
+
+    topic_command :delete, topic: topic
+  end
+
+  def create_topic(topic)
+    log "*** Creating Kafka topic: #{topic}"
+
+    required_topic_command :create,
+                           "replication-factor" => 1,
+                           "partitions"         => 1,
+                           "topic"              => topic
+  end
+
+  def list_topics
+    topic_command(:list).split("\n")
+  end
+
+  def topic_exists?(topic)
+    list_topics.include?(topic)
+  end
+
+  def fill_topic(topic, n = 100_000)
+    fail ArgumentError, "min messages is 1000" if n < 1000
+
+    producer = Poseidon::Producer.new([KAFKA_ADDR], "my_test_producer", type: :sync)
+
+    log "*** Filling topic with #{n} messages: #{topic}"
+
+    (n / 1000).times do |i|
+      messages = []
+
+      1000.times do |j|
+        n = (i * 1000 + j)
+        messages << Poseidon::MessageToSend.new(topic, n.to_s)
+      end
+
+      producer.send_messages(messages)
+    end
+  ensure
+    producer.close if producer
+  end
+
+  private
+
+  def kafka_path
+    File.expand_path("../../../../kafka", __FILE__)
+  end
+
+  def kafka_topics_bin_path
+    "#{kafka_path}/bin/kafka-topics.sh"
+  end
+
+  def kafka_args(args = {})
+    { zookeeper: ZOOKEEPER_ADDR }.merge(args).map { |k, v| "--#{k} #{v}" }.join(" ")
+  end
+
+  def topic_command(command, args = {})
+    cmd = "#{kafka_topics_bin_path} --#{command} #{kafka_args(args)}"
+    stdout_str, _stderr_str, status = Open3.capture3(cmd)
+    return unless status.success?
+    stdout_str
+  end
+
+  def required_topic_command(command, args = {})
+    result = topic_command(command, args)
+    fail "Kafka command failed!" unless result
+    true
+  end
+
+  def log(message)
+    STDERR.puts(message)
+  end
+end

data/lib/turbine/version.rb

@@ -0,0 +1,4 @@
+# Fault-tolerant multithreaded stream processing for Ruby
+module Turbine
+  VERSION = "1.0.0.pre"
+end

data/lib/turbine.rb

@@ -0,0 +1,11 @@
+require "turbine/version"
+
+require "concurrent"
+
+require "turbine/batch"
+require "turbine/consumer"
+require "turbine/processor"
+
+# Fault-tolerant multithreaded stream processing for Ruby
+module Turbine
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require "coveralls"
+Coveralls.wear!
+
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+require "turbine"
+
+RSpec.configure(&:disable_monkey_patching!)

data/spec/turbine/batch_spec.rb

@@ -0,0 +1,35 @@
+require "spec_helper"
+
+RSpec.describe Turbine::Batch do
+  let(:example_batch_size) { 13 }
+  let(:example_elements)   { (0...13).to_a }
+  let(:example_partition)  { 0 }
+  let(:example_batch)      { described_class.new(example_elements, example_partition) }
+
+  it "creates batches from arrays" do
+    example_elements.size.times do |n|
+      expect(example_batch[n]).to eq example_elements[n]
+    end
+  end
+
+  it "knows its size" do
+    expect(example_batch.size).to eq example_batch_size
+  end
+
+  it "knows its partition" do
+    expect(example_batch.partition).to eq example_partition
+  end
+
+  it "begins incomplete" do
+    expect(example_batch).not_to be_completed
+  end
+
+  it "can be completed" do
+    example_batch.complete
+    expect(example_batch).to be_completed
+  end
+
+  it "inspects" do
+    expect(example_batch.inspect).to include described_class.to_s
+  end
+end

data/spec/turbine/consumer/kafka_spec.rb

@@ -0,0 +1,54 @@
+require "spec_helper"
+require "turbine/consumer/kafka"
+require "turbine/rspec/kafka_helper"
+require "benchmark"
+
+RSpec.describe Turbine::Consumer::Kafka do
+  MESSAGE_COUNT = 100_000
+
+  let(:example_topic) { @example_topic }
+
+  def with_consumer
+    consumer = described_class.new(
+      "my-consumer-group",
+      ["localhost:9092"],
+      ["localhost:2181"],
+      example_topic
+    )
+
+    begin
+      yield consumer
+    ensure
+      consumer.close
+    end
+  end
+
+  before :all do
+    timestamp = Time.now.strftime("%Y%m%d%H%M%S%L")
+
+    @example_topic = "turbike-kafka-specs-#{timestamp}"
+    KafkaHelper.create_topic(@example_topic)
+    KafkaHelper.fill_topic(@example_topic, MESSAGE_COUNT)
+  end
+
+  after :all do
+    KafkaHelper.delete_topic(@example_topic)
+  end
+
+  it "fetches batches of messages" do
+    count = 0
+    with_consumer do |consumer|
+      rt = Benchmark.realtime do
+        while count < MESSAGE_COUNT
+          messages = consumer.fetch
+          count += messages.size
+        end
+      end
+
+      rate = "#{format('%.2f', count / rt)} msg/sec)"
+      STDERR.puts("*** Performance: #{count} messages in #{format('%.2f', rt)} seconds (#{rate})")
+    end
+
+    expect(count).to eq MESSAGE_COUNT
+  end
+end

data/spec/turbine/processor_spec.rb

@@ -0,0 +1,97 @@
+require "spec_helper"
+
+RSpec.describe Turbine::Processor do
+  MIN_THREAD_COUNT = 2
+  MAX_THREAD_COUNT = 16
+  QUEUE_SIZE       = 100
+
+  let(:example_batch_size)    { 100 }
+  let(:example_elements)      { (0...example_batch_size).to_a }
+  let(:example_partition)     { 0 }
+  let(:example_batch_count)   { 1000 }
+  let(:example_message_count) { example_batch_size * example_batch_count }
+
+  let(:example_batches) do
+    Array.new(example_batch_count).fill do
+      Turbine::Batch.new(example_elements, example_partition)
+    end
+  end
+
+  let(:mock_consumer) do
+    double(:consumer).tap do |consumer|
+      allow(consumer).to receive(:fetch).and_return(*example_batches, nil)
+      allow(consumer).to receive(:commit)
+    end
+  end
+
+  let(:example_processor) do
+    described_class.new(
+      min_threads: MAX_THREAD_COUNT,
+      max_threads: MAX_THREAD_COUNT,
+      max_queue:   QUEUE_SIZE
+    )
+  end
+
+  it "supports stopping the event loop" do
+    example_processor.stop
+    expect(example_processor.running?).to eq false
+  end
+
+  it "counts the number of messages processed" do
+    example_processor.process(mock_consumer) do |_msg|
+      # noop!
+    end
+
+    example_processor.drain
+
+    expect(example_processor.completed_count).to eq example_message_count
+  end
+
+  it "tolerates processing errors gracefully" do
+    # Check the default handler is printing to STDERR
+    expect(STDERR).to receive(:puts).exactly(example_message_count).times
+
+    example_processor.process(mock_consumer) do |_msg, _ex|
+      fail "uhoh!"
+    end
+
+    example_processor.drain
+  end
+
+  it "supports a custom error handler" do
+    handler_called = false
+
+    example_processor.error_handler do |_ex, _msg|
+      handler_called = true
+    end
+
+    example_processor.process(mock_consumer) do |_ex, _msg|
+      fail "uhoh!"
+    end
+
+    example_processor.drain
+    expect(handler_called).to eq true
+  end
+
+  context "message processing" do
+    MIN_THREAD_COUNT.upto(MAX_THREAD_COUNT) do |thread_count|
+      it "processes batches of messages with #{thread_count} threads" do
+        processor = described_class.new(
+          min_threads: thread_count,
+          max_threads: thread_count,
+          max_queue:   QUEUE_SIZE
+        )
+
+        processor.process(mock_consumer) do |_msg|
+          # noop!
+        end
+
+        processor.drain
+
+        example_batches.each do |example_batch|
+          expect(example_batch).to be_completed
+        end
+      end
+    end
+  end
+end

data/spec/turbine_spec.rb

@@ -0,0 +1,7 @@
+require "spec_helper"
+
+RSpec.describe Turbine do
+  it "has a version number" do
+    expect(Turbine::VERSION).not_to be nil
+  end
+end

data/tasks/kafka.rake

@@ -0,0 +1,55 @@
+require "rake/clean"
+require "colorize"
+require "socket"
+require "timeout"
+
+KAFKA_PORT    = 9092
+START_TIMEOUT = 5
+
+namespace :kafka do
+  KAFKA_VERSION = "0.8.2.1"
+  KAFKA_TARBALL = "kafka_2.10-#{KAFKA_VERSION}.tgz"
+
+  task download: "tmp/#{KAFKA_TARBALL}"
+  directory "tmp"
+
+  file "tmp/#{KAFKA_TARBALL}" => "tmp" do
+    puts "#{'***'.blue} #{'Downloading Kafka'.light_white}"
+    url = "https://www.apache.org/dist/kafka/#{KAFKA_VERSION}/kafka_2.10-#{KAFKA_VERSION}.tgz"
+    sh "curl #{url} -o tmp/#{KAFKA_TARBALL}"
+  end
+
+  task install: :download do
+    puts "#{'***'.blue} #{'Unpacking Kafka'.light_white}"
+
+    rm_rf "kafka" if File.exist? "kafka"
+    sh "tar -zxf tmp/#{KAFKA_TARBALL}"
+    mv "kafka_2.10-#{KAFKA_VERSION}", "kafka"
+  end
+
+  task start: %w(kafka zookeeper:start) do
+    puts "#{'***'.blue} #{'Starting Kafka'.light_white}"
+    sh "cd kafka && bin/kafka-server-start.sh config/server.properties &"
+
+    Timeout.timeout(START_TIMEOUT) do
+      begin
+        socket = TCPSocket.open("localhost", 9092)
+      rescue Errno::ECONNREFUSED
+        sleep 0.01
+        retry
+      end
+
+      socket.close
+    end
+
+    # Give Kafka some time to finish printing startup messages
+    sleep 0.5
+    puts "#{'***'.blue} #{'Kafka started!'.light_white}"
+  end
+end
+
+file "kafka" do
+  Rake::Task["kafka:install"].invoke
+end
+
+CLEAN.include "tmp", "kafka"

data/tasks/rspec.rake

@@ -0,0 +1,3 @@
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new

data/tasks/rubocop.rake

@@ -0,0 +1,2 @@
+require "rubocop/rake_task"
+RuboCop::RakeTask.new

data/tasks/zookeeper.rake

@@ -0,0 +1,63 @@
+require "rake/clean"
+require "colorize"
+
+def zookeeper_config(data)
+  <<-CONFIG
+# Zookeeper configuration
+
+# The number of milliseconds of each tick
+tickTime=2000
+# The number of ticks that the initial
+# synchronization phase can take
+initLimit=10
+# The number of ticks that can pass between
+# sending a request and getting an acknowledgement
+syncLimit=5
+# the directory where the snapshot is stored.
+dataDir=#{data}
+# the port at which the clients will connect
+clientPort=2181
+CONFIG
+end
+
+namespace :zookeeper do
+  ZK_VERSION = "3.4.6"
+  ZK_TARBALL = "zookeeper-#{ZK_VERSION}.tar.gz"
+
+  task download: "tmp/#{ZK_TARBALL}"
+  directory "tmp"
+
+  file "tmp/#{ZK_TARBALL}" => "tmp" do
+    puts "#{'***'.blue} #{'Downloading Zookeeper'.light_white}"
+    url = "https://archive.apache.org/dist/zookeeper/zookeeper-#{ZK_VERSION}/#{ZK_TARBALL}"
+    sh "curl #{url} -o tmp/#{ZK_TARBALL}"
+  end
+
+  task install: :download do
+    puts "#{'***'.blue} #{'Unpacking Zookeeper'.light_white}"
+
+    rm_rf "zookeeper" if File.exist? "zookeeper"
+    sh "tar -zxf tmp/#{ZK_TARBALL}"
+    mv "zookeeper-#{ZK_VERSION}", "zookeeper"
+    home = File.expand_path("../../zookeeper", __FILE__)
+
+    # Create base configuration
+    data = File.join(home, "data")
+    mkdir_p data
+    config = File.join(home, "conf", "zoo.cfg")
+    rm_r File.join(home, "conf", "zoo_sample.cfg")
+
+    File.open(config, "w") { |file| file << zookeeper_config(data) }
+  end
+
+  task start: :zookeeper do
+    puts "#{'***'.blue} #{'Starting Zookeeper'.light_white}"
+    sh "cd zookeeper && bin/zkServer.sh start"
+  end
+end
+
+file "zookeeper" do
+  Rake::Task["zookeeper:install"].invoke
+end
+
+CLEAN.include "tmp", "zookeeper"

data/turbine.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "turbine/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "turbine"
+  spec.version       = Turbine::VERSION
+  spec.authors       = ["Tony Arcieri"]
+  spec.email         = ["bascule@gmail.com"]
+
+  spec.summary       = "Fault-tolerant multithreaded stream processing for Ruby"
+  spec.description   = "Turbine is a performance-oriented stream processor built on Zookeeper"
+  spec.homepage      = "https://github.com/tarcieri/turbine"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "zk"
+  spec.add_runtime_dependency "poseidon", ">= 0.0.5"
+  spec.add_runtime_dependency "poseidon_cluster", ">= 0.3.0"
+  spec.add_runtime_dependency "concurrent-ruby"
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.2"
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: turbine
+version: !ruby/object:Gem::Version
+  version: 1.0.0.pre
+platform: ruby
+authors:
+- Tony Arcieri
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-05-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: zk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: poseidon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+- !ruby/object:Gem::Dependency
+  name: poseidon_cluster
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+description: Turbine is a performance-oriented stream processor built on Zookeeper
+email:
+- bascule@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/turbine.rb
+- lib/turbine/batch.rb
+- lib/turbine/consumer.rb
+- lib/turbine/consumer/kafka.rb
+- lib/turbine/processor.rb
+- lib/turbine/rspec/kafka_helper.rb
+- lib/turbine/version.rb
+- spec/spec_helper.rb
+- spec/turbine/batch_spec.rb
+- spec/turbine/consumer/kafka_spec.rb
+- spec/turbine/processor_spec.rb
+- spec/turbine_spec.rb
+- tasks/kafka.rake
+- tasks/rspec.rake
+- tasks/rubocop.rake
+- tasks/zookeeper.rake
+- turbine.gemspec
+- turbine.png
+homepage: https://github.com/tarcieri/turbine
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Fault-tolerant multithreaded stream processing for Ruby
+test_files: []