fluq-kafka 0.8.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 14bf6c533a3bc8171681579e6b2bd9d5c578424c
+  data.tar.gz: ddaa619f61a3d08a690ebbf987d8d3edadfad014
+SHA512:
+  metadata.gz: a8afc4eebc8ab90412d3eb6a16c93db5b8d4d91198eb43dd15d3084da622bf3398cc1aa7c17a1dae6241d2c998cc10f6c7705de622c6bbbb75dd072ccfa37adc
+  data.tar.gz: b5763d4b2b25ff2cbf5aa8afa657938d8f2114208ee14395b214bc9b97175adab136ea718c4b797186c46bdb41f5beb4c5d986c21e6019ab9b24b2036a6b10ec

data/.bundle/install.log

@@ -0,0 +1,23 @@
+# Logfile created on 2014-01-18 14:14:05 +0000 by logger.rb/44203
+I, [2014-01-18T14:14:05.461408 #26543]  INFO -- : 0:  rake (10.1.1) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/rake-10.1.1.gemspec
+I, [2014-01-18T14:14:05.461973 #26543]  INFO -- : 0:  bundler (1.5.0) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/bundler-1.5.0.gemspec
+I, [2014-01-18T14:14:05.462372 #26543]  INFO -- : 0:  timers (1.1.0) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/timers-1.1.0.gemspec
+I, [2014-01-18T14:14:05.462778 #26543]  INFO -- : 0:  celluloid (0.15.2) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/celluloid-0.15.2.gemspec
+I, [2014-01-18T14:14:05.463185 #26543]  INFO -- : 0:  nio4r (1.0.0) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/nio4r-1.0.0.gemspec
+I, [2014-01-18T14:14:05.463590 #26543]  INFO -- : 0:  celluloid-io (0.15.0) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/celluloid-io-0.15.0.gemspec
+I, [2014-01-18T14:14:05.464023 #26543]  INFO -- : 0:  diff-lcs (1.2.5) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/diff-lcs-1.2.5.gemspec
+I, [2014-01-18T14:14:05.464449 #26543]  INFO -- : 0:  multi_json (1.8.4) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/multi_json-1.8.4.gemspec
+I, [2014-01-18T14:14:05.464867 #26543]  INFO -- : 0:  timed_lru (0.3.1) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/timed_lru-0.3.1.gemspec
+I, [2014-01-18T14:14:05.540863 #26543]  INFO -- : 0:  fluq (0.8.0) from /home/dim/.rvm/gems/ruby-2.1.0/bundler/gems/fluq-87e8d5d493b0/fluq.gemspec
+I, [2014-01-18T14:14:06.855386 #26543]  INFO -- : 0:  poseidon (0.0.4) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/poseidon-0.0.4.gemspec
+I, [2014-01-18T14:14:06.856594 #26543]  INFO -- : 0:  little-plugger (1.1.3) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/little-plugger-1.1.3.gemspec
+I, [2014-01-18T14:14:06.857551 #26543]  INFO -- : 0:  logging (1.7.2) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/logging-1.7.2.gemspec
+I, [2014-01-18T14:14:06.858502 #26543]  INFO -- : 0:  zookeeper (1.4.8) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/zookeeper-1.4.8.gemspec
+I, [2014-01-18T14:14:06.859447 #26543]  INFO -- : 0:  zk (1.9.3) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/zk-1.9.3.gemspec
+I, [2014-01-18T14:14:08.458286 #26543]  INFO -- : 0:  poseidon_cluster (0.0.2) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/poseidon_cluster-0.0.2.gemspec
+I, [2014-01-18T14:14:08.467851 #26543]  INFO -- : 0:  fluq-kafka (0.8.0) from /home/dim/Work/fluq-kafka/fluq-kafka.gemspec
+I, [2014-01-18T14:14:08.468749 #26543]  INFO -- : 0:  rspec-core (2.14.7) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/rspec-core-2.14.7.gemspec
+I, [2014-01-18T14:14:08.469579 #26543]  INFO -- : 0:  rspec-expectations (2.14.4) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/rspec-expectations-2.14.4.gemspec
+I, [2014-01-18T14:14:08.470275 #26543]  INFO -- : 0:  rspec-mocks (2.14.4) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/rspec-mocks-2.14.4.gemspec
+I, [2014-01-18T14:14:08.470843 #26543]  INFO -- : 0:  rspec (2.14.1) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/rspec-2.14.1.gemspec
+I, [2014-01-18T14:14:08.471430 #26543]  INFO -- : 0:  yard (0.8.7.3) from /home/dim/.rvm/gems/ruby-2.1.0/specifications/yard-0.8.7.3.gemspec

data/.gitignore

@@ -0,0 +1,4 @@
+log/
+tmp/
+*.gem
+coverage/

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - jruby-19mode
+gemfile:
+  - Gemfile
+script: bundle exec rake spec:coveralls

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+gemspec
+gem "poseidon", git: "https://github.com/dim/poseidon.git"

data/Gemfile.lock

@@ -0,0 +1,88 @@
+GIT
+  remote: https://github.com/dim/poseidon.git
+  revision: f853e8f07a6f2ff4a520d3f7e1dac030453a18ed
+  specs:
+    poseidon (0.0.4)
+
+PATH
+  remote: .
+  specs:
+    fluq-kafka (0.8.0)
+      fluq (~> 0.8.0)
+      poseidon_cluster
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    celluloid-io (0.15.0)
+      celluloid (>= 0.15.0)
+      nio4r (>= 0.5.0)
+    coveralls (0.7.0)
+      multi_json (~> 1.3)
+      rest-client
+      simplecov (>= 0.7)
+      term-ansicolor
+      thor
+    diff-lcs (1.2.5)
+    docile (1.1.2)
+    fluq (0.8.0)
+      celluloid-io (~> 0.15.0)
+      multi_json
+      timed_lru
+    little-plugger (1.1.3)
+    logging (1.7.2)
+      little-plugger (>= 1.1.3)
+    mime-types (2.1)
+    multi_json (1.8.4)
+    nio4r (1.0.0)
+    nio4r (1.0.0-java)
+    poseidon_cluster (0.0.5)
+      poseidon
+      zk
+    rake (10.1.1)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+    simplecov (0.8.2)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    slyphon-log4j (1.2.15)
+    slyphon-zookeeper_jar (3.3.5-java)
+    term-ansicolor (1.2.2)
+      tins (~> 0.8)
+    thor (0.18.1)
+    timed_lru (0.3.1)
+    timers (1.1.0)
+    tins (0.13.1)
+    yard (0.8.7.3)
+    zk (1.9.3)
+      logging (~> 1.7.2)
+      zookeeper (~> 1.4.0)
+    zookeeper (1.4.8)
+    zookeeper (1.4.8-java)
+      slyphon-log4j (= 1.2.15)
+      slyphon-zookeeper_jar (= 3.3.5)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  bundler
+  coveralls
+  fluq-kafka!
+  poseidon!
+  rake
+  rspec
+  yard

data/README.md

@@ -0,0 +1,28 @@
+# FluQ - Kafka [![Build Status](https://travis-ci.org/bsm/fluq-kafka.png?branch=master)](https://travis-ci.org/bsm/fluq-kafka) [![Coverage Status](https://coveralls.io/repos/bsm/fluq-kafka/badge.png)](https://coveralls.io/r/bsm/fluq-kafka)
+
+Kafka 0.8+ input for FluQ.
+
+## Licence
+
+```
+Copyright (c) 2014 Black Square Media
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+```

data/Rakefile

@@ -0,0 +1,18 @@
+require 'rake'
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+require 'yard'
+require 'yard/rake/yardoc_task'
+YARD::Rake::YardocTask.new
+
+require 'coveralls/rake/task'
+Coveralls::RakeTask.new
+namespace :spec do
+  task coveralls: [:spec, 'coveralls:push']
+end
+
+desc 'Default: run specs.'
+task :default => :spec

data/fluq-kafka.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+
+$:.push File.expand_path("../lib", __FILE__)
+require "fluq/kafka/version"
+
+Gem::Specification.new do |s|
+  s.required_ruby_version = '>= 1.9.1'
+  s.required_rubygems_version = ">= 1.8.0"
+
+  s.name        = File.basename(__FILE__, '.gemspec')
+  s.summary     = "FluQ Kafka"
+  s.description = "Kafka plugin for FluQ"
+  s.version     = FluQ::Kafka::VERSION.dup
+
+  s.authors     = ["Black Square Media"]
+  s.email       = "info@blacksquaremedia.com"
+  s.homepage    = "https://github.com/bsm/fluq-kafka"
+
+  s.require_path = 'lib'
+  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.files = `git ls-files`.split("\n")
+  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+
+  s.add_dependency "fluq", "~> #{s.version}"
+  s.add_dependency "poseidon_cluster"
+
+  s.add_development_dependency "rake"
+  s.add_development_dependency "bundler"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "yard"
+  s.add_development_dependency "coveralls"
+end

data/lib/fluq-kafka.rb

@@ -0,0 +1 @@
+require 'fluq/kafka'

data/lib/fluq/input/kafka.rb

@@ -0,0 +1,93 @@
+class FluQ::Input::Kafka < FluQ::Input::Base
+
+  # Constructor.
+  # @option options [String] :topic the name of the topic to consume
+  # @option options [String] :group
+  #   The unique consumer group name. Each topic message is consumed only once by members of the same group.
+  #   If you have multiple thread/process instances running, make sure you assign them the same group name to
+  #   ensure events are not duplicated.
+  # @option options [String] :brokers an array of "HOST:PORT" broker addresses, e.g. ["localhost:9092"]
+  # @option options [String] :zookeepers an array of "HOST:PORT" zookeeper addresses, e.g. ["localhost:2181"]
+  # @option options [Integer] :max_bytes maximum number of bytes to fetch
+  #   Default: 1048576 (1MB)
+  # @option options [Integer] :max_wait_ms how long to block until the server sends us data.
+  #   Default: 100 (100ms)
+  # @option options [Integer] :min_bytes smallest amount of data the server should send us.
+  #   Default: 0 (send us data as soon as it is ready)
+  #
+  # @raises [ArgumentError] when no topic provided
+  #
+  # @example
+  #
+  #   FluQ::Input::Kafka.new handlers,
+  #     topic: "page-views",
+  #     group: "fluq",
+  #     brokers: ["localhost:9092"],
+  #     zookeepers: ["localhost:2181"]
+  def initialize(*)
+    super
+  end
+
+  # @return [String] short name
+  def name
+    "kafka:#{config[:topic]}"
+  end
+
+  # @return [String] descriptive name
+  def description
+    "#{name} (#{config[:group]} <- #{config[:brokers].join(',')})"
+  end
+
+  # Start the loop
+  def run
+    super
+
+    consumer.fetch_loop do |partition, bulk|
+      process partition, bulk
+    end
+  end
+
+  # Processes messages
+  # @param [Integer] partition
+  # @param [Array<Poseidon::Message>] messages
+  def process(partition, messages)
+    events = []
+    messages.each do |m|
+      events.concat format.parse(m.value)
+    end
+    events.each do |event|
+      event.meta.update topic: config[:topic], partition: partition
+    end
+    worker.process events
+  end
+
+  protected
+
+    def consumer
+      @consumer ||= ::Poseidon::ConsumerGroup.new config[:group], config[:brokers], config[:zookeepers], config[:topic],
+        min_bytes: config[:min_bytes],
+        max_bytes: config[:max_bytes],
+        max_wait_ms: config[:max_wait_ms]
+    end
+
+    def configure
+      raise ArgumentError, 'No topic provided, make sure you pass a :topic option' unless config[:topic]
+    end
+
+    def defaults
+      super.merge \
+        group: "fluq",
+        min_bytes: 0,
+        max_bytes: (1024 * 1024),
+        max_wait_ms: 100,
+        brokers: ["localhost:9092"],
+        zookeepers: ["localhost:2181"]
+    end
+
+    def before_terminate
+      @consumer.close if @consumer
+    rescue ThreadError
+    end
+
+end
+

data/lib/fluq/kafka.rb

@@ -0,0 +1,8 @@
+require 'fluq'
+require 'fluq/kafka/version'
+require 'poseidon_cluster'
+require 'fluq/input/kafka'
+
+class Poseidon::Connection
+  TCPSocket = Celluloid::IO::TCPSocket
+end

data/lib/fluq/kafka/version.rb

@@ -0,0 +1,5 @@
+module FluQ
+  module Kafka
+    VERSION = "0.8.0"
+  end
+end

data/spec/fluq/input/kafka_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe FluQ::Input::Kafka do
+
+  let(:message) { Poseidon::Message.new value: %({"a":1,"b":2}) }
+  let(:actors)  { [] }
+
+  let :mock_consumer do
+    double Poseidon::ConsumerGroup, claimed: [], fetch_loop: nil, close: true
+  end
+
+  def input(opts = {})
+    actor = described_class.new "my-feed", [[FluQ::Handler::Test]], opts
+    actors << actor
+    actor
+  end
+
+  before  { Poseidon::ConsumerGroup.stub new: mock_consumer }
+  after   { actors.each &:terminate }
+  subject { input topic: "my-topic" }
+
+  it { should be_a(FluQ::Input::Base) }
+  its(:description) { should == "kafka:my-topic (fluq <- localhost:9092)" }
+  its(:name)        { should == "kafka:my-topic" }
+  its(:config)      { should == {format: "json", format_options: {}, group: "fluq", min_bytes: 0, max_bytes: 1048576, max_wait_ms: 100, brokers: ["localhost:9092"], zookeepers: ["localhost:2181"], topic: "my-topic"} }
+
+  it 'should require a topic option' do
+    -> { input }.should raise_error(ArgumentError, /No topic provided/)
+  end
+
+  it 'should fetch messages' do
+    mock_consumer.should_receive(:fetch_loop).and_yield(3, [message]*10)
+    subject.worker.should have(1).handlers
+    subject.worker.handlers.first.should have(10).events
+    event = subject.worker.handlers.first.events.first
+    event.should == {"a"=>1,"b"=>2}
+    event.meta.should == {topic: "my-topic", partition: 3}
+  end
+
+end

data/spec/fluq/kafka_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe FluQ::Kafka do
+
+  it 'should force celluloid sockets' do
+    sock = double(Celluloid::IO::TCPSocket)
+    ::TCPSocket.should_not_receive(:new)
+    Celluloid::IO::TCPSocket.should_receive(:new).and_return(sock)
+
+    conn = Poseidon::Connection.new "localhost", 9092, "client-id"
+    conn.send(:ensure_connected).should == sock
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,16 @@
+ENV['FLUQ_ROOT'] ||= File.expand_path("../scenario/", __FILE__)
+
+require 'bundler/setup'
+require 'rspec'
+require 'coveralls'
+Coveralls.wear_merged!
+
+require 'fluq/kafka'
+require 'fluq/testing'
+
+FluQ.logger = Logger.new(FluQ.root.join("log", "fluq.log").to_s)
+RSpec.configure do |c|
+  c.after do
+    FileUtils.rm_rf FluQ.root.join('log/kafka').to_s
+  end
+end

metadata

@@ -0,0 +1,158 @@
+--- !ruby/object:Gem::Specification
+name: fluq-kafka
+version: !ruby/object:Gem::Version
+  version: 0.8.0
+platform: ruby
+authors:
+- Black Square Media
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: poseidon_cluster
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Kafka plugin for FluQ
+email: info@blacksquaremedia.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".bundle/install.log"
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- fluq-kafka.gemspec
+- lib/fluq-kafka.rb
+- lib/fluq/input/kafka.rb
+- lib/fluq/kafka.rb
+- lib/fluq/kafka/version.rb
+- spec/fluq/input/kafka_spec.rb
+- spec/fluq/kafka_spec.rb
+- spec/scenario/.gitkeep
+- spec/scenario/log/.gitkeep
+- spec/spec_helper.rb
+homepage: https://github.com/bsm/fluq-kafka
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.8.0
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
+specification_version: 4
+summary: FluQ Kafka
+test_files: []
+has_rdoc: