kafka_syrup 0.9.0

data/lib/kafka_syrup/topic_producer.rb

@@ -0,0 +1,59 @@
+module KafkaSyrup
+  class TopicProducer
+    include Utils
+
+    attr_accessor :topic, :brokers, :partitions, :partitioner, :opts
+
+    def initialize(topic, *args)
+      self.topic = topic
+      self.opts = args.last || {}
+
+      self.brokers = {}
+      self.partitions = {}
+
+      refresh_metadata
+
+      self.partitioner ||= ->(msg) do
+        msg.hash % partitions.count
+      end
+    end
+
+    def refresh_metadata
+      brokers.each{ |id, broker| broker && broker.socket && broker.socket.close }
+
+      self.brokers.clear
+      self.partitions.clear
+
+      meta = KafkaSyrup.get_metadata(topic)
+
+      meta.brokers.each do |broker|
+        broker.extend KafkaSyrup::Broker::Communications
+        brokers.store broker.node, broker
+      end
+
+      meta.topics.first.partitions.each do |partition|
+        partitions.store partition.id, partition
+      end
+    end
+
+    def send_message(msg)
+      id = partitioner.call(msg)
+
+      request = KafkaSyrup::Protocol::ProduceRequest.new
+      request.add_topic(topic).add_partition(id).add_message(msg)
+
+      broker_for_id(id).send_request(request)
+
+    rescue KafkaResponseErrors::NotLeaderForPartition, SocketReadError
+      refresh_metadata
+
+      retry
+    end
+
+    private
+
+    def broker_for_id(id)
+      brokers[partitions[id].leader]
+    end
+  end
+end

data/lib/kafka_syrup/utils.rb

@@ -0,0 +1,15 @@
+module KafkaSyrup
+  module Utils
+    def load_args(*args)
+      if args.last.is_a?(Hash)
+        args.last.each do |attr, val|
+          self.send("#{attr}=", val) if self.respond_to?("#{attr}=")
+        end
+      end
+    end
+
+    def log
+      KafkaSyrup.config.logger
+    end
+  end
+end

data/lib/kafka_syrup/version.rb

@@ -0,0 +1,3 @@
+module KafkaSyrup
+  VERSION = "0.9.0"
+end

metadata

@@ -0,0 +1,188 @@
+--- !ruby/object:Gem::Specification
+name: kafka_syrup
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+platform: ruby
+authors:
+- Delbert Mitten
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-06-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: zk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.6'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+description: A high level Kafka client that supports producer, low level consumers,
+  and high level consumers.
+email:
+- drmitten@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- kafka_syrup.gemspec
+- lib/kafka_syrup.rb
+- lib/kafka_syrup/broker.rb
+- lib/kafka_syrup/encoding.rb
+- lib/kafka_syrup/errors.rb
+- lib/kafka_syrup/partition_consumer.rb
+- lib/kafka_syrup/protocol.rb
+- lib/kafka_syrup/protocol/base.rb
+- lib/kafka_syrup/protocol/errors.rb
+- lib/kafka_syrup/protocol/fetch_request.rb
+- lib/kafka_syrup/protocol/fetch_response.rb
+- lib/kafka_syrup/protocol/message.rb
+- lib/kafka_syrup/protocol/message_set.rb
+- lib/kafka_syrup/protocol/metadata_request.rb
+- lib/kafka_syrup/protocol/metadata_response.rb
+- lib/kafka_syrup/protocol/offset_request.rb
+- lib/kafka_syrup/protocol/offset_response.rb
+- lib/kafka_syrup/protocol/produce_request.rb
+- lib/kafka_syrup/protocol/produce_response.rb
+- lib/kafka_syrup/protocol/request.rb
+- lib/kafka_syrup/protocol/response.rb
+- lib/kafka_syrup/topic_consumer.rb
+- lib/kafka_syrup/topic_producer.rb
+- lib/kafka_syrup/utils.rb
+- lib/kafka_syrup/version.rb
+homepage: https://github.com/drmitten/kafka_syrup
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: A high level Kafka client.
+test_files: []