ruby-kafka 0.1.0.pre.beta3 → 0.1.0.pre.beta4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e1c4fdc58659935c4d1e3af22cf746fc88f9dacc
-  data.tar.gz: 23f8e2ba29251ff66499cad11d22d029cbb24d58
+  metadata.gz: 71fcf73c1adb43253684cb0ef1406a0631c3ce3a
+  data.tar.gz: dde9f9123b31f20475c7d280532f0b135f6a84bf
 SHA512:
-  metadata.gz: 522eb1d15cedc44ce760a381a710c3bd4beda7f21adc6f30bf438837de2ae75be9dfc82a57417ee7d3d1ef99bb599d80ed92949c6d9b1633e46395e5898c3aa4
-  data.tar.gz: 49f0129fecb9724874895751fafcbe0da1f6f3434ffe31daacc5f69876e2dda68cd91f040f385a5620e90386effc0728cb943717be320f5e64da46194eaa708b
+  metadata.gz: 8c44cc61c8b50ab3854830596bfd965837e03e1af97c3917c3ecf4386bab38534184688ae5dc1f4c284c8f2878143637e9dffdfe35e95c504a8f1381ca4da6e5
+  data.tar.gz: 06de9ac78b013cf448e03d0ba4ec879059bfbc04c6626c373e30c4c0c95c3b23282b30c3607df8955ab4c617d5d31d67c8ae80033381212baccf107aefda0489

data/.yardopts

@@ -0,0 +1,3 @@
+--markup-provider=redcarpet
+--markup=markdown
+--title 'Kafka'

data/README.md

@@ -39,8 +39,20 @@ kafka = Kafka.new(
 producer = kafka.get_producer
 
 # `produce` will buffer the message in the producer.
-producer.produce("hello1", key: "x", topic: "test-messages", partition: 0)
-producer.produce("hello2", key: "y", topic: "test-messages", partition: 1)
+producer.produce("hello1", topic: "test-messages")
+
+# It's possible to specify a message key:
+producer.produce("hello2", key: "x", topic: "test-messages")
+
+# If you need to control which partition a message should be written to, you
+# can pass in the `partition` parameter:
+producer.produce("hello3", topic: "test-messages", partition: 1)
+
+# If you don't know exactly how many partitions are in the topic, or you'd
+# rather have some level of indirection, you can pass in `partition_key`.
+# Two messages with the same partition key will always be written to the
+# same partition.
+producer.produce("hello4", topic: "test-messages", partition_key: "yo")
 
 # `send_messages` will send the buffered messages to the cluster. Since messages
 # may be destined for different partitions, this could involve writing to more

data/examples/simple-producer.rb

@@ -0,0 +1,34 @@
+# Reads lines from STDIN, writing them to Kafka.
+
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+
+require "kafka"
+
+logger = Logger.new($stderr)
+brokers = ENV.fetch("KAFKA_BROKERS").split(",")
+
+# Make sure to create this topic in your Kafka cluster or configure the
+# cluster to auto-create topics.
+topic = "random-messages"
+
+kafka = Kafka.new(
+  seed_brokers: brokers,
+  client_id: "simple-producer",
+  logger: logger,
+)
+
+producer = kafka.get_producer
+
+begin
+  $stdin.each_with_index do |line, index|
+    producer.produce(line, topic: topic)
+
+    # Send messages for every 10 lines.
+    producer.send_messages if index % 10 == 0
+  end
+ensure
+  # Make sure to send any remaining messages.
+  producer.send_messages
+
+  producer.shutdown
+end

data/lib/kafka/broker_pool.rb

@@ -66,10 +66,10 @@ module Kafka
     # Fetches the cluster metadata.
     #
     # This is used to update the partition leadership information, among other things.
-    # The methods will go through each node listed in +seed_brokers+, connecting to the
+    # The methods will go through each node listed in `seed_brokers`, connecting to the
     # first one that is available. This node will be queried for the cluster metadata.
     #
-    # @raise [ConnectionError] if none of the nodes in +seed_brokers+ are available.
+    # @raise [ConnectionError] if none of the nodes in `seed_brokers` are available.
     # @return [Protocol::MetadataResponse] the cluster metadata.
     def fetch_cluster_info
       @seed_brokers.each do |node|
@@ -93,6 +93,8 @@ module Kafka
           return cluster_info
         rescue Error => e
           @logger.error "Failed to fetch metadata from #{node}: #{e}"
+        ensure
+          broker.disconnect unless broker.nil?
         end
       end
 

data/lib/kafka/client.rb

@@ -30,7 +30,7 @@ module Kafka
 
     # Builds a new producer.
     #
-    # +options+ are passed to {Producer#initialize}.
+    # `options` are passed to {Producer#initialize}.
     #
     # @see Producer#initialize
     # @return [Producer] the Kafka producer.

data/lib/kafka/connection.rb

@@ -1,4 +1,5 @@
 require "socket"
+require "stringio"
 require "kafka/protocol/request_message"
 require "kafka/protocol/encoder"
 require "kafka/protocol/decoder"
@@ -67,7 +68,7 @@ module Kafka
     # @param request [#encode] the request that should be encoded and written.
     # @param response_class [#decode] an object that can decode the response.
     #
-    # @return [Object] the response that was decoded by +response_class+.
+    # @return [Object] the response that was decoded by `response_class`.
     def request(api_key, request, response_class)
       write_request(api_key, request)
 

data/lib/kafka/producer.rb

@@ -6,18 +6,18 @@ module Kafka
 
   # Allows sending messages to a Kafka cluster.
   #
-  # == Buffering
+  # ## Buffering
   #
   # The producer buffers pending messages until {#send_messages} is called. Note that there is
   # a maximum buffer size (default is 1,000 messages) and writing messages after the
   # buffer has reached this size will result in a BufferOverflow exception. Make sure
-  # to periodically call {#send_messages} or set +max_buffer_size+ to an appropriate value.
+  # to periodically call {#send_messages} or set `max_buffer_size` to an appropriate value.
   #
   # Buffering messages and sending them in batches greatly improves performance, so
   # try to avoid sending messages after every write. The tradeoff between throughput and
   # message delays depends on your use case.
   #
-  # == Error Handling and Retries
+  # ## Error Handling and Retries
   #
   # The design of the error handling is based on having a {MessageBuffer} hold messages
   # for all topics/partitions. Whenever we want to send messages to the cluster, we
@@ -29,7 +29,43 @@ module Kafka
   #
   # After this, we check if the buffer is empty. If it is, we're all done. If it's
   # not, we do another round of requests, this time with just the remaining messages.
-  # We do this for as long as +max_retries+ permits.
+  # We do this for as long as `max_retries` permits.
+  #
+  # ## Example
+  #
+  # This is an example of an application which reads lines from stdin and writes them
+  # to Kafka:
+  #
+  #     require "kafka"
+  #
+  #     logger = Logger.new($stderr)
+  #     brokers = ENV.fetch("KAFKA_BROKERS").split(",")
+  #
+  #     # Make sure to create this topic in your Kafka cluster or configure the
+  #     # cluster to auto-create topics.
+  #     topic = "random-messages"
+  #
+  #     kafka = Kafka.new(
+  #       seed_brokers: brokers,
+  #       client_id: "simple-producer",
+  #       logger: logger,
+  #     )
+  #
+  #     producer = kafka.get_producer
+  #
+  #     begin
+  #       $stdin.each_with_index do |line, index|
+  #         producer.produce(line, topic: topic)
+  #
+  #         # Send messages for every 10 lines.
+  #         producer.send_messages if index % 10 == 0
+  #       end
+  #     ensure
+  #       # Make sure to send any remaining messages.
+  #       producer.send_messages
+  #
+  #       producer.shutdown
+  #     end
   #
   class Producer
 
@@ -68,7 +104,7 @@ module Kafka
     # Produces a message to the specified topic. Note that messages are buffered in
     # the producer until {#send_messages} is called.
     #
-    # == Partitioning
+    # ## Partitioning
     #
     # There are several options for specifying the partition that the message should
     # be written to.
@@ -77,9 +113,9 @@ module Kafka
     # partition number, in which case the message will be assigned a partition at
     # random.
     #
-    # You can also specify the +partition+ parameter yourself. This requires you to
+    # You can also specify the `partition` parameter yourself. This requires you to
     # know which partitions are available, however. Oftentimes the best option is
-    # to specify the +partition_key+ parameter: messages with the same partition
+    # to specify the `partition_key` parameter: messages with the same partition
     # key will always be assigned to the same partition, as long as the number of
     # partitions doesn't change. You can also omit the partition key and specify
     # a message key instead. The message key is part of the message payload, and
@@ -115,9 +151,9 @@ module Kafka
 
     # Sends all buffered messages to the Kafka brokers.
     #
-    # Depending on the value of +required_acks+ used when initializing the producer,
+    # Depending on the value of `required_acks` used when initializing the producer,
     # this call may block until the specified number of replicas have acknowledged
-    # the writes. The +ack_timeout+ setting places an upper bound on the amount of
+    # the writes. The `ack_timeout` setting places an upper bound on the amount of
     # time the call will block before failing.
     #
     # @raise [FailedToSendMessages] if not all messages could be successfully sent.

data/lib/kafka/protocol/encoder.rb

@@ -1,3 +1,5 @@
+require "stringio"
+
 module Kafka
   module Protocol
 

data/lib/kafka/protocol/message.rb

@@ -1,9 +1,10 @@
+require "stringio"
 require "zlib"
 
 module Kafka
   module Protocol
 
-    # == API Specification
+    # ## API Specification
     #
     #     Message => Crc MagicByte Attributes Key Value
     #         Crc => int32

data/lib/kafka/protocol/metadata_response.rb

@@ -8,11 +8,11 @@ module Kafka
     #
     # * For each broker a node id, host, and port is provided.
     # * For each topic partition the node id of the broker acting as partition leader,
-    #   as well as a list of node ids for the set of replicas, are given. The +isr+ list is
+    #   as well as a list of node ids for the set of replicas, are given. The `isr` list is
     #   the subset of replicas that are "in sync", i.e. have fully caught up with the
     #   leader.
     #
-    # == API Specification
+    # ## API Specification
     #
     #     MetadataResponse => [Broker][TopicMetadata]
     #       Broker => NodeId Host Port  (any number of brokers may be returned)
@@ -118,6 +118,11 @@ module Kafka
 
       def partitions_for(topic_name)
         topic = @topics.find {|t| t.topic_name == topic_name }
+
+        if topic.nil?
+          raise UnknownTopicOrPartition, "unknown topic #{topic_name}"
+        end
+
         topic.partitions
       end
 

data/lib/kafka/protocol/produce_request.rb

@@ -1,9 +1,11 @@
+require "stringio"
+
 module Kafka
   module Protocol
 
     # A produce request sends a message set to the server.
     #
-    # == API Specification
+    # ## API Specification
     #
     #     ProduceRequest => RequiredAcks Timeout [TopicName [Partition MessageSetSize MessageSet]]
     #         RequiredAcks => int16

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.1.0-beta3"
+  VERSION = "0.1.0-beta4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.beta3
+  version: 0.1.0.pre.beta4
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-29 00:00:00.000000000 Z
+date: 2016-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -106,6 +106,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".yardopts"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -113,6 +114,7 @@ files:
 - bin/console
 - bin/setup
 - circle.yml
+- examples/simple-producer.rb
 - kafka.gemspec
 - lib/kafka.rb
 - lib/kafka/broker.rb