ruby-kafka 0.1.0.pre.beta2 → 0.1.0.pre.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6d8c3294bd19a1c201194ff051f6d98d7a6dc273
-  data.tar.gz: 1769607ad012c3913b1ad707922ec271cc7a4567
+  metadata.gz: e1c4fdc58659935c4d1e3af22cf746fc88f9dacc
+  data.tar.gz: 23f8e2ba29251ff66499cad11d22d029cbb24d58
 SHA512:
-  metadata.gz: 02b14e2350078b1555cadf35a5de7eba1ca5f218a65ddfdef7e9e76f8a7f7089c85a8a1ae40623566f37ee1675c63354a240c9ce97c3e7797180b137155de2fe
-  data.tar.gz: 603e25244557ba9bd684c5f30543654b224ffabfb7bde6eff59ed88805e6618d89f846afcf40eb2113c9fbc44d02f57ab7c825ee03c9535c3f0be139739a8725
+  metadata.gz: 522eb1d15cedc44ce760a381a710c3bd4beda7f21adc6f30bf438837de2ae75be9dfc82a57417ee7d3d1ef99bb599d80ed92949c6d9b1633e46395e5898c3aa4
+  data.tar.gz: 49f0129fecb9724874895751fafcbe0da1f6f3434ffe31daacc5f69876e2dda68cd91f040f385a5620e90386effc0728cb943717be320f5e64da46194eaa708b

data/Gemfile

@@ -1,8 +1,4 @@
 source "https://rubygems.org"
 ruby "2.2.3"
 
-# Specify your gem's dependencies in kafka.gemspec
 gemspec
-
-gem "dotenv"
-gem "docker-api"

data/README.md

@@ -25,23 +25,27 @@ Or install it yourself as:
 Currently, only the Producer API is supported. A Kafka 0.9 compatible Consumer API is on the roadmap.
 
 ```ruby
-# The client must be initialized with at least one Kafka broker.
+# A client must be initialized with at least one Kafka broker. Each client keeps
+# a separate pool of broker connections. Don't use the same client from more than
+# one thread.
 kafka = Kafka.new(
   seed_brokers: ["kafka1:9092", "kafka2:9092"],
   client_id: "my-app",
   logger: Logger.new($stderr),
 )
 
-# Each producer keeps a separate pool of broker connections. Don't use the same
-# producer from more than one thread.
+# A producer buffers messages and sends them to the broker that is the leader of
+# the partition a given message is being produced to.
 producer = kafka.get_producer
 
-# `write` will buffer the message in the producer.
-producer.write("hello1", key: "x", topic: "test-messages", partition: 0)
-producer.write("hello2", key: "y", topic: "test-messages", partition: 1)
+# `produce` will buffer the message in the producer.
+producer.produce("hello1", key: "x", topic: "test-messages", partition: 0)
+producer.produce("hello2", key: "y", topic: "test-messages", partition: 1)
 
-# `flush` will send the buffered messages to the cluster.
-producer.flush
+# `send_messages` will send the buffered messages to the cluster. Since messages
+# may be destined for different partitions, this could involve writing to more
+# than one Kafka broker.
+producer.send_messages
 ```
 
 ## Development

data/kafka.gemspec

@@ -10,7 +10,13 @@ Gem::Specification.new do |spec|
   spec.email         = ["daniel.schierbeck@gmail.com"]
 
   spec.summary       = %q{A client library for the Kafka distributed commit log.}
-  spec.description   = %q{A client library for the Kafka distributed commit log. Still very much at the alpha stage.}
+
+  spec.description   = <<-DESC.gsub(/^    /, "").strip
+    A client library for the Kafka distributed commit log.
+
+    Currently, only the Producer API is implemented.
+  DESC
+
   spec.homepage      = "https://github.com/zendesk/ruby-kafka"
   spec.license       = "Apache License Version 2.0"
 
@@ -23,4 +29,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "pry"
+  spec.add_development_dependency "dotenv"
+  spec.add_development_dependency "docker-api"
 end

data/lib/kafka/broker_pool.rb

@@ -15,6 +15,7 @@ module Kafka
     # @param seed_brokers [Array<String>]
     # @param client_id [String]
     # @param logger [Logger]
+    # @param socket_timeout [Integer, nil] see {Connection#initialize}.
     def initialize(seed_brokers:, client_id:, logger:, socket_timeout: nil)
       @client_id = client_id
       @logger = logger
@@ -45,6 +46,10 @@ module Kafka
       cluster_info.partitions_for(topic)
     end
 
+    def topics
+      cluster_info.topics.map(&:topic_name)
+    end
+
     def shutdown
       @brokers.each do |id, broker|
         @logger.info "Disconnecting broker #{id}"

data/lib/kafka/client.rb

@@ -18,21 +18,35 @@ module Kafka
     #
     # @return [Client]
     def initialize(seed_brokers:, client_id:, logger:, socket_timeout: nil)
-      @seed_brokers = seed_brokers
-      @client_id = client_id
       @logger = logger
-      @socket_timeout = socket_timeout
+
+      @broker_pool = BrokerPool.new(
+        seed_brokers: seed_brokers,
+        client_id: client_id,
+        logger: logger,
+        socket_timeout: socket_timeout,
+      )
     end
 
+    # Builds a new producer.
+    #
+    # +options+ are passed to {Producer#initialize}.
+    #
+    # @see Producer#initialize
+    # @return [Producer] the Kafka producer.
     def get_producer(**options)
-      broker_pool = BrokerPool.new(
-        seed_brokers: @seed_brokers,
-        client_id: @client_id,
-        logger: @logger,
-        socket_timeout: @socket_timeout,
-      )
+      Producer.new(broker_pool: @broker_pool, logger: @logger, **options)
+    end
+
+    # Lists all topics in the cluster.
+    #
+    # @return [Array<String>] the list of topic names.
+    def topics
+      @broker_pool.topics
+    end
 
-      Producer.new(broker_pool: broker_pool, logger: @logger, **options)
+    def close
+      @broker_pool.shutdown
     end
   end
 end

data/lib/kafka/producer.rb

@@ -8,19 +8,19 @@ module Kafka
   #
   # == Buffering
   #
-  # The producer buffers pending messages until {#flush} is called. Note that there is
+  # The producer buffers pending messages until {#send_messages} is called. Note that there is
   # a maximum buffer size (default is 1,000 messages) and writing messages after the
   # buffer has reached this size will result in a BufferOverflow exception. Make sure
-  # to periodically call {#flush} or set +max_buffer_size+ to an appropriate value.
+  # to periodically call {#send_messages} or set +max_buffer_size+ to an appropriate value.
   #
   # Buffering messages and sending them in batches greatly improves performance, so
-  # try to avoid flushing after every write. The tradeoff between throughput and
+  # try to avoid sending messages after every write. The tradeoff between throughput and
   # message delays depends on your use case.
   #
   # == Error Handling and Retries
   #
   # The design of the error handling is based on having a {MessageBuffer} hold messages
-  # for all topics/partitions. Whenever we want to flush messages to the cluster, we
+  # for all topics/partitions. Whenever we want to send messages to the cluster, we
   # group the buffered messages by the broker they need to be sent to and fire off a
   # request to each broker. A request can be a partial success, so we go through the
   # response and inspect the error code for each partition that we wrote to. If the
@@ -39,7 +39,7 @@ module Kafka
     #
     # @param logger [Logger]
     #
-    # @param timeout [Integer] The number of seconds a broker can wait for
+    # @param ack_timeout [Integer] The number of seconds a broker can wait for
     #   replicas to acknowledge a write before responding with a timeout.
     #
     # @param required_acks [Integer] The number of replicas that must acknowledge
@@ -54,19 +54,19 @@ module Kafka
     # @param max_buffer_size [Integer] the number of messages allowed in the buffer
     #   before new writes will raise BufferOverflow exceptions.
     #
-    def initialize(broker_pool:, logger:, timeout: 10, required_acks: 1, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000)
+    def initialize(broker_pool:, logger:, ack_timeout: 10, required_acks: 1, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000)
       @broker_pool = broker_pool
       @logger = logger
       @required_acks = required_acks
-      @timeout = timeout
+      @ack_timeout = ack_timeout
       @max_retries = max_retries
       @retry_backoff = retry_backoff
       @max_buffer_size = max_buffer_size
       @buffer = MessageBuffer.new
     end
 
-    # Writes a message to the specified topic. Note that messages are buffered in
-    # the producer until {#flush} is called.
+    # Produces a message to the specified topic. Note that messages are buffered in
+    # the producer until {#send_messages} is called.
     #
     # == Partitioning
     #
@@ -94,7 +94,7 @@ module Kafka
     #
     # @raise [BufferOverflow] if the maximum buffer size has been reached.
     # @return [nil]
-    def write(value, key: nil, topic:, partition: nil, partition_key: nil)
+    def produce(value, key: nil, topic:, partition: nil, partition_key: nil)
       unless buffer_size < @max_buffer_size
         raise BufferOverflow, "Max buffer size #{@max_buffer_size} exceeded"
       end
@@ -113,20 +113,20 @@ module Kafka
       partition
     end
 
-    # Flushes all messages to the Kafka brokers.
+    # Sends all buffered messages to the Kafka brokers.
     #
     # Depending on the value of +required_acks+ used when initializing the producer,
     # this call may block until the specified number of replicas have acknowledged
-    # the writes. The +timeout+ setting places an upper bound on the amount of time
-    # the call will block before failing.
+    # the writes. The +ack_timeout+ setting places an upper bound on the amount of
+    # time the call will block before failing.
     #
     # @raise [FailedToSendMessages] if not all messages could be successfully sent.
     # @return [nil]
-    def flush
+    def send_messages
       attempt = 0
 
       loop do
-        @logger.info "Flushing #{@buffer.size} messages"
+        @logger.info "Sending #{@buffer.size} messages"
 
         attempt += 1
         transmit_messages
@@ -166,6 +166,9 @@ module Kafka
       @buffer.size
     end
 
+    # Closes all connections to the brokers.
+    #
+    # @return [nil]
     def shutdown
       @broker_pool.shutdown
     end
@@ -191,7 +194,7 @@ module Kafka
           response = broker.produce(
             messages_for_topics: message_set.to_h,
             required_acks: @required_acks,
-            timeout: @timeout * 1000, # Kafka expects the timeout in milliseconds.
+            timeout: @ack_timeout * 1000, # Kafka expects the timeout in milliseconds.
           )
 
           handle_response(response) if response
@@ -229,7 +232,7 @@ module Kafka
           @logger.error "Messages written, but to fewer in-sync replicas than required for #{topic}/#{partition}"
         else
           offset = partition_info.offset
-          @logger.info "Successfully flushed messages for #{topic}/#{partition}; new offset is #{offset}"
+          @logger.info "Successfully sent messages for #{topic}/#{partition}; new offset is #{offset}"
 
           # The messages were successfully written; clear them from the buffer.
           @buffer.clear_messages(topic: topic, partition: partition)

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.1.0-beta2"
+  VERSION = "0.1.0-beta3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.beta2
+  version: 0.1.0.pre.beta3
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-28 00:00:00.000000000 Z
+date: 2016-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,8 +66,38 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: A client library for the Kafka distributed commit log. Still very much
-  at the alpha stage.
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: docker-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |-
+  A client library for the Kafka distributed commit log.
+
+  Currently, only the Producer API is implemented.
 email:
 - daniel.schierbeck@gmail.com
 executables: []