fluent-plugin-kafka 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5d2a69320d98e30694e5599b3bf38bd492ba1338
-  data.tar.gz: 5faf0f964137b61e90c55bd7f2dccd3fc8022ab6
+  metadata.gz: 95430357052a2b7305214ffa1f5c0ad426e52914
+  data.tar.gz: 57736e74a0bc9b65441bc41086f7b9875e72a590
 SHA512:
-  metadata.gz: b21ee5f9e9c0e6a163aa4d82e0c0c028da102441da56868beac52d8cefa755ac08d38e8f225733b408a00f055e31bfa47766497863f812c675478bcc587ff881
-  data.tar.gz: 2e5f7d4e225822f14559cda6fb4c5d103e0517d63fdc61b3fe3356db90ab494125d606a0deeaa308eaddab809d08ff52ba974ad0b70724a8f1c7e8596ec8a41c
+  metadata.gz: a7f89337ddc1f28e8762de822738d467d16ef454a0bf64570f8bfbd13420dbbc53a2477b4cce836c8234d7aee96726a13d0fbf12ac9e33216f6cf82eea1ee174
+  data.tar.gz: 5568aec86102a4c40aae4e1e8b43c5e1829811e9014589e45981491fc2b1fece473f0042f1b7188e3b09d678eff8cb77e0ac63de7a00d09fe34f7b9bfe80b12f

data/.gitignore

@@ -0,0 +1,2 @@
+/Gemfile.lock
+*.swp

data/.travis.yml

@@ -0,0 +1,17 @@
+language: ruby
+
+rvm:
+  - 2.1
+  - 2.2
+  - 2.3.1
+  - ruby-head
+
+script:
+  - bundle exec rake test
+
+sudo: false
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+

data/README.md

@@ -1,6 +1,9 @@
-# Fluent::Plugin::Kafka
+# fluent-plugin-kafka, a plugin for [Fluentd](http://fluentd.org)
+
+[![Build Status](https://travis-ci.org/htgc/fluent-plugin-kafka.svg?branch=master)](https://travis-ci.org/htgc/fluent-plugin-kafka)
+
+A fluentd plugin to both consume and produce data for Apache Kafka.
 
-TODO: Write a gem description
 TODO: Also, I need to write tests
 
 ## Installation

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.2.1'
+  gem.version = '0.2.2'
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'poseidon_cluster'
   gem.add_dependency 'ltsv'

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -0,0 +1,24 @@
+require 'kafka/producer'
+
+module Kafka
+  class Producer
+    def produce2(value, key: nil, topic:, partition: nil, partition_key: nil)
+      create_time = Time.now
+
+      message = PendingMessage.new(
+        value,
+        key,
+        topic,
+        partition,
+        partition_key,
+        create_time,
+        key.to_s.bytesize + value.to_s.bytesize
+      )
+
+      @target_topics.add(topic)
+      @pending_message_queue.write(message)
+
+      nil
+    end
+  end
+end

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -8,6 +8,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
     super
 
     require 'kafka'
+    require 'fluent/plugin/kafka_producer_ext'
 
     @kafka = nil
     @producers = {}
@@ -126,8 +127,7 @@ DESC
 
     @formatter_proc = setup_formatter(conf)
 
-    @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks,
-                      max_buffer_size: @buffer.buffer_chunk_limit / 10, max_buffer_bytesize: @buffer.buffer_chunk_limit * 2}
+    @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks}
     @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
     @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
   end
@@ -233,7 +233,7 @@ DESC
         end
         log.on_trace { log.trace("message will send to #{topic} with key: #{partition_key} and value: #{record_buf}.") }
         messages += 1
-        producer.produce(record_buf, topic: topic, partition_key: partition_key)
+        producer.produce2(record_buf, topic: topic, partition_key: partition_key)
         messages_bytes += record_buf_bytes
 
         records_by_topic[topic] += 1

data/test/plugin/test_out_kafka.rb

@@ -1,11 +1,16 @@
 require 'helper'
+require 'fluent/output'
 
 class KafkaOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
-  CONFIG = %[
+  BASE_CONFIG = %[
+    type kafka_buffered
+  ]
+
+  CONFIG = BASE_CONFIG + %[
     default_topic kitagawakeiko
     brokers localhost:9092
   ]
@@ -15,6 +20,20 @@ class KafkaOutputTest < Test::Unit::TestCase
   end
 
   def test_configure
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(BASE_CONFIG)
+    }
+    
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(CONFIG)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(CONFIG + %[
+        buffer_type memory
+      ])
+    }
+
     d = create_driver
     assert_equal 'kitagawakeiko', d.instance.default_topic
     assert_equal 'localhost:9092', d.instance.brokers

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-06 00:00:00.000000000 Z
+date: 2016-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -121,6 +121,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE
 - README.md
@@ -128,6 +130,7 @@ files:
 - fluent-plugin-kafka.gemspec
 - lib/fluent/plugin/in_kafka.rb
 - lib/fluent/plugin/in_kafka_group.rb
+- lib/fluent/plugin/kafka_producer_ext.rb
 - lib/fluent/plugin/out_kafka.rb
 - lib/fluent/plugin/out_kafka_buffered.rb
 - test/helper.rb