fluent-plugin-kafkaclient 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 06d9d7dd61b1f86d40e0ac9970982a079cdabf93
+  data.tar.gz: fe644a4924d928183bfbcaa29272f8e5bc49412c
+SHA512:
+  metadata.gz: 122625d7a86924f1af1a795610d98207cd93a9118c5e85d8938818dcaf8d257ddb7f781b57c07bdc8fb8603b9537bdea3b8aba7e1863aa7435316439801399bd
+  data.tar.gz: 5b7771c0bc3d951070fe147929b6a30d78f7e470fd12dd5d5859bcfb3ec7b4b27ee43017c2d844fd4f1f38c02fad528ed592c9decb8e7b9cbadf4eaed01ee657

data/.gitignore

@@ -0,0 +1,11 @@
+*.gem
+.idea
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,49 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, and in the interest of
+fostering an open and welcoming community, we pledge to respect all people who
+contribute through reporting issues, posting feature requests, updating
+documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information, such as physical or electronic
+  addresses, without explicit permission
+* Other unethical or unprofessional conduct
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+By adopting this Code of Conduct, project maintainers commit themselves to
+fairly and consistently applying these principles to every aspect of managing
+this project. Project maintainers who do not follow or enforce the Code of
+Conduct may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting a project maintainer at minamiya@r.recruit.co.jp. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. Maintainers are
+obligated to maintain confidentiality with regard to the reporter of an
+incident.
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 1.3.0, available at
+[http://contributor-covenant.org/version/1/3/0/][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/3/0/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kafkaclient.gemspec
+gemspec

data/README.md

@@ -0,0 +1,112 @@
+# Fluent::Plugin::Kafkaclient
+Fluentd plugin for Apache Kafka.
+This pluing uses [ruby-kafka](https://github.com/zendesk/ruby-kafka) as ruby client library.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-kafkaclient'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-kafkaclient
+
+## Usage
+
+### Producing Messages to Kafka (Buffered Output Plugin)
+This is basic configuration.
+
+```
+<match *.**>
+  @type               kafka_out_buffered
+  client_id           <client id> :default => producer_000
+  brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>
+  topic               <kafka topic>
+  producer_type       (sync|async) :default => sync
+  partition_key       (string)     :default => nil
+  buffer_type         (file|memory)
+  output_data_type    (none|json|msgpack) :default => none
+  output_include_tag  (true|false) :default => false
+  output_include_time (true|false) :default => false
+</match>
+```
+
+#### Sync Producer
+
+These Parameters are used for Sync Producer.
+
+```
+  required_acks, :integer, :default => 1
+  ack_timeout, :integer, :default => 2
+  compression_codec, (snappy|gzip|nil), :default => nil
+  compression_threshold, :integer, :default => 1
+  max_retries, :string, :integer, :default => 2
+  retry_backoff, :string, :integer, :default => 1
+  max_buffer_size, :integer, :default => 1000
+```
+- ```required_acks``` The number of acknowledgments the producer requires the leader to have received before considering a request complete.
+- ```ack_timeout``` a timeout executed by a broker when the client is sending messages to it
+- ```compression_codec``` you can choose snappy or gzip to compress.
+- ```compression_threshold``` the number of messages to compress one time.
+- ```max_retries``` the maximum number of retries to attempt
+- ```retry_backoff``` the number of seconds to wait after a failed attempt to send messages to a Kafka broker before retrying.
+- ```max_buffer_size``` the maximum size of the producer buffer.
+
+#### Async Producer
+These Parameters can be used for Async Producer.
+
+```
+  max_queue_size, :integer, :default => 1000
+  delivery_threshold, :integer, :default => 0
+  delivery_interval, :integer, :default => 0
+```
+- ```deliver_threshold``` Trigger a delivery once 'deliver_threshold' messages have been buffered
+- ```delivery_interval``` Trigger a delivery every 'delivery_interval' seconds.
+
+### Encryption and Authentication using TSL
+
+```
+  encryption           :bool, :default => false
+  authentication       :bool, :default => false
+  ca_cert_path         <path to ca_cert>:default => nil
+  client_cert_path     <path to client cert> :default => nil
+  client_cert_key_path <path to client cert key> :default => nil
+```
+
+In order to encrypt messages, you just need to
+1. make param of encryption true
+2. set a valid CA certificate path to param of ca_cert_path
+
+In order to authenticate the client to the cluster, 
+1. make param of authentication true
+2. set each path of a certificate and key created for the client and trusted by the brokers.
+
+For details, Please see document about [ruby-kafka](https://github.com/zendesk/ruby-kafka) and [Apache Kafka](http://kafka.apache.org)
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (git checkout -b my-new-feature)
+3. Commit your changes (git commit -am 'Add some feature')
+4. Push to the branch (git push origin my-new-feature)
+5. Create new Pull Request
+
+## License
+Copyright 2015 Kazuki Minamiya
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fluent/plugin/kafkaclient"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/fluent-plugin-kafkaclient.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-kafkaclient"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Kazuki Minamiya"]
+  spec.email         = ["minami.ind@gmail.com"]
+
+  spec.summary       = %q{Kafka client plugin which supports version 0.9 of kafka.}
+  spec.description   = %q{Kafka client Plugin which supports version 0.9 of kafka.}
+  spec.homepage      = "https://github.com/nantani/fluent-plugin-kafkaclient"
+  spec.license       = "APLv2"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.required_ruby_version = '>= 2.2.0'
+  spec.add_dependency "fluentd"
+  spec.add_dependency "ruby-kafka"
+  spec.add_dependency "yajl"
+  spec.add_dependency "msgpack"
+  spec.add_dependency "activesupport", ">= 4.2.0", "< 5.1"
+  spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/fluent/plugin/kafka_out_buffered.rb

@@ -0,0 +1,137 @@
+# encode: utf-8
+class Fluent::KafkaOutBuffered < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('kafka_out_buffered', self)
+
+  # ruby-kafka plugin main options
+  config_param :client_id, :string, :default => 'producer_000'
+  config_param :brokers, :string, :default => 'localhost:9092'
+  config_param :topic, :string, :default => nil
+  config_param :partition_key, :string, :default => nil
+  config_param :output_data_type, :string, :default => nil
+  config_param :output_include_tag, :bool, :default => false
+  config_param :output_include_time, :bool, :default => false
+  config_param :producer_type, :string, :default => 'sync'
+
+  # Sync Producer options
+  config_param :required_acks, :integer, :default => 1
+  config_param :ack_timeout, :integer, :default => 2
+  config_param :compression_codec, :string, :default => nil
+  config_param :compression_threshold, :integer, :default => 1
+  config_param :max_retries, :string, :integer, :default => 2
+  config_param :retry_backoff, :string, :integer, :default => 1
+  config_param :max_buffer_size, :integer, :default => 1000
+
+  # Async Producer options
+  config_param :max_queue_size, :integer, :default => 1000
+  config_param :delivery_threshold, :integer, :default => 0
+  config_param :delivery_interval, :integer, :default => 0
+
+  # encryption and authentication options
+  config_param :encryption, :bool, :default => false
+  config_param :authentication, :bool, :default => false
+  config_param :ca_cert_path, :string,:default => nil
+  config_param :client_cert_path, :string, :default => nil
+  config_param :client_cert_key_path, :string, :default => nil
+
+  def initialize
+    super
+    require 'kafka'
+    require "active_support/notifications"
+    require 'Yajl'
+  end
+
+  def configure(conf)
+    super
+
+    if @encryption
+      raise Fluent::ConfigError, "CA cert file is not found or invalid" unless File.readable?(@ca_cert_path)
+      @ca_cert = File.read(@ca_cert_path)
+      $log.info "ca_cert is valid"
+    end
+
+    if @authentication
+      raise Fluent::ConfigError, "Client cert file is not found or invalid" unless File.readable?(@client_cert_path)
+      raise Fluent::ConfigError, "Client cert key is not found or invalid" unless File.readable?(@client_cert_key_path)
+      @client_cert = File.read(@client_cert_path)
+      @client_cert_key = File.read(@client_cert_key_path)
+      $log.info "client cert and client cert key is valid"
+    end
+
+  end
+
+  def  build_producer()
+    @kafka = Kafka.new(
+      seed_brokers: @brokers.split(','),
+      ssl_ca_cert:  @ca_cert,
+      ssl_client_cert: @client_cert,
+      ssl_client_cert_key: @client_cert_key
+    )
+
+    if @producer_type == 'sync'
+      @producer = @kafka.producer(
+        required_acks: @required_acks,
+        ack_timeout: @ack_timeout,
+        compression_codec: @compression_codec,
+        compression_threshold: @compression_threshold,
+        max_retries: @max_retries,
+        retry_backoff: @retry_backoff,
+        max_buffer_size: @max_buffer_size
+      )
+
+    elsif @producer_type == 'async'
+      @producer = @kafka.async_producer(
+        max_queue_size: @max_queue_size,
+        delivery_threshold: @delivery_threshold,
+        delivery_interval: @delivery_interval
+      )
+    else
+      raise Fluent::ConfigError, "Producer type parameter, #{@producer_type}, is invalid"
+
+    $log.info "produer type is #{@producer_type}"
+    end
+  end
+
+  def start
+    super
+    build_producer()
+  end
+
+  def shutdown
+    super
+    @producer.shutdown
+  end
+
+  def format(tag, time, record)
+    [tag, time, record].to_msgpack
+  end
+
+  def encode(record)
+    if @output_data_type == 'msgpack'
+      record.to_msgpack
+    elsif @output_data_type == 'json'
+      Yajl::Encoder.encode(record)
+    elsif none
+      record
+    end
+  end
+
+  def write(chunk)
+
+    chunk.msgpack_each { |(tag, time, record)|
+
+      record['tag'] = tag if @output_include_tag
+      record['time'] = time if @output_include_time
+      encoded_record=encode(record)
+
+      @producer.produce(
+        encoded_record,
+        topic: @topic,
+        partition_key: @partition_key
+      )
+
+      @producer.deliver_messages
+    }
+
+  end
+
+end

metadata

@@ -0,0 +1,171 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-kafkaclient
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Kazuki Minamiya
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-04-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-kafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yajl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Kafka client Plugin which supports version 0.9 of kafka.
+email:
+- minami.ind@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- fluent-plugin-kafkaclient.gemspec
+- lib/fluent/plugin/kafka_out_buffered.rb
+homepage: https://github.com/nantani/fluent-plugin-kafkaclient
+licenses:
+- APLv2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Kafka client plugin which supports version 0.9 of kafka.
+test_files: []