gelf2kafka 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f79e92751af9147fcde84f2ddff5627de86cccbc
+  data.tar.gz: 47b91830af26859fb69a9d23bd9f403b84ce7676
+SHA512:
+  metadata.gz: c6c11384ba8b1fa55d1339a590c48fffc32b417d669dffc15dc82a54d22f01034a84d777bd75e53307de98c0afdde67322bef54b1d679e6a23dc0905d01be48c
+  data.tar.gz: 35d31bd4bd2bf54a873b88b8b1afb87e0c5f161c3d229270d21a32ce69f7accff17dc4fbea193daeba7ed3be4837cba1a568b469c320952ba8d897393a4902fb

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2016 Supersonic LTD
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,50 @@
+# Gelf2Kafka
+
+Runs gelfd daemon that accepts gelf events and pushes them to kafka.
+
+## Installation
+
+Install libsnappy dev libs if you want to take advantage of compression
+
+    apt-get install libsnappy-dev
+
+Add this line to your application's Gemfile:
+
+    gem 'gelf2kafka'
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install gelf2kafka
+
+## Usage
+
+    $ gelf2kafka -h
+    Usage: gelf2kafka [options]
+            --config PATH                Path to settings config
+        -h, --help                       Display this screen
+    $
+
+## Config
+
+    gelfd:
+      host: localhost
+      port: 11211
+      flush_interval: 1
+      max_batch_events: 1024
+    kafka:
+      brokers: ["broker1:9092", "broker2:9092", "broker3:9092"]
+      producer_type: sync
+      produce: true
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+6. Go to 1

data/bin/gelf2kafka

@@ -0,0 +1,133 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'gelfd'
+require 'poseidon'
+require 'yaml'
+require 'hash_symbolizer'
+require 'schash'
+require 'socket'
+require 'timers'
+require 'logger'
+
+$stdout.sync = true
+
+Thread.abort_on_exception = true
+
+@config = nil
+
+loglevels = {
+  :debug => Logger::DEBUG,
+  :info => Logger::INFO,
+  :warn => Logger::WARN,
+  :error => Logger::Error,
+  :fatal => Logger::FATAL,
+  :unknown => Logger::UNKNOWN
+}
+
+@loglevel = Logger::INFO
+
+opts = OptionParser.new
+opts.banner = "Usage: #{$0} [options]"
+opts.on( '--config PATH', String, 'Path to settings config' ) { |c| @config = c }
+opts.on( '--log-level [LEVEL]', [:debug, :info, :warn, :error, :fatal, :unknown] ) { |l| @loglevel = loglevels[l] }
+opts.on( '-h', '--help', 'Display this screen' ) { puts opts; exit 0 }
+opts.parse!
+
+unless @config
+  puts opts
+  exit 1
+end
+
+@logger = Logger.new(STDOUT)
+
+@settings = YAML.load_file(@config).symbolize_keys(true)
+
+validator = Schash::Validator.new do
+  {
+    gelfd: {
+      host: string,
+      port: integer,
+      flush_interval: integer,
+      max_batch_events: integer
+    },
+    kafka: {
+      brokers: array_of(string),
+      producer_type: match(/^(sync|async)$/),
+      produce: optional(boolean),
+      topic: string
+    },
+  }
+end
+
+unless validator.validate(@settings).empty?
+  @logger.error("ERROR: bad settings")
+  @logger.error(validator.validate(@settings))
+  exit 1
+end
+
+@gelfd_host = @settings[:gelfd][:host]
+@gelfd_port = @settings[:gelfd][:port]
+@flush_interval = @settings[:gelfd][:flush_interval]
+@max_batch_events = @settings[:gelfd][:max_batch_events]
+@brokers = @settings[:kafka][:brokers]
+@producer_type = @settings[:kafka][:producer_type].to_sym
+@produce = @settings[:kafka].has_key?(:produce) ? @settings[:kafka][:produce] : true
+@topic = @settings[:kafka][:topic]
+@producer = Poseidon::Producer.new(@brokers, "#{Socket.gethostname}", :type => @producer_type, :compression_codec => :snappy, :compressed_topics => [ @topic ]) if @produce
+
+@mutex = Mutex.new
+
+@producer_queue = SizedQueue.new(10)
+
+@producer_thread = Thread.new do
+  loop do
+    messages = @producer_queue.pop
+    begin
+      @producer.send_messages(messages) if @produce
+    rescue Poseidon::Errors::UnableToFetchMetadata
+      @logger.warn("Got Poseidon::Errors::UnableToFetchMetadata while trying to produce kafka messages, retrying in 1 second ...")
+      sleep 1
+      retry
+    end
+  end
+end
+
+@timers = Timers::Group.new
+@flush_timer = @timers.every(@flush_interval) do
+  @logger.debug("Flush timer triggered with #{@messages.size} events")
+  @mutex.synchronize do
+    @producer_queue.push(@messages) unless @messages.empty?
+    @messages = []
+  end
+end
+Thread.new { loop { @timers.wait } }
+
+def gelfd
+  server = UDPSocket.new
+  @logger.info("binding gelfd to #{@gelfd_host}:#{@gelfd_port}")
+  server.bind(@gelfd_host, @gelfd_port)
+
+  @messages = []
+
+  loop do
+    data, addr = server.recvfrom(8192)
+    begin
+      res = Gelfd::Parser.parse(data)
+      unless res.nil?
+        @mutex.synchronize do
+          @messages << Poseidon::MessageToSend.new(@topic, res.strip)
+          if @messages.size >= @max_batch_events
+            @logger.debug("Max events threshold reached")
+            @producer_queue.push(@messages)
+            @messages = []
+          end
+        end
+      end
+    rescue Exception => e
+      @logger.warn("Failed parsing gelf message: #{e.message}")
+    end
+  end
+end
+
+gelfd

data/gelf2kafka.gemspec

@@ -0,0 +1,31 @@
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+require "gelf2kafka/version"
+ 
+Gem::Specification.new do |s|
+  s.name        = "gelf2kafka"
+  s.version     = Gelf2Kafka::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Alexander Piavlo"]
+  s.email       = ["devops@supersonic.com"]
+  s.homepage    = "http://github.com/SupersonicAds/gelf2kafka"
+  s.summary     = "Gelf deamon what sends events to kafka"
+  s.description = "Gelf deamon what sends events to kafka"
+  s.license     = 'MIT'
+  s.has_rdoc    = false
+
+  s.add_dependency('gelfd')
+  s.add_dependency('poseidon')
+  s.add_dependency('snappy')
+  s.add_dependency('hash_symbolizer')
+  s.add_dependency('schash')
+  s.add_dependency('timers')
+
+  s.add_development_dependency('rake')
+
+  s.files         = Dir.glob("{bin,lib}/**/*") + %w(gelf2kafka.gemspec LICENSE README.md)
+  s.executables   = Dir.glob('bin/**/*').map { |file| File.basename(file) }
+  s.test_files    = nil
+  s.require_paths = ['lib']
+end

data/lib/gelf2kafka/version.rb

@@ -0,0 +1,3 @@
+module Gelf2Kafka
+  VERSION ||= '0.1.0'
+end

data/lib/gelf2kafka.rb

@@ -0,0 +1,3 @@
+module Gelf2Kafka
+  require 'gelf2kafka/version.rb'
+end

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: gelf2kafka
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Alexander Piavlo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: gelfd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: poseidon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: snappy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hash_symbolizer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: schash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Gelf deamon what sends events to kafka
+email:
+- devops@supersonic.com
+executables:
+- gelf2kafka
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/gelf2kafka
+- gelf2kafka.gemspec
+- lib/gelf2kafka.rb
+- lib/gelf2kafka/version.rb
+homepage: http://github.com/SupersonicAds/gelf2kafka
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Gelf deamon what sends events to kafka
+test_files: []