logstash-logger 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84c02ef7b86b7f3bbd3342b9978e6b172f9705ee
-  data.tar.gz: fb9fe50daa68b2c7f441b21816500baeb9c7e081
+  metadata.gz: 03eb27ccbc6770bb3348ffcedd6fdc46ba789731
+  data.tar.gz: 3d4d59a4c5a2b5ede515b44595c97fe73949d89d
 SHA512:
-  metadata.gz: e759b59bb77b80ab16c15c99a22e33d33d81af4e151eaacd722fb6cf69648161f4e0053a5c54f97938a71c08dc63482e4ac5849fbf43854ca23be1520f8de8bd
-  data.tar.gz: 0da0fe1fd184fc8ddbe7ebb9a16c103496a9b8d9add08a1428ba707bcb794c87429fabce1a896e83d89b92452f467ccc9fd7b93fe7a6cfccbb76298a6ab73319
+  metadata.gz: 454217b66be5ceeaa97f904b0218d1c24651cb865c7cafb1bc69702025b90d8fe8ca3a5337b77296aea8dda1c2e78bce669348708f950195e30706436f8a7c0e
+  data.tar.gz: 3aa6228a59fbd9b540e5dc745caebebb2bfc11d65d16ad7265afd50a10710305e2cfc46ce1802118bbaf3fd8af25286b79c220efc93c7439c06924bb9a514735

data/.travis.yml

@@ -2,11 +2,8 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-  - 2.1.0
-  - 2.1.1
-  - 2.1.2
-  - 2.1.3
-  - 2.2.0
+  - 2.1
+  - 2.2
   - jruby-19mode
   - rbx-2
 gemfile:
@@ -14,3 +11,6 @@ gemfile:
   - gemfiles/rails_4.0.gemfile
   - gemfiles/rails_4.1.gemfile
   - gemfiles/rails_4.2.gemfile
+matrix:
+  allow_failures:
+    - rbx-2

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.10.0
+- Support for logging to Kafka.
+Fixes [#37](https://github.com/dwbutler/logstash-logger/issues/37).
+Thanks [Felix Bechstein](https://github.com/felixb)!
+
 ## 0.9.0
 - Support for customizing the fields on all logged messages via configuration.
 Fixes [#32](https://github.com/dwbutler/logstash-logger/pull/32).

data/README.md

@@ -44,6 +44,7 @@ tcp_logger = LogStashLogger.new(type: :tcp, host: 'localhost', port: 5229)
 file_logger = LogStashLogger.new(type: :file, path: 'log/development.log', sync: true)
 unix_logger = LogStashLogger.new(type: :unix, path: '/tmp/sock')
 redis_logger = LogStashLogger.new(type: :redis)
+kafka_logger = LogStashLogger.new(type: :kafka)
 stdout_logger = LogStashLogger.new(type: :stdout)
 stderr_logger = LogStashLogger.new(type: :stderr)
 io_logger = LogStashLogger.new(type: :io, io: io)
@@ -81,6 +82,7 @@ tcp://localhost:5229
 unix:///tmp/socket
 file:///path/to/file
 redis://localhost:6379
+kafka://localhost:9092
 stdout:/
 stderr:/
 ```
@@ -253,6 +255,10 @@ config.logstash.path = '/tmp/sock'
 
 #### Redis
 
+Add the redis gem to your Gemfile:
+
+    gem 'redis'
+
 ```ruby
 # Required
 config.logstash.type = :redis
@@ -271,6 +277,30 @@ config.logstash.host = 'localhost'
 config.logstash.port = 6379
 ```
 
+#### Kafka
+
+Add the poseidon gem to your Gemfile:
+
+    gem 'poseidon'
+
+```ruby
+# Required
+config.logstash.type = :kafka
+
+# Optional, will default to the 'logstash' topic
+config.logstash.path = 'logstash'
+
+# Optional, will default to the 'logstash-logger' producer
+config.logstash.producer = 'logstash-logger'
+
+# Optional, will default to localhost:9092 host/port
+config.logstash.hosts = ['localhost:9092']
+
+# Optional, will default to 1s backoff
+config.logstash.backoff = 1
+
+```
+
 #### File
 
 ```ruby
@@ -347,6 +377,11 @@ Heroku recommends installing the [rails_12factor](https://github.com/heroku/rail
 Unfortunately, this overrides LogStashLogger, preventing logs from being sent to their configured destination. The solution
 is to remove `rails_12factor` from your Gemfile.
 
+### Logging eventually stops in production
+This is most likely not a problem with LogStashLogger, but rather a different gem changing the log level of `Rails.logger`.
+This is especially likely if you're using a threaded server such as Puma, since gems often change the log level of
+`Rails.logger` in a non thread-safe way. See [#17](https://github.com/dwbutler/logstash-logger/issues/17) for more information.
+
 ## Breaking changes
 
 ### Version 0.5+
@@ -379,6 +414,7 @@ logger = LogStashLogger.new('localhost', 5228, :tcp)
 * [Jan Schulte](https://github.com/schultyy)
 * [Kurt Preston](https://github.com/KurtPreston)
 * [Chris Blatchley](https://github.com/chrisblatchley)
+* [Felix Bechstein](https://github.com/felixb)
 
 ## Contributing
 

data/lib/logstash-logger/device.rb

@@ -11,6 +11,7 @@ module LogStashLogger
     autoload :TCP, 'logstash-logger/device/tcp'
     autoload :Unix, 'logstash-logger/device/unix'
     autoload :Redis, 'logstash-logger/device/redis'
+    autoload :Kafka, 'logstash-logger/device/kafka'
     autoload :File, 'logstash-logger/device/file'
     autoload :IO, 'logstash-logger/device/io'
     autoload :Stdout, 'logstash-logger/device/stdout'
@@ -56,6 +57,7 @@ module LogStashLogger
         when :unix then Unix
         when :file then File
         when :redis then Redis
+        when :kafka then Kafka
         when :io then IO
         when :stdout then Stdout
         when :stderr then Stderr

data/lib/logstash-logger/device/kafka.rb

@@ -0,0 +1,81 @@
+require 'poseidon'
+require 'stud/buffer'
+
+module LogStashLogger
+  module Device
+    class Kafka < Connectable
+      include Stud::Buffer
+
+      DEFAULT_HOST = 'localhost'
+      DEFAULT_PORT = 9092
+      DEFAULT_TOPIC = 'logstash'
+      DEFAULT_PRODUCER = 'logstash-logger'
+      DEFAULT_BACKOFF = 1
+
+      attr_accessor :hosts, :topic, :producer, :backoff
+
+      def initialize(opts)
+        super
+        host = opts[:host] || DEFAULT_HOST
+        port = opts[:port] || DEFAULT_PORT
+        @hosts = opts[:hosts] || host.split(',').map { |h| "#{h}:#{port}" }
+        @topic = opts[:path] || DEFAULT_TOPIC
+        @producer = opts[:producer] || DEFAULT_PRODUCER
+        @backoff = opts[:backoff] || DEFAULT_BACKOFF
+
+        @batch_events = opts.fetch(:batch_events, 50)
+        @batch_timeout = opts.fetch(:batch_timeout, 5)
+
+        buffer_initialize max_items: @batch_events, max_interval: @batch_timeout
+      end
+
+      def connect
+        @io = ::Poseidon::Producer.new(@hosts, @producer)
+      end
+
+      def reconnect
+        @io.close
+        connect
+      end
+
+      def with_connection
+        connect unless @io
+        yield
+      rescue ::Poseidon::Errors::ChecksumError, Poseidon::Errors::UnableToFetchMetadata => e
+        warn "#{self.class} - #{e.class} -> reconnect/retry"
+        sleep backoff if backoff
+        reconnect
+        retry
+      rescue => e
+        warn "#{self.class} - #{e.class} - #{e.message} -> giving up"
+        @io = nil
+      end
+
+      def write(message)
+        buffer_receive Poseidon::MessageToSend.new(@topic, message)
+        buffer_flush(force: true) if @sync
+      end
+
+      def close
+        buffer_flush(final: true)
+        @io && @io.close
+      rescue => e
+        warn "#{self.class} - #{e.class} - #{e.message}"
+      ensure
+        @io = nil
+      end
+
+      def flush(*args)
+        if args.empty?
+          buffer_flush
+        else
+          messages = *args.first
+          with_connection do
+            @io.send_messages messages
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/logstash-logger/version.rb

@@ -1,3 +1,3 @@
 module LogStashLogger
-  VERSION = "0.9.0"
+  VERSION = "0.10.0"
 end

data/logstash-logger.gemspec

@@ -17,12 +17,13 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  
+
   gem.add_runtime_dependency 'logstash-event', '~> 1.2'
   gem.add_runtime_dependency 'stud'
 
   gem.add_development_dependency 'rails'
   gem.add_development_dependency 'redis'
+  gem.add_development_dependency 'poseidon'
   gem.add_development_dependency 'rspec', '>= 3'
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'pry'

data/spec/device/kafka_spec.rb

@@ -0,0 +1,32 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Device::Kafka do
+  include_context 'device'
+
+  let(:producer) { double("Poseidon::Producer") }
+
+  before(:each) do
+    allow(Poseidon::Producer).to receive(:new) { producer }
+  end
+
+  it "writes to a Kafka topic" do
+    expect(producer).to receive(:send_messages)
+    kafka_device.write "foo"
+  end
+
+  it "defaults the Kafka hosts to ['localhost:9092']" do
+    expect(kafka_device.hosts).to eq(['localhost:9092'])
+  end
+
+  it "defaults the Kafka topic to 'logstash'" do
+    expect(kafka_device.topic).to eq('logstash')
+  end
+
+  it "defaults the Kafka producer to 'logstash-logger'" do
+    expect(kafka_device.producer).to eq('logstash-logger')
+  end
+
+  it "defaults the Kafka backoff to 1" do
+    expect(kafka_device.backoff).to eq(1)
+  end
+end

data/spec/device_spec.rb

@@ -72,6 +72,11 @@ describe LogStashLogger::Device do
       it { is_expected.to be_a LogStashLogger::Device::Redis }
     end
 
+    context "when URI config is kafka" do
+      let(:uri_config) { kafka_uri_config }
+      it { is_expected.to be_a LogStashLogger::Device::Kafka }
+    end
+
     context "when URI config is stdout" do
       let(:uri_config) { stdout_uri_config }
       it { is_expected.to be_a LogStashLogger::Device::Stdout }

data/spec/spec_helper.rb

@@ -52,6 +52,7 @@ RSpec.shared_context 'device' do
   let(:io_device) { LogStashLogger::Device.new(type: :io, io: io)}
 
   let(:redis_device) { LogStashLogger::Device.new(type: :redis, sync: true) }
+  let(:kafka_device) { LogStashLogger::Device.new(type: :kafka, sync: true) }
   let(:multi_delegator_device) { LogStashLogger::Device.new([{type: :stdout}, {type: :io, io: io}]) }
 
   let(:udp_uri) { "udp://localhost:5228" }
@@ -59,6 +60,7 @@ RSpec.shared_context 'device' do
   let(:unix_uri) { "unix:///some/path/to/socket" }
   let(:file_uri) { "file://#{file.path}" }
   let(:redis_uri) { "redis://localhost:6379" }
+  let(:kafka_uri) { "kafka://localhost:9092" }
   let(:stdout_uri) { "stdout://localhost" }
   let(:stderr_uri) { "stderr://localhost" }
 
@@ -68,6 +70,7 @@ RSpec.shared_context 'device' do
   let(:unix_uri_config)    { {uri: unix_uri} }
   let(:file_uri_config)    { {uri: file_uri} }
   let(:redis_uri_config)   { {uri: redis_uri} }
+  let(:kafka_uri_config)   { {uri: kafka_uri} }
   let(:stdout_uri_config)  { {uri: stdout_uri} }
   let(:stderr_uri_config)  { {uri: stderr_uri} }
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-logger
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - David Butler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-12 00:00:00.000000000 Z
+date: 2015-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-event
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: poseidon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -163,6 +177,7 @@ files:
 - lib/logstash-logger/device/connectable.rb
 - lib/logstash-logger/device/file.rb
 - lib/logstash-logger/device/io.rb
+- lib/logstash-logger/device/kafka.rb
 - lib/logstash-logger/device/multi_delegator.rb
 - lib/logstash-logger/device/redis.rb
 - lib/logstash-logger/device/socket.rb
@@ -188,6 +203,7 @@ files:
 - spec/configuration_spec.rb
 - spec/device/file_spec.rb
 - spec/device/io_spec.rb
+- spec/device/kafka_spec.rb
 - spec/device/multi_delegator_spec.rb
 - spec/device/redis_spec.rb
 - spec/device/socket_spec.rb
@@ -229,6 +245,7 @@ test_files:
 - spec/configuration_spec.rb
 - spec/device/file_spec.rb
 - spec/device/io_spec.rb
+- spec/device/kafka_spec.rb
 - spec/device/multi_delegator_spec.rb
 - spec/device/redis_spec.rb
 - spec/device/socket_spec.rb