logstash-logger-yajl 0.27.0

data/lib/logstash/util/fieldreference.rb

@@ -0,0 +1,49 @@
+require "logstash/namespace"
+require "logstash/util"
+
+module LogStash::Util::FieldReference
+  def compile(str)
+    if str[0,1] != '['
+      return <<-"CODE"
+        lambda do |e, &block|
+          return block.call(e, #{str.inspect}) unless block.nil?
+          return e[#{str.inspect}]
+        end
+      CODE
+    end
+
+    code = "lambda do |e, &block|\n"
+    selectors = str.scan(/(?<=\[).+?(?=\])/)
+    selectors.each_with_index do |tok, i|
+      last = (i == selectors.count() - 1)
+      code << "   # [#{tok}]#{ last ? " (last selector)" : "" }\n"
+
+      if last
+        code << <<-"CODE"
+          return block.call(e, #{tok.inspect}) unless block.nil?
+        CODE
+      end
+
+      code << <<-"CODE"
+        if e.is_a?(Array)
+          e = e[#{tok.to_i}]
+        else
+          e = e[#{tok.inspect}]
+        end
+        return e if e.nil?
+      CODE
+
+    end
+    code << "return e\nend"
+    #puts code
+    return code
+  end # def compile
+
+  def exec(str, obj, &block)
+    @__fieldeval_cache ||= {}
+    @__fieldeval_cache[str] ||= eval(compile(str))
+    return @__fieldeval_cache[str].call(obj, &block)
+  end
+
+  extend self
+end # module LogStash::Util::FieldReference

data/logstash-logger.gemspec

@@ -0,0 +1,42 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'logstash-logger/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "logstash-logger-yajl"
+  gem.version       = LogStashLogger::VERSION
+  gem.authors       = ["David Butler"]
+  gem.email         = ["dwbutler@ucla.edu"]
+  gem.description   = %q{Ruby logger that writes directly to LogStash}
+  gem.summary       = %q{LogStash Logger for ruby}
+  gem.homepage      = "http://github.com/dwbutler/logstash-logger"
+  gem.license       = "MIT"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'yajl-ruby', '~> 1.0'
+
+  gem.add_development_dependency 'rails'
+  gem.add_development_dependency 'redis'
+  gem.add_development_dependency 'poseidon'
+  gem.add_development_dependency 'aws-sdk-kinesis'
+  gem.add_development_dependency 'aws-sdk-firehose'
+
+  if defined?(JRUBY_VERSION)
+    gem.add_development_dependency 'SyslogLogger'
+  end
+
+  gem.add_development_dependency 'rspec', '>= 3'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'wwtd'
+  gem.add_development_dependency 'appraisal'
+  gem.add_development_dependency 'rubocop'
+  gem.add_development_dependency 'insist', '1.0.0'
+  gem.add_development_dependency 'guard'
+  gem.add_development_dependency 'guard-rspec'
+end

data/samples/example.crt

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIICdTCCAd4CCQDLMpdv+B+sYDANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJV
+UzETMBEGA1UECBMKQ2FsaWZvcm5pYTERMA8GA1UEBxMIUGFzYWRlbmExDzANBgNV
+BAoTBkJ1dGxlcjEVMBMGA1UEAxMMRGF2aWQgQnV0bGVyMSAwHgYJKoZIhvcNAQkB
+FhFkd2J1dGxlckB1Y2xhLmVkdTAeFw0xNDA2MDkwNjIyMTZaFw0xNTA2MDkwNjIy
+MTZaMH8xCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMREwDwYDVQQH
+EwhQYXNhZGVuYTEPMA0GA1UEChMGQnV0bGVyMRUwEwYDVQQDEwxEYXZpZCBCdXRs
+ZXIxIDAeBgkqhkiG9w0BCQEWEWR3YnV0bGVyQHVjbGEuZWR1MIGfMA0GCSqGSIb3
+DQEBAQUAA4GNADCBiQKBgQDbBJ1t2m8RA72GZZ8XOUCsLI1EhRqZqWx0zYHpZbc4
+kVbfMeb5uyfpx1gd5QngZrsD0cvTf46F9z9Ng805Ns9jE5QBJxefl1b6iJ6f/Y9s
+W+WYQJgtNOcJahWWLeiBXqApPUllFI2ME/52+/umuSYVtSVAW7QmPx3YSLPpVjkQ
+3wIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAKiZm6yNzRXKJ9LBLL+Nep87WPlkdFmx
+YUoEtTF8XpgMNfOdzF8DlD4FmoNz/SD7uVOk+tARUoEQFVK+iqpd3/dLDVa/pslX
+a5v6AcmgfSFUHgLvKQZNvgY50jhLdqruf8PCgVnq+4L0ItguJx/viYMxF0p0fJTG
+oGX8W4ul0W5F
+-----END CERTIFICATE-----

data/samples/example.key

@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICWwIBAAKBgQDbBJ1t2m8RA72GZZ8XOUCsLI1EhRqZqWx0zYHpZbc4kVbfMeb5
+uyfpx1gd5QngZrsD0cvTf46F9z9Ng805Ns9jE5QBJxefl1b6iJ6f/Y9sW+WYQJgt
+NOcJahWWLeiBXqApPUllFI2ME/52+/umuSYVtSVAW7QmPx3YSLPpVjkQ3wIDAQAB
+AoGAYaFXBAchB3ahX22hU1rkJ1vcxTSIPQM3I4IQbRg4anDvRqMaESyKiD2iXAEj
+O/LPXs6Ai5EK2VDz2Pvt2ZlDLFX3GKMEChtIa2mpiQqIdoJS+42ibqfHrv3nlnTK
+OTeTcw+SbAAFTKFPJO94lGcBXhsYYW/LAFH08bVqp+Jhi1ECQQDzkz4O0eYH05Wb
+1bAwjNcjj+X6JKIRGgWYb1kkk3VyAFuD6qCEKBfS2t4nZs1yy83Ch4Bk9YmDHx1Q
+gdYCjPsLAkEA5jCwo51KqboxAz6m0pZwdiKlCav94TMdHMKTFcUE5fDoWrwzrIyH
+8rr7q7cl3culX7pUcPdKMl78Nw6/4JFF/QJARwNfrWxut0ttq+BSHOWC98BFWXeC
+tJ+0j+uuvqYrMJCAHeay47TYtUXQTQaA0X4vwA5HVafsbokMv+MKpPW7XwJAF713
+yjgDpkOMoIAKpndbe+OQz9GMKniiDQBIORuLqMdSv2Dfl3Ea6D6+i/QklJ5XHOtT
+oB7w6QcAzhDYWynAZQJAX0UF9alkX8CQq1rvsAG6zzh3bD+Udy7cU0Yonx/byZoz
+K6G/seM5YFTMVrnbY9V23aIhjJSxGWOB5fDNPG5p4w==
+-----END RSA PRIVATE KEY-----

data/samples/file.conf

@@ -0,0 +1,11 @@
+input {
+  file {
+    path => "/usr/local/var/logstash.log"
+    format => json
+  }
+}
+output {
+  stdout {
+    codec => json_lines
+  }
+}

data/samples/redis.conf

@@ -0,0 +1,12 @@
+input {
+  redis {
+    data_type => "list"
+    codec => json
+    key => "logstash"
+  }
+}
+output {
+  stdout {
+    codec => json_lines
+  }
+}

data/samples/ssl.conf

@@ -0,0 +1,15 @@
+input {
+  tcp {
+    port => 8443
+    ssl_enable => true
+    ssl_cert => "samples/example.crt"
+    ssl_key => "samples/example.key"
+    codec => json_lines
+  }
+}
+output {
+  stdout {
+    codec => json_lines
+  }
+}
+

data/samples/syslog.conf

@@ -0,0 +1,10 @@
+input {
+  syslog {
+    codec => json_lines
+  }
+}
+output {
+  stdout {
+    codec => json_lines
+  }
+}

data/samples/tcp.conf

@@ -0,0 +1,11 @@
+input {
+  tcp {
+    port => 5228
+    codec => json_lines
+  }
+}
+output {
+  stdout {
+    codec => json_lines
+  }
+}

data/samples/udp.conf

@@ -0,0 +1,11 @@
+input {
+  udp {
+    port => 5228
+    codec => json_lines
+  }
+}
+output {
+  stdout {
+    codec => json_lines
+  }
+}

data/samples/unix.conf

@@ -0,0 +1,11 @@
+input {
+  unix {
+    path => "/tmp/logstash"
+    codec => json_lines
+  }
+}
+output {
+  stdout {
+    codec => json_lines
+  }
+}

data/spec/configuration_spec.rb

@@ -0,0 +1,27 @@
+require 'logstash-logger'
+
+describe LogStashLogger do
+  describe "#configure" do
+    it 'auto initializes' do
+      config = LogStashLogger.configure
+      expect(config).to be_a LogStashLogger::Configuration
+      expect(LogStashLogger.configuration).to eq(config)
+    end
+
+    describe LogStashLogger::Configuration do
+      describe "#customize_event" do
+        it 'allows each LogStash::Event to be customized' do
+          config = LogStashLogger.configure do |config|
+            config.customize_event do |event|
+              event["test1"] = "response1"
+            end
+          end
+
+          event = LogStash::Event.new({})
+          config.customize_event_block.call(event)
+          expect(event["test1"]).to eq("response1")
+        end
+      end
+    end
+  end
+end

data/spec/constructor_spec.rb

@@ -0,0 +1,30 @@
+require 'logstash-logger'
+
+describe LogStashLogger do
+  describe '.new' do
+    it 'returns a Logger instance' do
+      expect(LogStashLogger.new(type: :stdout)).to be_a ::Logger
+    end
+
+    context 'type: :multi_logger' do
+      it "returns an instance of LogStashLogger::MultiLogger" do
+        expect(LogStashLogger.new(type: :multi_logger)).to be_a LogStashLogger::MultiLogger
+      end
+
+      it 'merges top level configuration into each logger' do
+        logger = LogStashLogger.new(type: :multi_logger, port: 1234, outputs: [ { type: :tcp  }, { type: :udp } ])
+        logger.loggers.each do |logger|
+          expect(logger.device.port).to eq(1234)
+        end
+      end
+    end
+
+    context 'logger_class: CustomLogger' do
+      class CustomLogger < Logger; end
+
+      it 'should create a logger of the specified class' do
+        expect(LogStashLogger.new(type: :stdout, :logger_class => CustomLogger)).to be_a CustomLogger
+      end
+    end
+  end
+end

data/spec/device/balancer_spec.rb

@@ -0,0 +1,31 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Device::Balancer do
+  include_context 'device'
+
+  # Create a Balancer writing to both STDOUT and a StringIO
+  subject { balancer_device }
+
+  describe '#write' do
+    before do
+      allow(subject.devices).to receive(:sample) { io }
+    end
+
+    it "writes to one device" do
+      expect(io).to receive(:write).once
+      expect($stdout).to_not receive(:write)
+      subject.write("log message")
+    end
+  end
+
+  describe '#flush, #close' do
+    [:flush, :close].each do |method_name|
+      it "call on all devices" do
+        subject.devices.each do |device|
+          expect(device).to receive(method_name).once
+        end
+        subject.send(method_name)
+      end
+    end
+  end
+end

data/spec/device/connectable_spec.rb

@@ -0,0 +1,74 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Device::Connectable do
+  include_context 'device'
+
+  let(:io) { double("IO") }
+
+  subject { udp_device }
+
+  describe "#reconnect" do
+    context "with active IO connection" do
+      before do
+        subject.instance_variable_set(:@io, io)
+      end
+
+      it "closes the connection" do
+        expect(io).to receive(:close).once
+        subject.reconnect
+      end
+    end
+
+    context "with no active IO connection" do
+      before do
+        subject.instance_variable_set(:@io, nil)
+      end
+
+      it "does nothing" do
+        expect(io).to_not receive(:close)
+        subject.reconnect
+      end
+    end
+  end
+
+  describe "#with_connection" do
+    context "on exception" do
+      before do
+        allow(subject).to receive(:connected?) { raise(StandardError) }
+        allow(subject).to receive(:warn)
+      end
+
+      context "with active IO connection" do
+        before do
+          subject.instance_variable_set(:@io, io)
+        end
+
+        it "closes the connection" do
+          expect(io).to receive(:close).once
+
+          expect {
+            subject.with_connection do |connection|
+              connection
+            end
+          }.to raise_error(StandardError)
+        end
+      end
+
+      context "with no active IO connection" do
+        before do
+          subject.instance_variable_set(:@io, nil)
+        end
+
+        it "does nothing" do
+          expect(io).to_not receive(:close)
+
+          expect {
+            subject.with_connection do |connection|
+              connection
+            end
+          }.to raise_error(StandardError)
+        end
+      end
+    end
+  end
+end

data/spec/device/file_spec.rb

@@ -0,0 +1,15 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Device::File do
+  include_context 'device'
+
+  it "writes to a file" do
+    expect(file_device.to_io).to be_a ::File
+  end
+
+  context "when path is not specified" do
+    it "raises an exception" do
+      expect { described_class.new }.to raise_error(ArgumentError)
+    end
+  end
+end

data/spec/device/firehose_spec.rb

@@ -0,0 +1,41 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Device::Firehose do
+  include_context 'device'
+
+  let(:client) { double("Aws::Firehose::Client") }
+
+  before(:each) do
+    allow(Aws::Firehose::Client).to receive(:new) { client }
+  end
+
+  it "writes to a Firehose stream" do
+    response = ::Aws::Firehose::Types::PutRecordBatchOutput.new
+    response.failed_put_count = 0
+    response.request_responses = []
+    expect(client).to receive(:put_record_batch) { response }
+    firehose_device.write "foo"
+
+    expect(firehose_device).to be_connected
+    firehose_device.close!
+    expect(firehose_device).not_to be_connected
+  end
+
+  it "it puts records with recoverable errors back in the buffer" do
+    failed_record = ::Aws::Firehose::Types::PutRecordBatchResponseEntry.new
+    failed_record.error_code = "InternalFailure"
+    failed_record.error_message = "InternalFailure"
+    response = ::Aws::Firehose::Types::PutRecordBatchOutput.new
+    response.failed_put_count = 1
+    response.request_responses = [failed_record]
+
+    expect(client).to receive(:put_record_batch) { response }
+    expect(firehose_device).to receive(:write).with("foo")
+
+    firehose_device.write_one "foo"
+  end
+
+  it "defaults the Firehose stream to logstash" do
+    expect(firehose_device.stream).to eq('logstash')
+  end
+end