logstash-input-log4j 1.0.0-java → 2.0.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1ab88bef150e74e2f22c7bb03d3bdcdbd7d5a553
-  data.tar.gz: 58d7c26b34abc5f90007625d6837d45d813b527e
+  metadata.gz: 774f1acdc7c060981ab35fe6c00004b554363589
+  data.tar.gz: 9b7193a589bac96a74407fe3f89acba434dbe103
 SHA512:
-  metadata.gz: 0c309848b6639a2a9f1a5e0b96f16a3919d23b3c876644d977613b8638b6dc71abeb0db2b965c905dee882450719b8f4f80087e5d221e6e4d13716d19d472e29
-  data.tar.gz: abdafa2a2547ab35ea4d9ee92340b13919393b0e38dadc4bcd4ce4d79606e20a02f9946574fc79fd6d260382f63b9176fb617fbf0e7344467c85258a690a63c5
+  metadata.gz: 6d3f400870d90165e36b6ae6ae52182ec0bffbaff788ede37ae2ab0331c68ff8b2c1a0a5dd9264818e4ea80961df25b44de149ff047b6bb94e635e974791643a
+  data.tar.gz: 14871fb9b6d28ff05e85b596be48669b6db190da5d1af8c3687bdb1668d159708f39cc053c5c8cb000f8f07704f8beaa82c3bb06adc03ccc63f456dd2c851483

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+# 1.0.1
+  - refactor test to allow unit testing of logstash event creation

data/README.md

@@ -1,15 +1,15 @@
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
@@ -83,4 +83,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/inputs/log4j.rb

@@ -14,6 +14,21 @@ require 'logstash-input-log4j_jars'
 # depending on `mode`. Depending on which `mode` is configured,
 # you need a matching SocketAppender or a SocketHubAppender
 # on the remote side.
+#
+# One event is created per received log4j LoggingEvent with the following schema:
+#
+# * `timestamp` => the number of milliseconds elapsed from 1/1/1970 until logging event was created.
+# * `path` => the name of the logger
+# * `priority` => the level of this event
+# * `logger_name` => the name of the logger
+# * `thread` => the thread name making the logging request
+# * `class` => the fully qualified class name of the caller making the logging request.
+# * `file` => the source file name and line number of the caller making the logging request in a colon-separated format "fileName:lineNumber".
+# * `method` => the method name of the caller making the logging request.
+# * `NDC` => the NDC string
+# * `stack_trace` => the multi-line stack-trace
+#
+# Also if the original log4j LoggingEvent contains MDC hash entries, they will be merged in the event as fields.
 class LogStash::Inputs::Log4j < LogStash::Inputs::Base
 
   config_name "log4j"
@@ -42,7 +57,6 @@ class LogStash::Inputs::Log4j < LogStash::Inputs::Base
 
   public
   def register
-#    LogStash::Environment.load_elasticsearch_jars!
     require "java"
     require "jruby/serialization"
 
@@ -59,41 +73,45 @@ class LogStash::Inputs::Log4j < LogStash::Inputs::Base
     @logger.info("Log4j input")
   end # def register
 
+  public
+  def create_event(log4j_obj)
+    # NOTE: log4j_obj is org.apache.log4j.spi.LoggingEvent
+    event = LogStash::Event.new("message" => log4j_obj.getRenderedMessage)
+    event["timestamp"] = log4j_obj.getTimeStamp
+    event["path"] = log4j_obj.getLoggerName
+    event["priority"] = log4j_obj.getLevel.toString
+    event["logger_name"] = log4j_obj.getLoggerName
+    event["thread"] = log4j_obj.getThreadName
+    event["class"] = log4j_obj.getLocationInformation.getClassName
+    event["file"] = log4j_obj.getLocationInformation.getFileName + ":" + log4j_obj.getLocationInformation.getLineNumber
+    event["method"] = log4j_obj.getLocationInformation.getMethodName
+    event["NDC"] = log4j_obj.getNDC if log4j_obj.getNDC
+    event["stack_trace"] = log4j_obj.getThrowableStrRep.to_a.join("\n") if log4j_obj.getThrowableInformation
+
+    # Add the MDC context properties to event
+    if log4j_obj.getProperties
+      log4j_obj.getPropertyKeySet.each do |key|
+        event[key] = log4j_obj.getProperty(key)
+      end
+    end
+    return event
+  end # def create_event
+
   private
   def handle_socket(socket, output_queue)
     begin
-      # JRubyObjectInputStream uses JRuby class path to find the class to de-serialize to
-      ois = JRubyObjectInputStream.new(java.io.BufferedInputStream.new(socket.to_inputstream))
-      loop do
-        # NOTE: log4j_obj is org.apache.log4j.spi.LoggingEvent
-        log4j_obj = ois.readObject
-        event = LogStash::Event.new("message" => log4j_obj.getRenderedMessage)
-        decorate(event)
+      ois = socket_to_inputstream(socket)
+      while !stop?
+        event = create_event(ois.readObject)
         event["host"] = socket.peer
-        event["path"] = log4j_obj.getLoggerName
-        event["priority"] = log4j_obj.getLevel.toString
-        event["logger_name"] = log4j_obj.getLoggerName
-        event["thread"] = log4j_obj.getThreadName
-        event["class"] = log4j_obj.getLocationInformation.getClassName
-        event["file"] = log4j_obj.getLocationInformation.getFileName + ":" + log4j_obj.getLocationInformation.getLineNumber
-        event["method"] = log4j_obj.getLocationInformation.getMethodName
-        event["NDC"] = log4j_obj.getNDC if log4j_obj.getNDC
-        event["stack_trace"] = log4j_obj.getThrowableStrRep.to_a.join("\n") if log4j_obj.getThrowableInformation
-
-        # Add the MDC context properties to '@fields'
-        if log4j_obj.getProperties
-          log4j_obj.getPropertyKeySet.each do |key|
-            event[key] = log4j_obj.getProperty(key)
-          end
-        end
-
+        decorate(event)
         output_queue << event
       end # loop do
     rescue => e
-      @logger.debug("Closing connection", :client => socket.peer,
+      @logger.debug? && @logger.debug("Closing connection", :client => socket.peer,
                     :exception => e)
     rescue Timeout::Error
-      @logger.debug("Closing connection after read timeout",
+      @logger.debug? && @logger.debug("Closing connection after read timeout",
                     :client => socket.peer)
     end # begin
   ensure
@@ -104,6 +122,12 @@ class LogStash::Inputs::Log4j < LogStash::Inputs::Base
     end # begin
   end
 
+  private
+  def socket_to_inputstream(socket)
+    # JRubyObjectInputStream uses JRuby class path to find the class to de-serialize to
+    JRubyObjectInputStream.new(java.io.BufferedInputStream.new(socket.to_inputstream))
+  end
+
   private
   def server?
     @mode == "server"
@@ -114,26 +138,29 @@ class LogStash::Inputs::Log4j < LogStash::Inputs::Base
     line = socket.readline
   end # def readline
 
+  public
+  # method used to stop the plugin and unblock
+  # pending blocking operatings like sockets and others.
+  def stop
+    @server_socket.close if @server_socket && !@server_socket.closed?
+  end
+
   public
   def run(output_queue)
     if server?
-      loop do
-        # Start a new thread for each connection.
+      while !stop?
         Thread.start(@server_socket.accept) do |s|
-          # TODO(sissel): put this block in its own method.
-
-          # monkeypatch a 'peer' method onto the socket.
           s.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
-          @logger.debug("Accepted connection", :client => s.peer,
+          @logger.debug? && @logger.debug("Accepted connection", :client => s.peer,
                         :server => "#{@host}:#{@port}")
           handle_socket(s, output_queue)
         end # Thread.start
       end # loop
     else
-      loop do
+      while !stop?
         client_socket = TCPSocket.new(@host, @port)
         client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
-        @logger.debug("Opened connection", :client => "#{client_socket.peer}")
+        @logger.debug? && @logger.debug("Opened connection", :client => "#{client_socket.peer}")
         handle_socket(client_socket, output_queue)
       end # loop
     end

data/logstash-input-log4j.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-log4j'
-  s.version         = '1.0.0'
+  s.version         = '2.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Read events over a TCP socket from a Log4j SocketAppender"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = `git ls-files`.split($\)
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -23,7 +23,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'logstash-codec-plain'
 
-  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency "logstash-core", "~> 2.0.0.snapshot"
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/inputs/log4j_spec.rb

@@ -1,8 +1,9 @@
 # encoding: utf-8
-
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/log4j"
 require "logstash/plugin"
 
-describe "inputs/log4j" do
+describe LogStash::Inputs::Log4j do
 
   it "should register" do
     input = LogStash::Plugin.lookup("input", "log4j").new("mode" => "client")
@@ -10,4 +11,74 @@ describe "inputs/log4j" do
     # register will try to load jars and raise if it cannot find jars or if org.apache.log4j.spi.LoggingEvent class is not present
     expect {input.register}.to_not raise_error
   end
+
+  context "when interrupting the plugin" do
+
+    it_behaves_like "an interruptible input plugin" do
+      let(:config) { { "mode" =>  "server" } }
+    end
+
+    it_behaves_like "an interruptible input plugin" do
+      let(:config) { { "mode" =>  "client" } }
+      let(:socket) { double("socket") }
+      let(:ois)    { double("ois") }
+      before(:each) do
+        allow(socket).to receive(:peer).and_return("localhost")
+        allow(ois).to receive(:readObject).and_return({})
+        allow(TCPSocket).to receive(:new).and_return(socket)
+        expect(subject).to receive(:socket_to_inputstream).with(socket).and_return(ois)
+        expect(subject).to receive(:create_event).and_return(LogStash::Event.new).at_least(:once)
+      end
+    end
+  end
+
+  context "reading general information from a org.apache.log4j.spi.LoggingEvent" do
+    let (:input) { LogStash::Plugin.lookup("input", "log4j").new("mode" => "client") }
+    let (:log_obj) {
+      org.apache.log4j.spi.LoggingEvent.new(
+        "org.apache.log4j.Logger",
+        org.apache.log4j.Logger.getLogger("org.apache.log4j.LayoutTest"),
+        1426366971,
+        org.apache.log4j.Level::INFO,
+        "Hello, World",
+        nil
+      )
+    }
+
+    let (:log_obj_with_stacktrace) {
+      org.apache.log4j.spi.LoggingEvent.new(
+        "org.apache.log4j.Logger",
+        org.apache.log4j.Logger.getLogger("org.apache.log4j.LayoutTest"),
+        1426366971,
+        org.apache.log4j.Level::INFO,
+        "Hello, World",
+        java.lang.IllegalStateException.new()
+      )
+    }
+
+    it "creates event with general information" do
+      subject = input.create_event(log_obj)
+      expect(subject["timestamp"]).to eq(1426366971)
+      expect(subject["path"]).to eq("org.apache.log4j.LayoutTest")
+      expect(subject["priority"]).to eq("INFO")
+      expect(subject["logger_name"]).to eq("org.apache.log4j.LayoutTest")
+      expect(subject["thread"]).to eq("main")
+      expect(subject["message"]).to eq("Hello, World")
+      # checks locationInformation is collected, but testing exact values is not meaningful in jruby
+      expect(subject["class"]).not_to be_empty
+      expect(subject["file"]).not_to be_empty
+      expect(subject["method"]).not_to be_empty
+    end
+
+    it "creates event without stacktrace" do
+      subject = input.create_event(log_obj)
+      expect(subject["stack_trace"]).to be_nil
+    end
+
+    it "creates event with stacktrace" do
+      subject = input.create_event(log_obj_with_stacktrace)
+      #checks stack_trace is collected, exact value is too monstruous
+      expect(subject["stack_trace"]).not_to be_empty
+    end
+  end
 end

metadata

@@ -1,77 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-log4j
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-24 00:00:00.000000000 Z
+date: 2015-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-plain
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-core
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
-      - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
-      - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.0.snapshot
+  name: logstash-core
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.0.snapshot
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
 - CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
 - README.md
-- Rakefile
 - lib/logstash-input-log4j_jars.rb
 - lib/logstash/inputs/log4j.rb
 - logstash-input-log4j.gemspec
@@ -99,7 +91,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Read events over a TCP socket from a Log4j SocketAppender

data/.gitignore

@@ -1,3 +0,0 @@
-*.gem
-Gemfile.lock
-.bundle

data/Rakefile

@@ -1,7 +0,0 @@
-@files=[]
-
-task :default do
-  system("rake -T")
-end
-
-require "logstash/devutils/rake"