fluent-plugin-websphere-iib 1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 23a31f4344dea2d0a5ad8c145b6c9ba5358d0706
+  data.tar.gz: 95376f4b90e6408e2ef9d51f54f6818c88231a73
+SHA512:
+  metadata.gz: f94732be82fadf093b4ca71275adabff2d322922f3d78d94aadbd473eb4a3d43527f3b915c8865df8eca7fe7e6e21ebc783a3cf54864eb19152523a83db6d5a2
+  data.tar.gz: e264f933eb08a3ef4c89324dadbd325edc1dfc5bb20d498e8c76b42a39d2c5116d367edf7c3d4705fb05e35d80da54a476dc9b4dc9d20fdea1d16c2947b6fa32

data/README.md

@@ -0,0 +1,39 @@
+fluent-plugin-websphere-iib
+===========================
+
+fluentd plugin for parsing IBM websphere IIB logs inthe syslog /var/log/messages
+
+#Available format Plugins:
+* websphere_iib_syslog: format logs in /var/log/messages for IBM websphere IIB
+* websphere_iib_stdout: format logs in stdout for IBM websphere IIB
+
+#Plugin Settings:
+Both plugins have the same configuration options:
+
+* remote_syslog: fqdn or ip of the remote syslog instance
+* port: the port, where the remote syslog instance is listening
+* hostname: hostname to be set for syslog messages
+* remove_tag_prefix: remove tag prefix for tag placeholder. 
+* tag_key: use the field specified in tag_key from record to set the syslog key
+* facility: Syslog log facility
+* severity: Syslog log severity
+* use_record: Use severity and facility from record if available
+* payload_key: Use the field specified in payload_key from record to set payload
+
+#Configuration example:
+```
+<match site.*>
+  type syslog_buffered
+  remote_syslog your.syslog.host
+  port 25
+  hostname ${hostname}
+  facility local0
+  severity debug
+</match>
+```
+
+
+Contributors:
+
+* Victor Guillen
+* [superguillen](http://github.com/superguillen)

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/lib/fluentd/plugin/parser_multiline_websphere_iib_stdout.rb

@@ -0,0 +1,206 @@
+module Fluent
+  class TextParser
+     class WebsphereSysout < Parser
+          Plugin.register_parser("multiline_websphere_iib", self)
+
+          config_param :time_format, :string, :default =>'%m/%d/%y %H:%M:%S:%L %Z'
+          config_param :output_time_format, :string, :default =>'%Y-%m-%dT%H:%M:%S.%L%z'
+          config_param :format_firstline, :string, :default =>'/\[\d{1,2}\/\d{1,2}\/\d{2,4} \d{1,2}:\d{1,2}:\d{1,2}:\d{1,3}\s.{3}\]/'
+          config_param :format1, :string, :default =>'/\[(?<timestamp>\d{1,2}\/\d{1,2}\/\d{2,4} \d{1,2}:\d{1,2}:\d{1,2}:\d{1,3}\s.{3})\]\s+(?<treadid>\S+)\s+(?<msgshortname>\S+)\s+(?<eventype>\S+)\s+(?<msgid>\S+)\s*(?<message>.*)/'
+          config_param :ambiente, :string, :default =>'unknow'
+          config_param :producto, :string, :default =>'unknow'
+          config_param :tipolog, :string, :default =>'websphere.integration_bus'
+          config_param :integration_node, :string, :default =>'unknow'
+          config_param :integration_server, :string, :default =>'unknow'
+
+          REGEXP_PMRM0003I = '^.*type=(?<type>.+)\s+detail=(?<detail>.+)\s+elapsed=(?<elapsed>\S+)$'
+          FORMAT_MAX_NUM = 20
+
+          def initialize
+            super
+            @mutex = Mutex.new
+            @regexp_pmrm0003i = Regexp.new(REGEXP_PMRM0003I)
+          end
+
+          def configure(conf)
+            super
+
+            conf["format1"] ||= @format1
+
+            $log.info "format1:  -> "+@format1
+
+            formats = parse_formats(conf).compact.map { |f| f[1..-2] }.join
+            $log.info "formats:  -> "+formats
+            begin
+              @regex = Regexp.new(formats, Regexp::MULTILINE)
+              if @regex.named_captures.empty?
+                raise "No named captures"
+              end
+              @parser = RegexpParser.new(@regex, conf)
+            rescue => e
+              raise ConfigError, "Invalid regexp '#{formats}': #{e}"
+            end
+
+            if @format_firstline
+              check_format_regexp(@format_firstline, 'format_firstline')
+              @firstline_regex = Regexp.new(@format_firstline[1..-2])
+            end
+          end
+
+          def parse(text, &block)
+            m = @regex.match(text)
+
+            unless m
+              if block_given?
+                yield nil, nil
+                return
+              else
+                return nil, nil
+              end
+            end
+
+            record = {}
+            record["tipolog"] = @tipolog
+            record["integration_node"] = @integration_node
+            record["integration_server"] = @integration_server
+            record["producto"] = @producto
+            record["ambiente"] = @ambiente
+            record["eventype"] = "INFO"
+            record["severity"] = "LOW"
+            record["severity_level"] = 5
+            record["hostname"] = Socket.gethostname 
+
+            m.names.each { |name|
+                if value = m[name]
+                  case name
+                  when "timestamp"
+                    #Se calcula timestmap adicionando timezone
+                    timestamp = @mutex.synchronize { DateTime.strptime(value,@time_format).strftime(@output_time_format) }
+                    time = @mutex.synchronize { DateTime.strptime(value,@time_format).to_time.to_i }
+                    #$log.info "timestamp: #{value+@timezone_offset}"
+                    record[name] = timestamp 
+                  when "eventype"
+                     record[name] = value
+                     case record[name]
+                     when "I"
+                        record["eventype"] = "INFO"
+                        record["severity"] = "LOW"
+                        record["severity_level"] = 5
+                     when "D"
+                        record["eventype"] = "DETAIL"
+                        record["severity"] = "LOW"
+                        record["severity_level"] = 6
+                     when "E"
+                        record["eventype"] = "ERROR"
+                        record["severity"] ="HIGH"
+                        record["severity_level"] = 5
+                     when "W"
+                        record["eventype"] = "WARNING"
+                        record["severity"] = "MEDIUM"
+                        record["severity_level"] = 5
+                     when "F"
+                        record["eventype"] = "FATAL"
+                        record["severity"] = "HIGH"
+                        record["severity_level"] = 4
+                     when "C"
+                        record["eventype"] = "CONFIGURATION"
+                        record["severity"] = "MEDIUM"
+                        record["severity_level"] = 5
+                     when "O"
+                        record["eventype"] = "SYSTEM_OUTPUT"
+                        record["severity"] = "LOW"
+                        record["severity_level"] = 5
+                     when "R"
+                        record["eventype"] = "SYSTEM_ERROR"
+                        record["severity"] = "LOW"
+                        record["severity_level"] = 5
+                     when "Z"
+                        record["eventype"] = "NOT_RECOGNIZED"
+                        record["severity"] = "LOW"
+                        record["severity_level"] = 5
+                     end
+
+                  when "message"
+                     case record["msgid"]
+                     when "PMRM0003I:"
+                        msg = value
+                        #Se extrae datos de request metrics
+                        if requestMetrics = @regexp_pmrm0003i.match(msg)
+                           record["type"] = requestMetrics["type"]
+                           record["detail"] = requestMetrics["detail"]
+                           record["elapsed"] = requestMetrics["elapsed"]
+                           record["tipolog"] = 'requestmetrics.'+record["tipolog"]
+                           record["mesage"].delete
+                        end
+                     else
+                       record[name] = value
+                     end
+                  else
+                    record[name] = value
+                  end
+                end
+              }
+
+            if @estimate_current_event
+              time ||= Engine.now
+            end
+
+            if block_given?
+              yield time, record
+            else
+              return time, record
+            end
+
+          end
+
+          def has_firstline?
+            !!@format_firstline
+          end
+
+          def firstline?(text)
+            @firstline_regex.match(text)
+          end
+
+          private
+
+          def parse_formats(conf)
+            check_format_range(conf)
+
+            prev_format = nil
+            (1..FORMAT_MAX_NUM).map { |i|
+              format = conf["format#{i}"]
+              if (i > 1) && prev_format.nil? && !format.nil?
+                raise ConfigError, "Jump of format index found. format#{i - 1} is missing."
+              end
+              prev_format = format
+              next if format.nil?
+
+              check_format_regexp(format, "format#{i}")
+              format
+            }
+          end
+
+          def check_format_range(conf)
+            invalid_formats = conf.keys.select { |k|
+              m = k.match(/^format(\d+)$/)
+              m ? !((1..FORMAT_MAX_NUM).include?(m[1].to_i)) : false
+            }
+            unless invalid_formats.empty?
+              raise ConfigError, "Invalid formatN found. N should be 1 - #{FORMAT_MAX_NUM}: " + invalid_formats.join(",")
+            end
+          end
+
+          def check_format_regexp(format, key)
+            if format[0] == '/' && format[-1] == '/'
+              begin
+                Regexp.new(format[1..-2], Regexp::MULTILINE)
+              rescue => e
+                raise ConfigError, "Invalid regexp in #{key}: #{e}"
+              end
+            else
+              raise ConfigError, "format should be Regexp, need //, in #{key}: '#{format}'"
+            end
+          end
+        end
+    end
+end

data/lib/fluentd/plugin/parser_websphere_iib_syslog.rb

@@ -0,0 +1,130 @@
+module Fluent
+  class TextParser
+    class SyslogParserCustom < Parser
+      Plugin.register_parser("syslogcustom", self)
+      # From existence TextParser pattern
+      REGEXP = '^(?<timestamp>[^ ]*\s*[^ ]* [^ ]*) (?<hostname>[^ ]*) (?<identificador>[a-zA-Z-1-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$'
+      # From in_syslog default pattern
+      REGEXP_WITH_PRI = '^\<(?<priority>[0-9]+)\>(?<timestamp>[^ ]* {1,2}[^ ]* [^ ]*) (?<hostname>[^ ]*) (?<identificador>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$'
+      #Expresion regular cuando esta presente el log del IIB
+      IIB_REGEXP = '^(?<product_name>[^\(]+)\((?<nodo>[^\(]+)\) \[Thread(?<thread>[^\[]+)\] \(Msg (?<msg>[^\(]+)\) (?<msgid>[^\:]+)\: (?<message>.*)$'
+
+      config_param :time_format, :string, :default => "%b %d %H:%M:%S"
+      #Incluye timezone (se agrega a la fecha de entrada)
+      config_param :output_time_format, :string, :default => "%Y-%m-%dT%H:%M:%S.%L%z"
+      config_param :with_priority, :bool, :default => false
+      config_param :keep_time_key, :bool, :default => true
+      config_param :ambiente, :string, :default => nil
+
+      def initialize
+        super
+        @mutex = Mutex.new
+      end
+
+      def configure(conf)
+        super
+
+        require 'active_support/time'
+
+        @timezone_offset = Time.now.formatted_offset
+        @regexp = @with_priority ? Regexp.new(REGEXP_WITH_PRI) : Regexp.new(REGEXP)
+        @iib_regexp = Regexp.new(IIB_REGEXP)
+        @time_parser = TextParser::TimeParser.new(@time_format)
+      end
+
+      def patterns
+        {'format' => @regexp, 'time_format' => @time_format, 'subformat' => @iib_regexp}
+      end
+
+      def parse(text)
+        m = @regexp.match(text)
+        #n = @iib_regexp.match(text)
+        unless m
+          if block_given?
+            yield nil, nil
+            return
+          else
+            return nil, nil
+          end
+        end
+
+        time = nil
+        msg = nil
+        
+        record = {}
+	record["eventype"] = "INFO"
+        record["severity"] = "LOW"
+        record["severity_level"] = 4
+	record["hostname"] = Socket.gethostname 
+        m.names.each { |name|
+          if value = m[name]
+            #$log.info ">>>>>>: #{name}"
+            case name
+            when "priority"
+              record['priority'] = value.to_i
+            when "message"
+                case record["identificador"]
+		when "IIB"
+                #$log.info "message:  -> #{value}"
+                msg = value
+                n = @iib_regexp.match(msg)
+                n.names.each { |name| 
+                  if msg = n[name]
+                    #$log.info ">>>>>>: #{name}"
+                    record[name] = msg
+                  end
+                }
+
+                if record.has_key?("nodo")
+                   record["integration_node"] = record["nodo"].split(".")[0]
+                   record["integration_server"] = record["nodo"].split(".")[1]
+                   record.delete("nodo")
+                end
+
+		record["producto"] = record["identificador"]
+		record["ambiente"] = @ambiente
+		record.delete("identificador")
+		record["msgshortname"] = record["msgid"]
+		record["eventype"] =  record["msgid"][-1]
+		case record["eventype"]
+		when "E"
+		   record["eventype"] = "ERROR"
+		   record["severity"] ="HIGH"
+		   record["severity_level"] = 5
+		when "W"
+		   record["eventype"] = "WARNING"
+		   record["severity"] = "MEDIUM"
+		   record["severity_level"] = 5
+		else
+		   record["eventype"] = "INFO"
+		   record["severity"] = "LOW"
+		   record["severity_level"] = 5
+		end		
+		else
+		record[name] = value
+		end
+            when "timestamp"
+              time = @mutex.synchronize { @time_parser.parse(value.gsub(/ +/, ' ')) }
+              #Se calcula timestmap adicionando timezone
+              timestamp = @mutex.synchronize { DateTime.strptime(value+@timezone_offset,@time_format+'%z').strftime(@output_time_format) }
+              #$log.info "timestamp: #{value+@timezone_offset}"
+              record[name] = timestamp 
+            else
+              record[name] = value
+            end
+          end
+        }
+
+        if @estimate_current_event
+          time ||= Engine.now
+        end
+
+        if block_given?
+          yield time, record
+        else
+          return time, record
+        end
+      end
+    end
+  end #textParser
+end

data/test/helper.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluentd/plugin/out_syslog'
+require 'fluentd/plugin/out_syslog_buffered'
+
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_syslog.rb

@@ -0,0 +1,138 @@
+require 'helper'
+
+class SyslogOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    remote_syslog 127.0.0.1
+    port    25
+    hostname      testhost
+    remove_tag_prefix        test
+    severity  debug
+    facility user
+    payload_key message
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::SyslogOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+      hostname      testhost
+      remove_tag_prefix        test
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remote_syslog 127.0.0.1
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remote_syslog 127.0.0.1
+        port    639
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remote_syslog 127.0.0.1
+        port    25
+        hostname      deathstar
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+       remote_syslog 127.0.0.1
+        port    25
+        hostname      testhost
+        remove_tag_prefix        test123
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remote_syslog 127.0.0.1
+        port    25
+        hostname      testhost
+        remove_tag_prefix        test
+        tag_key   tagtag
+        severity  debug
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remote_syslog 127.0.0.1
+        port    25
+        hostname      testhost
+        remove_tag_prefix        test
+        tag_key   tagtag
+        severity  debug
+        facility user
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remote_syslog 127.0.0.1
+        port    25
+        hostname      testhost
+        remove_tag_prefix        test
+        tag_key   tagtag
+        severity  debug
+        facility user
+        payload_key message
+      ]
+    }
+    d = create_driver %[
+        remote_syslog 127.0.0.1
+        port    25
+        hostname      testhost
+        remove_tag_prefix        test
+        tag_key   tagtag
+        severity  debug
+        facility user
+        payload_key message
+    ]
+    assert_equal 25, d.instance.port
+    assert_equal "127.0.0.1", d.instance.remote_syslog
+    assert_equal "testhost", d.instance.hostname
+    assert_equal Regexp.new('^' + Regexp.escape("test")), d.instance.remove_tag_prefix
+    assert_equal "tagtag", d.instance.tag_key
+    assert_equal "debug", d.instance.severity
+    assert_equal "user", d.instance.facility
+    assert_equal "message", d.instance.payload_key
+
+  end
+  def test_emit
+    d1 = create_driver(CONFIG, 'test.in')
+    d1.run do
+      d1.emit({'message' => 'asd asd'})
+      d1.emit({'message' => 'dsa xasd'})
+      d1.emit({'message' => 'ddd ddddd'})
+      d1.emit({'message' => '7sssss8 ssssdasd'})
+      d1.emit({'message' => 'aaassddffg asdasdasfasf'})
+    end
+    assert_equal 0, d1.emits.size
+
+  end
+
+  def test_emit_with_time_and_without_time
+    d1 = create_driver(CONFIG, 'test.in')
+    d1.run do
+      d1.emit({'message' => 'asd asd', 'time' => '2007-01-31 12:22:26'})
+      d1.emit({'message' => 'dsa xasd'})
+      d1.emit({'message' => 'ddd ddddd', 'time' => '2007-03-01 12:22:26'})
+      d1.emit({'message' => '7sssss8 ssssdasd', 'time' => '2011-03-01 12:22:26'})
+      d1.emit({'message' => 'aaassddffg asdasdasfasf', 'time' => '2016-03-01 12:22:26'})
+    end
+    assert_equal 0, d1.emits.size
+
+  end
+
+
+end

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-websphere-iib
+version: !ruby/object:Gem::Version
+  version: '1.0'
+platform: ruby
+authors:
+- superguillen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-09-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.45
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.45
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Input plugin for websphere Integration Bus syslog
+email: superguillen.public@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- lib/fluentd/plugin/parser_multiline_websphere_iib_stdout.rb
+- lib/fluentd/plugin/parser_websphere_iib_syslog.rb
+- test/helper.rb
+- test/plugin/test_out_syslog.rb
+homepage: https://github.com/superguillen/fluent-plugin-websphere-iib
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Input plugin for websphere Integration Bus syslog
+test_files: []