fluent-plugin-parser_cef 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50a5b3a6d8a81e4c38c569ba14bf9d8bbdcb8d94
-  data.tar.gz: afaee81b11a943f046d75e2fb2af1c2a484903e3
+  metadata.gz: 0e2eedd443b1b7fec47ddc045ee1f7506de43022
+  data.tar.gz: ccbc339b8394d269ab365f4b4e8fc22d49a7d57b
 SHA512:
-  metadata.gz: 654f5597b82a2c2d41d7646e54fd43bfc0593e93f16f51732ca840eb2e9b6027b1cb69814a08c5e399feb99209f23bd601f00adae96ac9f67aa892f2ca2edfb4
-  data.tar.gz: 68d5646c3f7b1f021357483d78df60d7357acedcfab0766c5dc7244c4a9d992fd2f640f0cfacbeb4737f6a9dcaf233feeb9a7ebc3b9af8245c9b2e171718d838
+  metadata.gz: 4e2b5fa97f9308e265b5baba0ddeb0abd211384adda34631d4c9b7793c5411b9f0edf38e7288b78ebea281a4baf577a9682a82137c05ee5061b4dcf08589d264
+  data.tar.gz: 3e44f6c3731f048dbf4012678dc123c5b1ca69d64e6353e60c53b5684070072c2c507b9bbae35a4d09d5f4cf006bcf1efd13479c963a6e436992b9a0e0d9c191

data/.codeclimate.yml

@@ -1,4 +1,3 @@
----
 engines:
   duplication:
     enabled: true
@@ -12,6 +11,11 @@ engines:
     enabled: true
   rubocop:
     enabled: true
+    checks:
+      Rubocop/Metrics/MethodLength:
+        enabled: false
+      Rubocop/Metrics/CyclomaticComplexity:
+        enabled: false
 ratings:
   paths:
   - "**.inc"
@@ -23,11 +27,3 @@ ratings:
   - "**.rb"
 exclude_paths:
 - spec/
-engines:
-  rubocop:
-    enabled: true
-    checks:
-      Rubocop/Metrics/MethodLength:
-        enabled: false
-      Rubocop/Metrics/CyclomaticComplexity:
-        enabled: false

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/Gemfile

@@ -1,2 +1,4 @@
 source 'https://rubygems.org'
 gemspec
+gem 'coveralls', require: false
+gem 'simplecov'

data/README.md

@@ -3,6 +3,7 @@
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-parser_cef.svg)](https://badge.fury.io/rb/fluent-plugin-parser_cef)
 [![Build Status](https://travis-ci.org/lunardial/fluent-plugin-parser_cef.svg?branch=master)](https://travis-ci.org/lunardial/fluent-plugin-parser_cef)
 [![Code Climate](https://codeclimate.com/github/lunardial/fluent-plugin-parser_cef/badges/gpa.svg)](https://codeclimate.com/github/lunardial/fluent-plugin-parser_cef)
+[![Coverage Status](https://coveralls.io/repos/github/lunardial/fluent-plugin-parser_cef/badge.svg?branch=master)](https://coveralls.io/github/lunardial/fluent-plugin-parser_cef?branch=master)
 [![downloads](https://img.shields.io/gem/dt/fluent-plugin-parser_cef.svg)](https://rubygems.org/gems/fluent-plugin-parser_cef)
 [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE)
 
@@ -45,6 +46,14 @@ td-agent-gem install fluent-plugin-parser_cef
 
   input log format, currently only 'syslog' is valid
 
+* `log_utc_offset` (default: nil)
+
+  set log utc_offset if each record does not have timezone information and the timezone is not local timezone
+
+  if log_utc_offset set to nil or invalid value, then use system timezone
+
+  if a log have timezone information, log_utc_offset is ignored
+
 * `syslog_timestamp` (default: '\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}')
 
   syslog timestamp format, the default is traditional syslog timestamp

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.2.0

data/lib/fluent/plugin/parser_cef.rb

@@ -12,6 +12,7 @@ module Fluent
       Plugin.register_parser("cef", self)
 
       config_param :log_format, :string, :default => "syslog"
+      config_param :log_utc_offset, :string, :default => nil
       config_param :syslog_timestamp_format, :string, :default => '\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}'
       config_param :cef_version, :integer, :default => 0
       config_param :parse_strict_mode, :bool, :default => true
@@ -24,6 +25,7 @@ module Fluent
 
         @key_value_format_regexp = /([^\s=]+)=(.*?)(?:(?=[^\s=]+=)|\z)/
         @valid_format_regexp = create_valid_format_regexp
+        @utc_offset = get_utc_offset(@log_utc_offset)
 
         begin
           if @parse_strict_mode
@@ -45,6 +47,18 @@ module Fluent
         end
       end
 
+      def get_utc_offset(text)
+        utc_offset = nil
+        begin
+          utc_offset = Time.new.localtime(text).strftime("%:z")
+          $log.info "utc_offset: #{utc_offset}"
+        rescue => e
+          utc_offset = Time.new.localtime.strftime("%:z")
+          $log.info "#{e.message}, use localtime"
+          $log.info "utc_offset: #{utc_offset}"
+        end
+        return utc_offset
+      end
 
       def create_valid_format_regexp()
         case @log_format
@@ -66,6 +80,7 @@ module Fluent
           valid_format_regexp = /
               \A
                 #{syslog_header}
+                (?:\u{feff})?
                 #{cef_header}\|
                 (?<cef_extension>.*)
               \z
@@ -77,6 +92,21 @@ module Fluent
       end
 
 
+      def get_unixtime_with_utc_offset(timestamp, utc_offset)
+        unixtime = nil
+        begin
+          if timestamp =~ /[-+]\d{2}:?\d{2}\z/
+            unixtime = Time.parse(timestamp).to_i
+          else
+            unixtime = Time.parse("#{timestamp} #{utc_offset}").to_i
+          end
+        rescue
+          unixtime = Engine.now
+        end
+        return unixtime
+      end
+
+
       def parse(text)
         if text.nil? || text.empty?
           if block_given?
@@ -87,6 +117,7 @@ module Fluent
           end
         end
 
+        text.force_encoding("utf-8")
         record = {}
         record_overview = @valid_format_regexp.match(text)
         if record_overview.nil?
@@ -98,11 +129,7 @@ module Fluent
           end
         end
 
-        begin
-          time = Time.parse(record_overview["syslog_timestamp"]).to_i
-        rescue
-          time = Engine.now
-        end
+        time = get_unixtime_with_utc_offset(record_overview["syslog_timestamp"], @utc_offset)
 
         begin
           record_overview.names.each {|key| record[key] = record_overview[key] }

data/spec/fluent/plugin/parser_cef_spec.rb

@@ -155,5 +155,133 @@ RSpec.describe Fluent::TextParser::CommonEventFormatParser do
           "cef_name" => "Name",
           "cef_severity" => "Severity" }]}
     end
+    context "timestamp is rfc3339, UTC+3" do
+      let (:config) {%[
+        syslog_timestamp_format  \\d{4}-{,1}\\d{2}-{,1}\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+){,1}(?:\\Z|\\+\\d{2}:\\d{2})
+      ]}
+      let (:text) { "2014-06-07T18:55:09.019283+03:00 hostname tag CEF:0|Vendor|Product|Version|ID|Name|Severity|foo=bar" }
+      subject do
+        allow(Fluent::Engine).to receive(:now).and_return(Time.now.to_i)
+        @timestamp = Time.parse("2014-06-07T18:55:09.019283+03:00").to_i
+        @test_driver = create_driver(config)
+        @test_driver.parse(text)
+      end
+      it { is_expected.to eq [
+        @timestamp, {
+          "syslog_timestamp" => "2014-06-07T18:55:09.019283+03:00",
+          "syslog_hostname" => "hostname",
+          "syslog_tag" => "tag",
+          "cef_version" => "0",
+          "cef_device_vendor" => "Vendor",
+          "cef_device_product" => "Product",
+          "cef_device_version" => "Version",
+          "cef_device_event_class_id" => "ID",
+          "cef_name" => "Name",
+          "cef_severity" => "Severity" }]}
+    end
+    context "timestamp is rfc3339, UTC+0" do
+      let (:config) {%[
+        syslog_timestamp_format  \\d{4}-{,1}\\d{2}-{,1}\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+){,1}(?:Z|\\+\\d{2}:\\d{2})
+      ]}
+      let (:text) { "2014-06-07T18:55:09.019283Z hostname tag CEF:0|Vendor|Product|Version|ID|Name|Severity|foo=bar" }
+      subject do
+        allow(Fluent::Engine).to receive(:now).and_return(Time.now.to_i)
+        @timestamp = Time.parse("2014-06-07T18:55:09.019283Z").to_i
+        @test_driver = create_driver(config)
+        @test_driver.parse(text)
+      end
+      it { is_expected.to eq [
+        @timestamp, {
+          "syslog_timestamp" => "2014-06-07T18:55:09.019283Z",
+          "syslog_hostname" => "hostname",
+          "syslog_tag" => "tag",
+          "cef_version" => "0",
+          "cef_device_vendor" => "Vendor",
+          "cef_device_product" => "Product",
+          "cef_device_version" => "Version",
+          "cef_device_event_class_id" => "ID",
+          "cef_name" => "Name",
+          "cef_severity" => "Severity" }]}
+    end
+    context "utc offset set to +04:00" do
+      let (:config) {%[
+        log_utc_offset  +04:00
+      ]}
+      let (:text) { "Dec  2 03:17:06 hostname tag CEF:0|Vendor|Product|Version|ID|Name|Severity|cs1=test" }
+      subject do
+        allow(Fluent::Engine).to receive(:now).and_return(Time.now.to_i)
+        @timestamp = Time.parse("Dec  2 03:17:06 +04:00").to_i
+        @test_driver = create_driver(config)
+        @test_driver.parse(text)
+      end
+      it { is_expected.to eq [
+        @timestamp, {
+          "syslog_timestamp" => "Dec  2 03:17:06",
+          "syslog_hostname" => "hostname",
+          "syslog_tag" => "tag",
+          "cef_version" => "0",
+          "cef_device_vendor" => "Vendor",
+          "cef_device_product" => "Product",
+          "cef_device_version" => "Version",
+          "cef_device_event_class_id" => "ID",
+          "cef_name" => "Name",
+          "cef_severity" => "Severity",
+          "cs1" => "test" }]}
+    end
+    context "utc offset set to -11:00, but log timestamp has timezone information, so utc offset is ignored" do
+      let (:config) {%[
+        syslog_timestamp_format  \\d{4}-{,1}\\d{2}-{,1}\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+){,1}(?:\\Z|\\+\\d{2}:\\d{2})
+        log_utc_offset  -11:00
+      ]}
+      let (:text) { "2013-07-24T12:34:56.923984+03:30 hostname tag CEF:0|Vendor|Product|Version|ID|Name|Severity|cs1=test" }
+      subject do
+        allow(Fluent::Engine).to receive(:now).and_return(Time.now.to_i)
+        @timestamp = Time.parse("2013-07-24T12:34:56.923984+03:30").to_i
+        @test_driver = create_driver(config)
+        @test_driver.parse(text)
+      end
+      it { is_expected.to eq [
+        @timestamp, {
+          "syslog_timestamp" => "2013-07-24T12:34:56.923984+03:30",
+          "syslog_hostname" => "hostname",
+          "syslog_tag" => "tag",
+          "cef_version" => "0",
+          "cef_device_vendor" => "Vendor",
+          "cef_device_product" => "Product",
+          "cef_device_version" => "Version",
+          "cef_device_event_class_id" => "ID",
+          "cef_name" => "Name",
+          "cef_severity" => "Severity",
+          "cs1" => "test" }]}
+    end
+    context "syslog message is UTF-8, with BOM" do
+      let (:config) {%[
+        log_utc_offset  -07:00
+      ]}
+      let (:text) { "Dec  2 03:17:06 hostname tag ***CEF:0|Vendor|Product|Version|ID|Name|Severity|cs1=test" }
+      subject do
+        allow(Fluent::Engine).to receive(:now).and_return(Time.now.to_i)
+        @timestamp = Time.parse("Dec  2 03:17:06 -07:00").to_i
+        @test_driver = create_driver(config)
+        text.setbyte(29, 0xef)
+        text.setbyte(30, 0xbb)
+        text.setbyte(31, 0xbf)
+        text.force_encoding("ascii-8bit")
+        @test_driver.parse(text)
+      end
+      it { is_expected.to eq [
+        @timestamp, {
+          "syslog_timestamp" => "Dec  2 03:17:06",
+          "syslog_hostname" => "hostname",
+          "syslog_tag" => "tag",
+          "cef_version" => "0",
+          "cef_device_vendor" => "Vendor",
+          "cef_device_product" => "Product",
+          "cef_device_version" => "Version",
+          "cef_device_event_class_id" => "ID",
+          "cef_name" => "Name",
+          "cef_severity" => "Severity",
+          "cs1" => "test" }]}
+    end
   end
 end

data/spec/spec_helper.rb

@@ -20,6 +20,16 @@
 $LOAD_PATH.unshift(File.expand_path('../lib', __FILE__))
 $LOAD_PATH.unshift(File.expand_path('../spec', __FILE__))
 
+require 'simplecov'
+require 'coveralls'
+Coveralls.wear!
+
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+  SimpleCov::Formatter::HTMLFormatter,
+  Coveralls::SimpleCov::Formatter
+]
+SimpleCov.start
+
 RSpec.configure do |config|
   # rspec-expectations config goes here. You can use an alternate
   # assertion/expectation library such as wrong or the stdlib/minitest

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser_cef
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Tomoyuki Sugimura
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-06 00:00:00.000000000 Z
+date: 2017-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -108,6 +108,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".codeclimate.yml"
+- ".coveralls.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"