fluent-plugin-fortigate-log-parser 0.1.0

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-fortigate-log-parser.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 TERAOKA Yoshinori
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,92 @@
+fluent-plugin-fortigate-log-parser
+==================================
+
+FortiGate syslog parser for Fluentd
+
+This fluentd plugin parse CSV format FortiGate log
+
+You can use GeoLite Country file for World Map in Kibana.
+
+You can build poor man's FortiAnalyzer.
+
+## Plugin setup
+
+With td-agent:
+
+Put `out_fortigate_syslog_parser.rb` into /etc/td-agent/plugin directory.
+
+## FortiGate config
+
+```
+$ show log syslogd2 setting
+config log syslogd2 setting
+    set status enable
+    set server "10.20.30.40"
+    set csv enable
+    set facility local6
+end
+```
+
+## Sample td-agent (fluentd) config
+
+```:td-agent.conf
+<source>
+  type tail
+  format none
+  path /var/log/local6/%Y%m%d/fortigate
+  tag raw.fortigate
+  pos_file /var/log/td-agent/fortidate.pos
+</source>
+
+<match raw.fortigate>
+  type fortigate_log_parser
+  remove_prefix raw
+  country_map_file /etc/td-agent/country.map
+  fortios_version 5
+</match>
+
+<match fortigate>
+  type rewrite_tag_filter
+  rewriterule1 type ^traffic$ traffic.fortigate
+  rewriterule2 type ^utm$     utm.fortigate
+  rewriterule3 type ^event$   event.fortigate
+</match>
+
+<match traffic.fortigate>
+  type elasticsearch
+  host kibana-server
+  port 9200
+  logstash_format true
+  logstash_prefix fg_traffic
+  flush_interval 5s
+</match>
+
+<match utm.fortigate>
+  type elasticsearch
+  host kibana-server
+  port 9200
+  logstash_format true
+  logstash_prefix fg_utm
+  flush_interval 10s
+</match>
+
+<match event.fortigate>
+  type elasticsearch
+  host kibana-server
+  port 9200
+  logstash_format true
+  logstash_prefix fg_event
+  flush_interval 10s
+</match>
+```
+
+Fluentd merged fluent-plugin-tail-ex features in v0.10.45.
+And td-agent 1.1.19 include fluentd v0.10.45.
+
+## Generate country.map file
+
+```
+curl -O http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip
+unzip GeoIPCountryCSV.zip
+ruby bin/gen_fg_country_map GeoIPCountryWhois.csv | sort -u > country.map
+```

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+test.libs << 'lib' << 'test'
+test.pattern = 'test/**/test_*.rb'
+test.verbose = true
+end
+task :default => :test

data/bin/gen_fg_country_map

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+#
+# Usage:
+#
+# curl -O http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip
+# unzip GeoIPCountryCSV.zip
+# ruby mkCountryMap.rb GeoIPCountryWhois.csv > country.map
+#
+require 'csv'
+
+reader = CSV.open(ARGV[0], 'r')
+reader.each do |row|
+  puts row[5] + "\t" + row[4]
+end

data/fluent-plugin-fortigate-log-parser.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-fortigate-log-parser"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Yoshinori TERAOKA"]
+  spec.email         = ["jyobijyoba@gmail.com"]
+  spec.summary       = %q{fluentd plugin for parse FortiGate log}
+  spec.description   = spec.summary
+  spec.homepage      = "https://github.com/yteraoka/fluent-plugin-fortigate-log-parser"
+  spec.license       = "MIT"
+  spec.has_rdoc      = false
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "fluentd"
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/fluent/plugin/out_fortigate_syslog_parser.rb

@@ -0,0 +1,141 @@
+module Fluent
+  class FortigateSyslogParseOutput < Output
+    Fluent::Plugin.register_output('fortigate_log_parser', self)
+
+    require 'uri'
+
+    config_param :remove_prefix,    :string, :default => nil
+    config_param :add_prefix,       :string, :default => nil
+    config_param :message_key,      :string, :default => 'message'
+    config_param :keys,             :string, :default => nil
+    config_param :remove_keys,      :string, :default => nil
+    config_param :country_map_file, :string, :default => nil
+    config_param :fortios_version,  :integer, :default => 5
+
+    def configure(conf)
+      super
+
+      @prev_time_str = nil
+      @prev_time = nil
+      @country_map = nil
+
+      if @message_key == ''
+        fail ConfigError, 'message_key required a value'
+      end
+
+      if @remove_prefix
+        @removed_prefix_string = @remove_prefix + '.'
+        @removed_length = @removed_prefix_string.length
+      end
+      if @add_prefix
+        @added_prefix_string = @add_prefix + '.'
+      end
+
+      if @keys
+        if @remove_keys
+          fail ConfigError, "fortigate_log_parser: 'keys' and 'remove_keys'" \
+                             ' parameters are exclusive'
+        end
+        @keys = Hash[@keys.split(',').map { |x| [x, 1] }]
+      end
+      if @remove_keys
+        @remove_keys = Hash[@remove_keys.split(',').map { |x| [x, 1] }]
+      end
+
+      if @country_map_file
+        unless File.exist?(@country_map_file)
+          fail ConfigError, "#{@country_map_file} does not exist"
+        end
+        @country_map = {}
+        File.open(@country_map_file, 'r') do |f|
+          f.each_line do |line|
+            (country_name, country_code) = line.chomp.split(/\t/, 2)
+            @country_map[country_name] = country_code
+          end
+        end
+      end
+
+      if @fortios_version >= 5
+        @srccountry_key = 'srccountry'
+        @dstcountry_key = 'dstcountry'
+      else
+        @srccountry_key = 'src_country'
+        @dstcountry_key = 'dst_country'
+      end
+    end
+
+    def emit(tag, es, chain)
+      _tag = tag.clone
+
+      if @remove_prefix &&
+        ((tag.start_with?(@removed_prefix_string) &&
+          tag.length > @removed_length) || tag == @remove_prefix)
+        tag = tag[@removed_length..-1] || ''
+      end
+
+      if @add_prefix
+        tag = tag && tag.length > 0 ? @added_prefix_string + tag : @add_prefix
+      end
+
+      es.each do |time, record|
+        time, record = parse(record)
+        Engine.emit(tag, time, record)
+      end
+
+      chain.next
+    end
+
+    def parse(record)
+      message = record[@message_key]
+      record.delete(@message_key)
+      data = message.split(/\s+/, 5).pop
+      data.gsub(/\G[^,=]+=(:?"[^"]*"|[^,]+)(:?,|$)/) do |kv|
+        (k, v) = kv.strip.split(/=/, 2)
+        if k == 'date' || k == 'time' ||
+           (@keys && @keys.key?(k)) ||
+           (@remove_keys && !@remove_keys.key?(k)) ||
+           (!@keys && !@remove_keys)
+          record[k] = v.gsub(/,$/, '').gsub(/^"(.*)"$/, '\1')
+        end
+      end
+
+      time_str = record['date'] + ' ' + record['time']
+      time = nil
+
+      if @prev_time && time_str == @prev_time_str
+        time = @prev_time
+      else
+        # XXX FortiGate BUG (time format)
+        time = Time.strptime(time_str, '%Y-%m-%d %H: %M:%S').to_i
+        @prev_time = time
+        @prev_time_str = time_str
+      end
+
+      if @country_map
+        if record.key?(@srccountry_key) &&
+           @country_map.key?(record[@srccountry_key])
+          record[@srccountry_key + '_code'] =
+              @country_map[record[@srccountry_key]]
+        end
+        if record.key?(@dstcountry_key) &&
+           @country_map.key?(record[@dstcountry_key])
+          record[@dstcountry_key + '_code'] =
+              @country_map[record[@dstcountry_key]]
+        end
+      end
+
+      if record.key?('file')
+        record['file'] = URI.escape(record['file'])
+      end
+
+      if record.key?('filename')
+        record['filename'] = URI.escape(record['filename'])
+      end
+
+      record.delete('date')
+      record.delete('time')
+
+      [time, record]
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,25 @@
+require 'rubygems'
+require 'bundler'
+begin
+Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+$stderr.puts e.message
+$stderr.puts "Run `bundle install` to install missing gems"
+exit e.status_code
+end
+require 'test/unit'
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+nulllogger = Object.new
+nulllogger.instance_eval {|obj|
+def method_missing(method, *args)
+# pass
+end
+}
+$log = nulllogger
+end
+require 'fluent/plugin/out_fortigate_syslog_parser'
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_fortigate_syslog_parser.rb

@@ -0,0 +1,156 @@
+# coding: utf-8
+require 'helper'
+
+class FortigateSyslogParserOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+  ]
+
+  CONFIG_REWRITE_TAG = %[
+    remove_prefix before
+    add_prefix after
+  ]
+
+  CONFIG_MESSAGE_KEY = %[
+    message_key mykey
+  ]
+
+  CONFIG_COUNTRY_MAP = %[
+    country_map_file test/test_country.map
+  ]
+
+  CONFIG_OS_VERSION4 = %[
+    country_map_file test/test_country.map
+    fortios_version 4
+  ]
+
+  CONFIG_KEYS = %[
+    keys a,b,c
+  ]
+
+  CONFIG_REMOVE_KEYS = %[
+    remove_keys a,b,c
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::FortigateSyslogParseOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        keys a,b,c
+        remove_keys x,y,z
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('country_map_file not_exist')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('message_key')
+    }
+  end
+
+  def test_emit
+    d1 = create_driver(CONFIG)
+    d1.run do
+      d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,devname=TEST_NAME,devid=TEST_ID,logid=0000000001'})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal '0000000001', emits[0][2]['logid']
+  end
+
+  def test_emit_rewrite_tag
+    d1 = create_driver(CONFIG)
+    d1.run do
+      d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,file=あああ,filename=いいい'})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal '%E3%81%82%E3%81%82%E3%81%82', emits[0][2]['file']
+    assert_equal '%E3%81%84%E3%81%84%E3%81%84', emits[0][2]['filename']
+  end
+
+  def test_emit_rewrite_tag
+    d1 = create_driver(CONFIG_REWRITE_TAG, 'before.test')
+    d1.run do
+      d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59'})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'after.test', emits[0][0]
+  end
+
+  def test_emit_message_key
+    d1 = create_driver(CONFIG_MESSAGE_KEY)
+    d1.run do
+      d1.emit({'mykey' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,key1=value1,key2=value2'})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'value1', emits[0][2]['key1']
+    assert_equal 'value2', emits[0][2]['key2']
+  end
+
+  def test_emit_date_parse
+    d1 = create_driver()
+    d1.run do
+      d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59'})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 1408201199, emits[0][1]
+  end
+
+  def test_emit_country_map
+    d1 = create_driver(CONFIG_COUNTRY_MAP)
+    d1.run do
+      d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,srccountry=Japan,dstcountry=United States'})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'Japan', emits[0][2]['srccountry']
+    assert_equal 'United States', emits[0][2]['dstcountry']
+    assert_equal 'JP', emits[0][2]['srccountry_code']
+    assert_equal 'US', emits[0][2]['dstcountry_code']
+  end
+
+  def test_emit_os_version4
+    d1 = create_driver(CONFIG_OS_VERSION4)
+    d1.run do
+      d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,src_country=Japan,dst_country=United States'})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal 'Japan', emits[0][2]['src_country']
+    assert_equal 'United States', emits[0][2]['dst_country']
+    assert_equal 'JP', emits[0][2]['src_country_code']
+    assert_equal 'US', emits[0][2]['dst_country_code']
+  end
+
+  def test_emit_keys
+    d1 = create_driver(CONFIG_KEYS)
+    d1.run do
+      d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,a=A,b=B,c=C,x=X,y=Y,z=Z'})
+    end
+    expected = {'a' => 'A', 'b' => 'B', 'c' => 'C'}
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal expected, emits[0][2]
+  end
+
+  def test_emit_remove_keys
+    d1 = create_driver(CONFIG_REMOVE_KEYS)
+    d1.run do
+      d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,a=A,b=B,c=C,x=X,y=Y,z=Z'})
+    end
+    expected = {'x' => 'X', 'y' => 'Y', 'z' => 'Z'}
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal expected, emits[0][2]
+  end
+end

data/test/test_country.map

@@ -0,0 +1,2 @@
+Japan	JP
+United States	US

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-fortigate-log-parser
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Yoshinori TERAOKA
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: fluentd plugin for parse FortiGate log
+email:
+- jyobijyoba@gmail.com
+executables:
+- gen_fg_country_map
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/gen_fg_country_map
+- fluent-plugin-fortigate-log-parser.gemspec
+- lib/fluent/plugin/out_fortigate_syslog_parser.rb
+- test/helper.rb
+- test/plugin/test_out_fortigate_syslog_parser.rb
+- test/test_country.map
+homepage: https://github.com/yteraoka/fluent-plugin-fortigate-log-parser
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: fluentd plugin for parse FortiGate log
+test_files:
+- test/helper.rb
+- test/plugin/test_out_fortigate_syslog_parser.rb
+- test/test_country.map