fluent-plugin-fortigate-log-parser 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,14 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ *.bundle
11
+ *.so
12
+ *.o
13
+ *.a
14
+ mkmf.log
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-fortigate-log-parser.gemspec
4
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2014 TERAOKA Yoshinori
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
6
+ this software and associated documentation files (the "Software"), to deal in
7
+ the Software without restriction, including without limitation the rights to
8
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
9
+ the Software, and to permit persons to whom the Software is furnished to do so,
10
+ subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,92 @@
1
+ fluent-plugin-fortigate-log-parser
2
+ ==================================
3
+
4
+ FortiGate syslog parser for Fluentd
5
+
6
+ This fluentd plugin parse CSV format FortiGate log
7
+
8
+ You can use GeoLite Country file for World Map in Kibana.
9
+
10
+ You can build poor man's FortiAnalyzer.
11
+
12
+ ## Plugin setup
13
+
14
+ With td-agent:
15
+
16
+ Put `out_fortigate_syslog_parser.rb` into /etc/td-agent/plugin directory.
17
+
18
+ ## FortiGate config
19
+
20
+ ```
21
+ $ show log syslogd2 setting
22
+ config log syslogd2 setting
23
+ set status enable
24
+ set server "10.20.30.40"
25
+ set csv enable
26
+ set facility local6
27
+ end
28
+ ```
29
+
30
+ ## Sample td-agent (fluentd) config
31
+
32
+ ```:td-agent.conf
33
+ <source>
34
+ type tail
35
+ format none
36
+ path /var/log/local6/%Y%m%d/fortigate
37
+ tag raw.fortigate
38
+ pos_file /var/log/td-agent/fortidate.pos
39
+ </source>
40
+
41
+ <match raw.fortigate>
42
+ type fortigate_log_parser
43
+ remove_prefix raw
44
+ country_map_file /etc/td-agent/country.map
45
+ fortios_version 5
46
+ </match>
47
+
48
+ <match fortigate>
49
+ type rewrite_tag_filter
50
+ rewriterule1 type ^traffic$ traffic.fortigate
51
+ rewriterule2 type ^utm$ utm.fortigate
52
+ rewriterule3 type ^event$ event.fortigate
53
+ </match>
54
+
55
+ <match traffic.fortigate>
56
+ type elasticsearch
57
+ host kibana-server
58
+ port 9200
59
+ logstash_format true
60
+ logstash_prefix fg_traffic
61
+ flush_interval 5s
62
+ </match>
63
+
64
+ <match utm.fortigate>
65
+ type elasticsearch
66
+ host kibana-server
67
+ port 9200
68
+ logstash_format true
69
+ logstash_prefix fg_utm
70
+ flush_interval 10s
71
+ </match>
72
+
73
+ <match event.fortigate>
74
+ type elasticsearch
75
+ host kibana-server
76
+ port 9200
77
+ logstash_format true
78
+ logstash_prefix fg_event
79
+ flush_interval 10s
80
+ </match>
81
+ ```
82
+
83
+ Fluentd merged fluent-plugin-tail-ex features in v0.10.45.
84
+ And td-agent 1.1.19 include fluentd v0.10.45.
85
+
86
+ ## Generate country.map file
87
+
88
+ ```
89
+ curl -O http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip
90
+ unzip GeoIPCountryCSV.zip
91
+ ruby bin/gen_fg_country_map GeoIPCountryWhois.csv | sort -u > country.map
92
+ ```
@@ -0,0 +1,8 @@
1
+ require "bundler/gem_tasks"
2
+ require 'rake/testtask'
3
+ Rake::TestTask.new(:test) do |test|
4
+ test.libs << 'lib' << 'test'
5
+ test.pattern = 'test/**/test_*.rb'
6
+ test.verbose = true
7
+ end
8
+ task :default => :test
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env ruby
2
+ # coding: utf-8
3
+ #
4
+ # Usage:
5
+ #
6
+ # curl -O http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip
7
+ # unzip GeoIPCountryCSV.zip
8
+ # ruby mkCountryMap.rb GeoIPCountryWhois.csv > country.map
9
+ #
10
+ require 'csv'
11
+
12
+ reader = CSV.open(ARGV[0], 'r')
13
+ reader.each do |row|
14
+ puts row[5] + "\t" + row[4]
15
+ end
@@ -0,0 +1,24 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "fluent-plugin-fortigate-log-parser"
7
+ spec.version = "0.1.0"
8
+ spec.authors = ["Yoshinori TERAOKA"]
9
+ spec.email = ["jyobijyoba@gmail.com"]
10
+ spec.summary = %q{fluentd plugin for parse FortiGate log}
11
+ spec.description = spec.summary
12
+ spec.homepage = "https://github.com/yteraoka/fluent-plugin-fortigate-log-parser"
13
+ spec.license = "MIT"
14
+ spec.has_rdoc = false
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_dependency "fluentd"
22
+ spec.add_development_dependency "bundler", "~> 1.7"
23
+ spec.add_development_dependency "rake", "~> 10.0"
24
+ end
@@ -0,0 +1,141 @@
1
+ module Fluent
2
+ class FortigateSyslogParseOutput < Output
3
+ Fluent::Plugin.register_output('fortigate_log_parser', self)
4
+
5
+ require 'uri'
6
+
7
+ config_param :remove_prefix, :string, :default => nil
8
+ config_param :add_prefix, :string, :default => nil
9
+ config_param :message_key, :string, :default => 'message'
10
+ config_param :keys, :string, :default => nil
11
+ config_param :remove_keys, :string, :default => nil
12
+ config_param :country_map_file, :string, :default => nil
13
+ config_param :fortios_version, :integer, :default => 5
14
+
15
+ def configure(conf)
16
+ super
17
+
18
+ @prev_time_str = nil
19
+ @prev_time = nil
20
+ @country_map = nil
21
+
22
+ if @message_key == ''
23
+ fail ConfigError, 'message_key required a value'
24
+ end
25
+
26
+ if @remove_prefix
27
+ @removed_prefix_string = @remove_prefix + '.'
28
+ @removed_length = @removed_prefix_string.length
29
+ end
30
+ if @add_prefix
31
+ @added_prefix_string = @add_prefix + '.'
32
+ end
33
+
34
+ if @keys
35
+ if @remove_keys
36
+ fail ConfigError, "fortigate_log_parser: 'keys' and 'remove_keys'" \
37
+ ' parameters are exclusive'
38
+ end
39
+ @keys = Hash[@keys.split(',').map { |x| [x, 1] }]
40
+ end
41
+ if @remove_keys
42
+ @remove_keys = Hash[@remove_keys.split(',').map { |x| [x, 1] }]
43
+ end
44
+
45
+ if @country_map_file
46
+ unless File.exist?(@country_map_file)
47
+ fail ConfigError, "#{@country_map_file} does not exist"
48
+ end
49
+ @country_map = {}
50
+ File.open(@country_map_file, 'r') do |f|
51
+ f.each_line do |line|
52
+ (country_name, country_code) = line.chomp.split(/\t/, 2)
53
+ @country_map[country_name] = country_code
54
+ end
55
+ end
56
+ end
57
+
58
+ if @fortios_version >= 5
59
+ @srccountry_key = 'srccountry'
60
+ @dstcountry_key = 'dstcountry'
61
+ else
62
+ @srccountry_key = 'src_country'
63
+ @dstcountry_key = 'dst_country'
64
+ end
65
+ end
66
+
67
+ def emit(tag, es, chain)
68
+ _tag = tag.clone
69
+
70
+ if @remove_prefix &&
71
+ ((tag.start_with?(@removed_prefix_string) &&
72
+ tag.length > @removed_length) || tag == @remove_prefix)
73
+ tag = tag[@removed_length..-1] || ''
74
+ end
75
+
76
+ if @add_prefix
77
+ tag = tag && tag.length > 0 ? @added_prefix_string + tag : @add_prefix
78
+ end
79
+
80
+ es.each do |time, record|
81
+ time, record = parse(record)
82
+ Engine.emit(tag, time, record)
83
+ end
84
+
85
+ chain.next
86
+ end
87
+
88
+ def parse(record)
89
+ message = record[@message_key]
90
+ record.delete(@message_key)
91
+ data = message.split(/\s+/, 5).pop
92
+ data.gsub(/\G[^,=]+=(:?"[^"]*"|[^,]+)(:?,|$)/) do |kv|
93
+ (k, v) = kv.strip.split(/=/, 2)
94
+ if k == 'date' || k == 'time' ||
95
+ (@keys && @keys.key?(k)) ||
96
+ (@remove_keys && !@remove_keys.key?(k)) ||
97
+ (!@keys && !@remove_keys)
98
+ record[k] = v.gsub(/,$/, '').gsub(/^"(.*)"$/, '\1')
99
+ end
100
+ end
101
+
102
+ time_str = record['date'] + ' ' + record['time']
103
+ time = nil
104
+
105
+ if @prev_time && time_str == @prev_time_str
106
+ time = @prev_time
107
+ else
108
+ # XXX FortiGate BUG (time format)
109
+ time = Time.strptime(time_str, '%Y-%m-%d %H: %M:%S').to_i
110
+ @prev_time = time
111
+ @prev_time_str = time_str
112
+ end
113
+
114
+ if @country_map
115
+ if record.key?(@srccountry_key) &&
116
+ @country_map.key?(record[@srccountry_key])
117
+ record[@srccountry_key + '_code'] =
118
+ @country_map[record[@srccountry_key]]
119
+ end
120
+ if record.key?(@dstcountry_key) &&
121
+ @country_map.key?(record[@dstcountry_key])
122
+ record[@dstcountry_key + '_code'] =
123
+ @country_map[record[@dstcountry_key]]
124
+ end
125
+ end
126
+
127
+ if record.key?('file')
128
+ record['file'] = URI.escape(record['file'])
129
+ end
130
+
131
+ if record.key?('filename')
132
+ record['filename'] = URI.escape(record['filename'])
133
+ end
134
+
135
+ record.delete('date')
136
+ record.delete('time')
137
+
138
+ [time, record]
139
+ end
140
+ end
141
+ end
@@ -0,0 +1,25 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
12
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
13
+ require 'fluent/test'
14
+ unless ENV.has_key?('VERBOSE')
15
+ nulllogger = Object.new
16
+ nulllogger.instance_eval {|obj|
17
+ def method_missing(method, *args)
18
+ # pass
19
+ end
20
+ }
21
+ $log = nulllogger
22
+ end
23
+ require 'fluent/plugin/out_fortigate_syslog_parser'
24
+ class Test::Unit::TestCase
25
+ end
@@ -0,0 +1,156 @@
1
+ # coding: utf-8
2
+ require 'helper'
3
+
4
+ class FortigateSyslogParserOutputTest < Test::Unit::TestCase
5
+ def setup
6
+ Fluent::Test.setup
7
+ end
8
+
9
+ CONFIG = %[
10
+ ]
11
+
12
+ CONFIG_REWRITE_TAG = %[
13
+ remove_prefix before
14
+ add_prefix after
15
+ ]
16
+
17
+ CONFIG_MESSAGE_KEY = %[
18
+ message_key mykey
19
+ ]
20
+
21
+ CONFIG_COUNTRY_MAP = %[
22
+ country_map_file test/test_country.map
23
+ ]
24
+
25
+ CONFIG_OS_VERSION4 = %[
26
+ country_map_file test/test_country.map
27
+ fortios_version 4
28
+ ]
29
+
30
+ CONFIG_KEYS = %[
31
+ keys a,b,c
32
+ ]
33
+
34
+ CONFIG_REMOVE_KEYS = %[
35
+ remove_keys a,b,c
36
+ ]
37
+
38
+ def create_driver(conf=CONFIG,tag='test')
39
+ Fluent::Test::OutputTestDriver.new(Fluent::FortigateSyslogParseOutput, tag).configure(conf)
40
+ end
41
+
42
+ def test_configure
43
+ assert_raise(Fluent::ConfigError) {
44
+ d = create_driver %[
45
+ keys a,b,c
46
+ remove_keys x,y,z
47
+ ]
48
+ }
49
+ assert_raise(Fluent::ConfigError) {
50
+ d = create_driver('country_map_file not_exist')
51
+ }
52
+ assert_raise(Fluent::ConfigError) {
53
+ d = create_driver('message_key')
54
+ }
55
+ end
56
+
57
+ def test_emit
58
+ d1 = create_driver(CONFIG)
59
+ d1.run do
60
+ d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,devname=TEST_NAME,devid=TEST_ID,logid=0000000001'})
61
+ end
62
+ emits = d1.emits
63
+ assert_equal 1, emits.length
64
+ assert_equal '0000000001', emits[0][2]['logid']
65
+ end
66
+
67
+ def test_emit_rewrite_tag
68
+ d1 = create_driver(CONFIG)
69
+ d1.run do
70
+ d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,file=あああ,filename=いいい'})
71
+ end
72
+ emits = d1.emits
73
+ assert_equal 1, emits.length
74
+ assert_equal '%E3%81%82%E3%81%82%E3%81%82', emits[0][2]['file']
75
+ assert_equal '%E3%81%84%E3%81%84%E3%81%84', emits[0][2]['filename']
76
+ end
77
+
78
+ def test_emit_rewrite_tag
79
+ d1 = create_driver(CONFIG_REWRITE_TAG, 'before.test')
80
+ d1.run do
81
+ d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59'})
82
+ end
83
+ emits = d1.emits
84
+ assert_equal 1, emits.length
85
+ assert_equal 'after.test', emits[0][0]
86
+ end
87
+
88
+ def test_emit_message_key
89
+ d1 = create_driver(CONFIG_MESSAGE_KEY)
90
+ d1.run do
91
+ d1.emit({'mykey' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,key1=value1,key2=value2'})
92
+ end
93
+ emits = d1.emits
94
+ assert_equal 1, emits.length
95
+ assert_equal 'value1', emits[0][2]['key1']
96
+ assert_equal 'value2', emits[0][2]['key2']
97
+ end
98
+
99
+ def test_emit_date_parse
100
+ d1 = create_driver()
101
+ d1.run do
102
+ d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59'})
103
+ end
104
+ emits = d1.emits
105
+ assert_equal 1, emits.length
106
+ assert_equal 1408201199, emits[0][1]
107
+ end
108
+
109
+ def test_emit_country_map
110
+ d1 = create_driver(CONFIG_COUNTRY_MAP)
111
+ d1.run do
112
+ d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,srccountry=Japan,dstcountry=United States'})
113
+ end
114
+ emits = d1.emits
115
+ assert_equal 1, emits.length
116
+ assert_equal 'Japan', emits[0][2]['srccountry']
117
+ assert_equal 'United States', emits[0][2]['dstcountry']
118
+ assert_equal 'JP', emits[0][2]['srccountry_code']
119
+ assert_equal 'US', emits[0][2]['dstcountry_code']
120
+ end
121
+
122
+ def test_emit_os_version4
123
+ d1 = create_driver(CONFIG_OS_VERSION4)
124
+ d1.run do
125
+ d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,src_country=Japan,dst_country=United States'})
126
+ end
127
+ emits = d1.emits
128
+ assert_equal 1, emits.length
129
+ assert_equal 'Japan', emits[0][2]['src_country']
130
+ assert_equal 'United States', emits[0][2]['dst_country']
131
+ assert_equal 'JP', emits[0][2]['src_country_code']
132
+ assert_equal 'US', emits[0][2]['dst_country_code']
133
+ end
134
+
135
+ def test_emit_keys
136
+ d1 = create_driver(CONFIG_KEYS)
137
+ d1.run do
138
+ d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,a=A,b=B,c=C,x=X,y=Y,z=Z'})
139
+ end
140
+ expected = {'a' => 'A', 'b' => 'B', 'c' => 'C'}
141
+ emits = d1.emits
142
+ assert_equal 1, emits.length
143
+ assert_equal expected, emits[0][2]
144
+ end
145
+
146
+ def test_emit_remove_keys
147
+ d1 = create_driver(CONFIG_REMOVE_KEYS)
148
+ d1.run do
149
+ d1.emit({'message' => 'Aug 17 00:00:00 fortigate date=2014-08-16,time=23: 59:59,a=A,b=B,c=C,x=X,y=Y,z=Z'})
150
+ end
151
+ expected = {'x' => 'X', 'y' => 'Y', 'z' => 'Z'}
152
+ emits = d1.emits
153
+ assert_equal 1, emits.length
154
+ assert_equal expected, emits[0][2]
155
+ end
156
+ end
@@ -0,0 +1,2 @@
1
+ Japan JP
2
+ United States US
metadata ADDED
@@ -0,0 +1,109 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-fortigate-log-parser
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Yoshinori TERAOKA
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2014-08-17 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ - !ruby/object:Gem::Dependency
31
+ name: bundler
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ~>
36
+ - !ruby/object:Gem::Version
37
+ version: '1.7'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ~>
44
+ - !ruby/object:Gem::Version
45
+ version: '1.7'
46
+ - !ruby/object:Gem::Dependency
47
+ name: rake
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ~>
52
+ - !ruby/object:Gem::Version
53
+ version: '10.0'
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ~>
60
+ - !ruby/object:Gem::Version
61
+ version: '10.0'
62
+ description: fluentd plugin for parse FortiGate log
63
+ email:
64
+ - jyobijyoba@gmail.com
65
+ executables:
66
+ - gen_fg_country_map
67
+ extensions: []
68
+ extra_rdoc_files: []
69
+ files:
70
+ - .gitignore
71
+ - Gemfile
72
+ - LICENSE
73
+ - README.md
74
+ - Rakefile
75
+ - bin/gen_fg_country_map
76
+ - fluent-plugin-fortigate-log-parser.gemspec
77
+ - lib/fluent/plugin/out_fortigate_syslog_parser.rb
78
+ - test/helper.rb
79
+ - test/plugin/test_out_fortigate_syslog_parser.rb
80
+ - test/test_country.map
81
+ homepage: https://github.com/yteraoka/fluent-plugin-fortigate-log-parser
82
+ licenses:
83
+ - MIT
84
+ post_install_message:
85
+ rdoc_options: []
86
+ require_paths:
87
+ - lib
88
+ required_ruby_version: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
94
+ required_rubygems_version: !ruby/object:Gem::Requirement
95
+ none: false
96
+ requirements:
97
+ - - ! '>='
98
+ - !ruby/object:Gem::Version
99
+ version: '0'
100
+ requirements: []
101
+ rubyforge_project:
102
+ rubygems_version: 1.8.23
103
+ signing_key:
104
+ specification_version: 3
105
+ summary: fluentd plugin for parse FortiGate log
106
+ test_files:
107
+ - test/helper.rb
108
+ - test/plugin/test_out_fortigate_syslog_parser.rb
109
+ - test/test_country.map