fluent-plugin-parser_cef 0.1.1 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.codeclimate.yml +5 -9
- data/.coveralls.yml +1 -0
- data/Gemfile +2 -0
- data/README.md +9 -0
- data/VERSION +1 -1
- data/lib/fluent/plugin/parser_cef.rb +32 -5
- data/spec/fluent/plugin/parser_cef_spec.rb +128 -0
- data/spec/spec_helper.rb +10 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0e2eedd443b1b7fec47ddc045ee1f7506de43022
|
4
|
+
data.tar.gz: ccbc339b8394d269ab365f4b4e8fc22d49a7d57b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4e2b5fa97f9308e265b5baba0ddeb0abd211384adda34631d4c9b7793c5411b9f0edf38e7288b78ebea281a4baf577a9682a82137c05ee5061b4dcf08589d264
|
7
|
+
data.tar.gz: 3e44f6c3731f048dbf4012678dc123c5b1ca69d64e6353e60c53b5684070072c2c507b9bbae35a4d09d5f4cf006bcf1efd13479c963a6e436992b9a0e0d9c191
|
data/.codeclimate.yml
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
---
|
2
1
|
engines:
|
3
2
|
duplication:
|
4
3
|
enabled: true
|
@@ -12,6 +11,11 @@ engines:
|
|
12
11
|
enabled: true
|
13
12
|
rubocop:
|
14
13
|
enabled: true
|
14
|
+
checks:
|
15
|
+
Rubocop/Metrics/MethodLength:
|
16
|
+
enabled: false
|
17
|
+
Rubocop/Metrics/CyclomaticComplexity:
|
18
|
+
enabled: false
|
15
19
|
ratings:
|
16
20
|
paths:
|
17
21
|
- "**.inc"
|
@@ -23,11 +27,3 @@ ratings:
|
|
23
27
|
- "**.rb"
|
24
28
|
exclude_paths:
|
25
29
|
- spec/
|
26
|
-
engines:
|
27
|
-
rubocop:
|
28
|
-
enabled: true
|
29
|
-
checks:
|
30
|
-
Rubocop/Metrics/MethodLength:
|
31
|
-
enabled: false
|
32
|
-
Rubocop/Metrics/CyclomaticComplexity:
|
33
|
-
enabled: false
|
data/.coveralls.yml
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
service_name: travis-ci
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/fluent-plugin-parser_cef.svg)](https://badge.fury.io/rb/fluent-plugin-parser_cef)
|
4
4
|
[![Build Status](https://travis-ci.org/lunardial/fluent-plugin-parser_cef.svg?branch=master)](https://travis-ci.org/lunardial/fluent-plugin-parser_cef)
|
5
5
|
[![Code Climate](https://codeclimate.com/github/lunardial/fluent-plugin-parser_cef/badges/gpa.svg)](https://codeclimate.com/github/lunardial/fluent-plugin-parser_cef)
|
6
|
+
[![Coverage Status](https://coveralls.io/repos/github/lunardial/fluent-plugin-parser_cef/badge.svg?branch=master)](https://coveralls.io/github/lunardial/fluent-plugin-parser_cef?branch=master)
|
6
7
|
[![downloads](https://img.shields.io/gem/dt/fluent-plugin-parser_cef.svg)](https://rubygems.org/gems/fluent-plugin-parser_cef)
|
7
8
|
[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE)
|
8
9
|
|
@@ -45,6 +46,14 @@ td-agent-gem install fluent-plugin-parser_cef
|
|
45
46
|
|
46
47
|
input log format, currently only 'syslog' is valid
|
47
48
|
|
49
|
+
* `log_utc_offset` (default: nil)
|
50
|
+
|
51
|
+
set log utc_offset if each record does not have timezone information and the timezone is not local timezone
|
52
|
+
|
53
|
+
if log_utc_offset set to nil or invalid value, then use system timezone
|
54
|
+
|
55
|
+
if a log have timezone information, log_utc_offset is ignored
|
56
|
+
|
48
57
|
* `syslog_timestamp` (default: '\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}')
|
49
58
|
|
50
59
|
syslog timestamp format, the default is traditional syslog timestamp
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.0
|
@@ -12,6 +12,7 @@ module Fluent
|
|
12
12
|
Plugin.register_parser("cef", self)
|
13
13
|
|
14
14
|
config_param :log_format, :string, :default => "syslog"
|
15
|
+
config_param :log_utc_offset, :string, :default => nil
|
15
16
|
config_param :syslog_timestamp_format, :string, :default => '\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}'
|
16
17
|
config_param :cef_version, :integer, :default => 0
|
17
18
|
config_param :parse_strict_mode, :bool, :default => true
|
@@ -24,6 +25,7 @@ module Fluent
|
|
24
25
|
|
25
26
|
@key_value_format_regexp = /([^\s=]+)=(.*?)(?:(?=[^\s=]+=)|\z)/
|
26
27
|
@valid_format_regexp = create_valid_format_regexp
|
28
|
+
@utc_offset = get_utc_offset(@log_utc_offset)
|
27
29
|
|
28
30
|
begin
|
29
31
|
if @parse_strict_mode
|
@@ -45,6 +47,18 @@ module Fluent
|
|
45
47
|
end
|
46
48
|
end
|
47
49
|
|
50
|
+
def get_utc_offset(text)
|
51
|
+
utc_offset = nil
|
52
|
+
begin
|
53
|
+
utc_offset = Time.new.localtime(text).strftime("%:z")
|
54
|
+
$log.info "utc_offset: #{utc_offset}"
|
55
|
+
rescue => e
|
56
|
+
utc_offset = Time.new.localtime.strftime("%:z")
|
57
|
+
$log.info "#{e.message}, use localtime"
|
58
|
+
$log.info "utc_offset: #{utc_offset}"
|
59
|
+
end
|
60
|
+
return utc_offset
|
61
|
+
end
|
48
62
|
|
49
63
|
def create_valid_format_regexp()
|
50
64
|
case @log_format
|
@@ -66,6 +80,7 @@ module Fluent
|
|
66
80
|
valid_format_regexp = /
|
67
81
|
\A
|
68
82
|
#{syslog_header}
|
83
|
+
(?:\u{feff})?
|
69
84
|
#{cef_header}\|
|
70
85
|
(?<cef_extension>.*)
|
71
86
|
\z
|
@@ -77,6 +92,21 @@ module Fluent
|
|
77
92
|
end
|
78
93
|
|
79
94
|
|
95
|
+
def get_unixtime_with_utc_offset(timestamp, utc_offset)
|
96
|
+
unixtime = nil
|
97
|
+
begin
|
98
|
+
if timestamp =~ /[-+]\d{2}:?\d{2}\z/
|
99
|
+
unixtime = Time.parse(timestamp).to_i
|
100
|
+
else
|
101
|
+
unixtime = Time.parse("#{timestamp} #{utc_offset}").to_i
|
102
|
+
end
|
103
|
+
rescue
|
104
|
+
unixtime = Engine.now
|
105
|
+
end
|
106
|
+
return unixtime
|
107
|
+
end
|
108
|
+
|
109
|
+
|
80
110
|
def parse(text)
|
81
111
|
if text.nil? || text.empty?
|
82
112
|
if block_given?
|
@@ -87,6 +117,7 @@ module Fluent
|
|
87
117
|
end
|
88
118
|
end
|
89
119
|
|
120
|
+
text.force_encoding("utf-8")
|
90
121
|
record = {}
|
91
122
|
record_overview = @valid_format_regexp.match(text)
|
92
123
|
if record_overview.nil?
|
@@ -98,11 +129,7 @@ module Fluent
|
|
98
129
|
end
|
99
130
|
end
|
100
131
|
|
101
|
-
|
102
|
-
time = Time.parse(record_overview["syslog_timestamp"]).to_i
|
103
|
-
rescue
|
104
|
-
time = Engine.now
|
105
|
-
end
|
132
|
+
time = get_unixtime_with_utc_offset(record_overview["syslog_timestamp"], @utc_offset)
|
106
133
|
|
107
134
|
begin
|
108
135
|
record_overview.names.each {|key| record[key] = record_overview[key] }
|
@@ -155,5 +155,133 @@ RSpec.describe Fluent::TextParser::CommonEventFormatParser do
|
|
155
155
|
"cef_name" => "Name",
|
156
156
|
"cef_severity" => "Severity" }]}
|
157
157
|
end
|
158
|
+
context "timestamp is rfc3339, UTC+3" do
|
159
|
+
let (:config) {%[
|
160
|
+
syslog_timestamp_format \\d{4}-{,1}\\d{2}-{,1}\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+){,1}(?:\\Z|\\+\\d{2}:\\d{2})
|
161
|
+
]}
|
162
|
+
let (:text) { "2014-06-07T18:55:09.019283+03:00 hostname tag CEF:0|Vendor|Product|Version|ID|Name|Severity|foo=bar" }
|
163
|
+
subject do
|
164
|
+
allow(Fluent::Engine).to receive(:now).and_return(Time.now.to_i)
|
165
|
+
@timestamp = Time.parse("2014-06-07T18:55:09.019283+03:00").to_i
|
166
|
+
@test_driver = create_driver(config)
|
167
|
+
@test_driver.parse(text)
|
168
|
+
end
|
169
|
+
it { is_expected.to eq [
|
170
|
+
@timestamp, {
|
171
|
+
"syslog_timestamp" => "2014-06-07T18:55:09.019283+03:00",
|
172
|
+
"syslog_hostname" => "hostname",
|
173
|
+
"syslog_tag" => "tag",
|
174
|
+
"cef_version" => "0",
|
175
|
+
"cef_device_vendor" => "Vendor",
|
176
|
+
"cef_device_product" => "Product",
|
177
|
+
"cef_device_version" => "Version",
|
178
|
+
"cef_device_event_class_id" => "ID",
|
179
|
+
"cef_name" => "Name",
|
180
|
+
"cef_severity" => "Severity" }]}
|
181
|
+
end
|
182
|
+
context "timestamp is rfc3339, UTC+0" do
|
183
|
+
let (:config) {%[
|
184
|
+
syslog_timestamp_format \\d{4}-{,1}\\d{2}-{,1}\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+){,1}(?:Z|\\+\\d{2}:\\d{2})
|
185
|
+
]}
|
186
|
+
let (:text) { "2014-06-07T18:55:09.019283Z hostname tag CEF:0|Vendor|Product|Version|ID|Name|Severity|foo=bar" }
|
187
|
+
subject do
|
188
|
+
allow(Fluent::Engine).to receive(:now).and_return(Time.now.to_i)
|
189
|
+
@timestamp = Time.parse("2014-06-07T18:55:09.019283Z").to_i
|
190
|
+
@test_driver = create_driver(config)
|
191
|
+
@test_driver.parse(text)
|
192
|
+
end
|
193
|
+
it { is_expected.to eq [
|
194
|
+
@timestamp, {
|
195
|
+
"syslog_timestamp" => "2014-06-07T18:55:09.019283Z",
|
196
|
+
"syslog_hostname" => "hostname",
|
197
|
+
"syslog_tag" => "tag",
|
198
|
+
"cef_version" => "0",
|
199
|
+
"cef_device_vendor" => "Vendor",
|
200
|
+
"cef_device_product" => "Product",
|
201
|
+
"cef_device_version" => "Version",
|
202
|
+
"cef_device_event_class_id" => "ID",
|
203
|
+
"cef_name" => "Name",
|
204
|
+
"cef_severity" => "Severity" }]}
|
205
|
+
end
|
206
|
+
context "utc offset set to +04:00" do
|
207
|
+
let (:config) {%[
|
208
|
+
log_utc_offset +04:00
|
209
|
+
]}
|
210
|
+
let (:text) { "Dec 2 03:17:06 hostname tag CEF:0|Vendor|Product|Version|ID|Name|Severity|cs1=test" }
|
211
|
+
subject do
|
212
|
+
allow(Fluent::Engine).to receive(:now).and_return(Time.now.to_i)
|
213
|
+
@timestamp = Time.parse("Dec 2 03:17:06 +04:00").to_i
|
214
|
+
@test_driver = create_driver(config)
|
215
|
+
@test_driver.parse(text)
|
216
|
+
end
|
217
|
+
it { is_expected.to eq [
|
218
|
+
@timestamp, {
|
219
|
+
"syslog_timestamp" => "Dec 2 03:17:06",
|
220
|
+
"syslog_hostname" => "hostname",
|
221
|
+
"syslog_tag" => "tag",
|
222
|
+
"cef_version" => "0",
|
223
|
+
"cef_device_vendor" => "Vendor",
|
224
|
+
"cef_device_product" => "Product",
|
225
|
+
"cef_device_version" => "Version",
|
226
|
+
"cef_device_event_class_id" => "ID",
|
227
|
+
"cef_name" => "Name",
|
228
|
+
"cef_severity" => "Severity",
|
229
|
+
"cs1" => "test" }]}
|
230
|
+
end
|
231
|
+
context "utc offset set to -11:00, but log timestamp has timezone information, so utc offset is ignored" do
|
232
|
+
let (:config) {%[
|
233
|
+
syslog_timestamp_format \\d{4}-{,1}\\d{2}-{,1}\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+){,1}(?:\\Z|\\+\\d{2}:\\d{2})
|
234
|
+
log_utc_offset -11:00
|
235
|
+
]}
|
236
|
+
let (:text) { "2013-07-24T12:34:56.923984+03:30 hostname tag CEF:0|Vendor|Product|Version|ID|Name|Severity|cs1=test" }
|
237
|
+
subject do
|
238
|
+
allow(Fluent::Engine).to receive(:now).and_return(Time.now.to_i)
|
239
|
+
@timestamp = Time.parse("2013-07-24T12:34:56.923984+03:30").to_i
|
240
|
+
@test_driver = create_driver(config)
|
241
|
+
@test_driver.parse(text)
|
242
|
+
end
|
243
|
+
it { is_expected.to eq [
|
244
|
+
@timestamp, {
|
245
|
+
"syslog_timestamp" => "2013-07-24T12:34:56.923984+03:30",
|
246
|
+
"syslog_hostname" => "hostname",
|
247
|
+
"syslog_tag" => "tag",
|
248
|
+
"cef_version" => "0",
|
249
|
+
"cef_device_vendor" => "Vendor",
|
250
|
+
"cef_device_product" => "Product",
|
251
|
+
"cef_device_version" => "Version",
|
252
|
+
"cef_device_event_class_id" => "ID",
|
253
|
+
"cef_name" => "Name",
|
254
|
+
"cef_severity" => "Severity",
|
255
|
+
"cs1" => "test" }]}
|
256
|
+
end
|
257
|
+
context "syslog message is UTF-8, with BOM" do
|
258
|
+
let (:config) {%[
|
259
|
+
log_utc_offset -07:00
|
260
|
+
]}
|
261
|
+
let (:text) { "Dec 2 03:17:06 hostname tag ***CEF:0|Vendor|Product|Version|ID|Name|Severity|cs1=test" }
|
262
|
+
subject do
|
263
|
+
allow(Fluent::Engine).to receive(:now).and_return(Time.now.to_i)
|
264
|
+
@timestamp = Time.parse("Dec 2 03:17:06 -07:00").to_i
|
265
|
+
@test_driver = create_driver(config)
|
266
|
+
text.setbyte(29, 0xef)
|
267
|
+
text.setbyte(30, 0xbb)
|
268
|
+
text.setbyte(31, 0xbf)
|
269
|
+
text.force_encoding("ascii-8bit")
|
270
|
+
@test_driver.parse(text)
|
271
|
+
end
|
272
|
+
it { is_expected.to eq [
|
273
|
+
@timestamp, {
|
274
|
+
"syslog_timestamp" => "Dec 2 03:17:06",
|
275
|
+
"syslog_hostname" => "hostname",
|
276
|
+
"syslog_tag" => "tag",
|
277
|
+
"cef_version" => "0",
|
278
|
+
"cef_device_vendor" => "Vendor",
|
279
|
+
"cef_device_product" => "Product",
|
280
|
+
"cef_device_version" => "Version",
|
281
|
+
"cef_device_event_class_id" => "ID",
|
282
|
+
"cef_name" => "Name",
|
283
|
+
"cef_severity" => "Severity",
|
284
|
+
"cs1" => "test" }]}
|
285
|
+
end
|
158
286
|
end
|
159
287
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -20,6 +20,16 @@
|
|
20
20
|
$LOAD_PATH.unshift(File.expand_path('../lib', __FILE__))
|
21
21
|
$LOAD_PATH.unshift(File.expand_path('../spec', __FILE__))
|
22
22
|
|
23
|
+
require 'simplecov'
|
24
|
+
require 'coveralls'
|
25
|
+
Coveralls.wear!
|
26
|
+
|
27
|
+
SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
|
28
|
+
SimpleCov::Formatter::HTMLFormatter,
|
29
|
+
Coveralls::SimpleCov::Formatter
|
30
|
+
]
|
31
|
+
SimpleCov.start
|
32
|
+
|
23
33
|
RSpec.configure do |config|
|
24
34
|
# rspec-expectations config goes here. You can use an alternate
|
25
35
|
# assertion/expectation library such as wrong or the stdlib/minitest
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-parser_cef
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tomoyuki Sugimura
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-01-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -108,6 +108,7 @@ extensions: []
|
|
108
108
|
extra_rdoc_files: []
|
109
109
|
files:
|
110
110
|
- ".codeclimate.yml"
|
111
|
+
- ".coveralls.yml"
|
111
112
|
- ".gitignore"
|
112
113
|
- ".rspec"
|
113
114
|
- ".rubocop.yml"
|