fluent-plugin-grok-parser 2.3.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: bb787befdf7b8b3d1451c48cff1462e0f3ca40db0bf3c97b3a76ae112e509cdf
4
- data.tar.gz: f783cd9143f07e7c5c95cc5da9aeb3a80c20549befa29ca8bcc7814b31fde0ee
3
+ metadata.gz: 7ee0a491eb3c582a83f228c34648869c693cf028aba0beed0404f0ffa3dff182
4
+ data.tar.gz: b421584afadea006497e075a5e198bbd2497654cb446148e1223320172a85aac
5
5
  SHA512:
6
- metadata.gz: 3052cb7600aeeea43ac4c2d7ccf610cfc552e8d88c5e309487d90a843a054ff57d94ae139effa2d46e343a95a095b3c18b42f9d447d4e2b8e644103fd0ad2b6f
7
- data.tar.gz: 9b48ea10a3f51661f9343d691271a0ac55a5c971834033401b54ecd6a001507e414bc5240d3d5ccaf4699e8b6d410d2142c697a269abb1fd7d855fea57adfa84
6
+ metadata.gz: 0db5d145e6d33b285393c75a8383b93120ba623f99859356831e2b08622ca8d1c12ac37d9597ea5fc123d2997c7d42d6b75f4015e9502eaccc830fe41a1919db
7
+ data.tar.gz: 55d52fd681a761529947a779ebec12de1b7468e1d95546b086fd34d1dcaba6a4122726a954ddbc2f26cd872ca401ec0d6a4ce1294118b8db4bc48a47df79fade
@@ -1,10 +1,15 @@
1
1
  version: '{build}'
2
+
3
+ # init:
4
+ # - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
5
+
2
6
  install:
3
7
  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
4
- - "%devkit%\\devkitvars.bat"
8
+ - IF %ridk%==0 "%devkit%\\devkitvars.bat"
5
9
  - ruby --version
6
10
  - gem --version
7
- - bundle install
11
+ - IF %ridk%==0 bundle install
12
+ - IF %ridk%==1 ridk.cmd exec bundle install
8
13
  build: off
9
14
  test_script:
10
15
  - bundle exec rake test
@@ -12,20 +17,34 @@ test_script:
12
17
  # https://www.appveyor.com/docs/installed-software/#ruby
13
18
  environment:
14
19
  matrix:
20
+ - ruby_version: "25-x64"
21
+ ridk: 1
22
+ - ruby_version: "25"
23
+ ridk: 1
24
+ - ruby_version: "24-x64"
25
+ ridk: 1
26
+ - ruby_version: "24"
27
+ ridk: 1
15
28
  - ruby_version: "23-x64"
16
29
  devkit: C:\Ruby23-x64\DevKit
30
+ ridk: 0
17
31
  - ruby_version: "23"
18
32
  devkit: C:\Ruby23\DevKit
33
+ ridk: 0
19
34
  - ruby_version: "22-x64"
20
35
  devkit: C:\Ruby23-x64\DevKit
36
+ ridk: 0
21
37
  - ruby_version: "21-x64"
22
38
  devkit: C:\Ruby23-x64\DevKit
39
+ ridk: 0
23
40
  - ruby_version: "22"
24
41
  devkit: C:\Ruby23\DevKit
25
42
  WIN_RAPID: true
43
+ ridk: 0
26
44
  - ruby_version: "21"
27
45
  devkit: C:\Ruby23\DevKit
28
46
  WIN_RAPID: true
47
+ ridk: 0
29
48
  matrix:
30
49
  allow_failures:
31
50
  - ruby_version: "21"
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-grok-parser"
7
- spec.version = "2.3.0"
7
+ spec.version = "2.4.0"
8
8
  spec.authors = ["kiyoto", "Kenji Okimoto"]
9
9
  spec.email = ["kiyoto@treasure-data.com", "okimoto@clear-code.com"]
10
10
  spec.summary = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}
@@ -19,5 +19,5 @@ Gem::Specification.new do |spec|
19
19
  spec.add_development_dependency "bundler"
20
20
  spec.add_development_dependency "rake"
21
21
  spec.add_development_dependency "test-unit", ">=3.1.5"
22
- spec.add_runtime_dependency "fluentd", ">=0.14.6"
22
+ spec.add_runtime_dependency "fluentd", ">=0.14.6", "< 2"
23
23
  end
@@ -36,6 +36,9 @@ module Fluent
36
36
  if @plugin.respond_to?(:keep_time_key)
37
37
  @keep_time_key = @plugin.keep_time_key
38
38
  end
39
+ if @plugin.respond_to?(:time_format)
40
+ @time_format = @plugin.time_format
41
+ end
39
42
  end
40
43
 
41
44
  def add_patterns_from_file(path)
@@ -48,10 +51,10 @@ module Fluent
48
51
 
49
52
  def setup
50
53
  if @plugin.grok_pattern
51
- @parsers[:grok_pattern] = expand_pattern_expression(@plugin.grok_pattern, @conf)
54
+ @parsers[:grok_pattern] = expand_pattern_expression_grok_pattern(@plugin.grok_pattern, @conf)
52
55
  else
53
56
  @plugin.grok_confs.each.with_index do |grok_conf, index|
54
- @parsers[grok_conf.name || index] = expand_pattern_expression(grok_conf.pattern, grok_conf)
57
+ @parsers[grok_conf.name || index] = expand_pattern_expression_grok_section(grok_conf)
55
58
  end
56
59
  end
57
60
  @parsers.reject! do |key, parser|
@@ -64,7 +67,7 @@ module Fluent
64
67
 
65
68
  private
66
69
 
67
- def expand_pattern_expression(grok_pattern, conf)
70
+ def expand_pattern_expression_grok_pattern(grok_pattern, conf)
68
71
  regexp, types = expand_pattern(grok_pattern)
69
72
  $log.info "Expanded the pattern #{grok_pattern} into #{regexp}"
70
73
  _conf = conf.to_h
@@ -83,6 +86,37 @@ module Fluent
83
86
  nil
84
87
  end
85
88
 
89
+ def expand_pattern_expression_grok_section(conf)
90
+ regexp, types = expand_pattern(conf.pattern)
91
+ $log.info "Expanded the pattern #{conf.pattern} into #{regexp}"
92
+ _conf = conf.to_h
93
+ unless types.empty?
94
+ _conf["types"] = types.map{|subname,type| "#{subname}:#{type}" }.join(",")
95
+ end
96
+ if conf["multiline"] || @multiline_mode
97
+ _conf["multiline"] = conf["multiline"] || @multiline_mode
98
+ end
99
+ if conf["keep_time_key"] || @keep_time_key
100
+ _conf["keep_time_key"] = conf["keep_time_key"] || @keep_time_key
101
+ end
102
+ if conf["time_key"]
103
+ _conf["time_key"] = conf["time_key"]
104
+ end
105
+ if conf["time_format"] || @time_format
106
+ _conf["time_format"] = conf["time_format"] || @time_format
107
+ end
108
+ _conf["expression"] = regexp
109
+ config = Fluent::Config::Element.new("parse", "", _conf, [])
110
+ parser = Fluent::Plugin::RegexpParser.new
111
+ parser.configure(config)
112
+ parser
113
+ rescue GrokPatternNotFoundError => e
114
+ raise e
115
+ rescue => e
116
+ $log.error(error: e)
117
+ nil
118
+ end
119
+
86
120
  def expand_pattern(pattern)
87
121
  # It's okay to modify in place. no need to expand it more than once.
88
122
  type_map = {}
@@ -22,6 +22,9 @@ module Fluent
22
22
  config_param :name, :string, default: nil
23
23
  desc "The pattern of grok"
24
24
  config_param :pattern, :string
25
+ config_param :keep_time_key, :bool, default: false
26
+ config_param :time_key, :string, default: "time"
27
+ config_param :time_format, :string, default: nil
25
28
  end
26
29
 
27
30
  def initialize
@@ -321,6 +321,34 @@ class GrokParserTest < ::Test::Unit::TestCase
321
321
  end
322
322
  end
323
323
 
324
+ sub_test_case "grok section" do
325
+ test "complex pattern" do
326
+ d = create_driver(%[
327
+ <grok>
328
+ pattern %{COMBINEDAPACHELOG}
329
+ time_key timestamp
330
+ time_format %d/%b/%Y:%H:%M:%S %z
331
+ </grok>
332
+ ])
333
+ expected_record = {
334
+ "clientip" => "127.0.0.1",
335
+ "ident" => "192.168.0.1",
336
+ "auth" => "-",
337
+ "verb" => "GET",
338
+ "request" => "/",
339
+ "httpversion" => "1.1",
340
+ "response" => "200",
341
+ "bytes" => "777",
342
+ "referrer" => "\"-\"",
343
+ "agent" => "\"Opera/12.0\""
344
+ }
345
+ d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') do |time, record|
346
+ assert_equal(expected_record, record)
347
+ assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: "%d/%b/%Y:%H:%M:%S %z"), time)
348
+ end
349
+ end
350
+ end
351
+
324
352
  private
325
353
 
326
354
  def create_driver(conf)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-grok-parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.3.0
4
+ version: 2.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - kiyoto
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2018-09-10 00:00:00.000000000 Z
12
+ date: 2018-11-28 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
@@ -60,6 +60,9 @@ dependencies:
60
60
  - - ">="
61
61
  - !ruby/object:Gem::Version
62
62
  version: 0.14.6
63
+ - - "<"
64
+ - !ruby/object:Gem::Version
65
+ version: '2'
63
66
  type: :runtime
64
67
  prerelease: false
65
68
  version_requirements: !ruby/object:Gem::Requirement
@@ -67,6 +70,9 @@ dependencies:
67
70
  - - ">="
68
71
  - !ruby/object:Gem::Version
69
72
  version: 0.14.6
73
+ - - "<"
74
+ - !ruby/object:Gem::Version
75
+ version: '2'
70
76
  description:
71
77
  email:
72
78
  - kiyoto@treasure-data.com