fluent-plugin-parser 0.3.3 → 0.3.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2326d0682f48ee2aa993c41114c4b6135cf610e5
4
- data.tar.gz: aa40d1f259c0781492004e21a79c64118d5ae2cf
3
+ metadata.gz: 11c8d0bf5065c2ea089887da434f380c6477797c
4
+ data.tar.gz: 05dd0a148aed3f8020eb839fd5042298464296ed
5
5
  SHA512:
6
- metadata.gz: 10d523349d0c40468ce83b886e74bcbb2d89d3138cf36d09858c3302d398d67b0114e8b1693407b8fcc774ea9b2114d3b94cc8d560fcd4045a885f62f15d598b
7
- data.tar.gz: cc0a2694b738d268faff7bdf3d7da786da0ffd785662ff130da252aa18b3c1cf0e6931b5417dae4047cae5ef5ee34b60163fbf6da26f505f60a6acbb2543aa3f
6
+ metadata.gz: 4bffef34b921fe845581381aa759fdeea68bf40ea7b95031f7e8909bd5516828f9b31964b550048bd1507e2dd337394e3c2f51ca38afd814c37c17c610353d40
7
+ data.tar.gz: 3471c464f75070f30e4ce2bd97e6bb5a10662e779d7cce00b4d0f41f2efdf6d5ba60c77f36a73ad46a81aa48f17ea10ff0e3f6527f9e1f82b7872ebb47fb1fbf
@@ -1,7 +1,7 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  Gem::Specification.new do |gem|
3
3
  gem.name = "fluent-plugin-parser"
4
- gem.version = "0.3.3"
4
+ gem.version = "0.3.4"
5
5
  gem.authors = ["TAGOMORI Satoshi"]
6
6
  gem.email = ["tagomoris@gmail.com"]
7
7
  gem.description = %q{fluentd plugin to parse single field, or to combine log structure into single field}
@@ -11,10 +11,18 @@ class FluentExt::TextParser
11
11
  config_param :time_format, :string, :default => nil
12
12
  config_param :time_parse, :bool, :default => true
13
13
 
14
- @cache1_key = nil
15
- @cache1_time = nil
16
- @cache2_key = nil
17
- @cache2_time = nil
14
+ attr_accessor :log
15
+
16
+ def initialize
17
+ super
18
+
19
+ @cache1_key = nil
20
+ @cache1_time = nil
21
+ @cache2_key = nil
22
+ @cache2_time = nil
23
+
24
+ @log = nil
25
+ end
18
26
 
19
27
  def parse_time(record)
20
28
  time = nil
@@ -40,7 +48,7 @@ class FluentExt::TextParser
40
48
  @cache2_key = value
41
49
  @cache2_time = time
42
50
  rescue TypeError, ArgumentError => e
43
- $log.warn "Failed to parse time", :key => @time_key, :value => value
51
+ @log.warn "Failed to parse time", :key => @time_key, :value => value
44
52
  record[@time_key] = value
45
53
  end
46
54
  end
@@ -67,7 +75,7 @@ class FluentExt::TextParser
67
75
  m = @regexp.match(text)
68
76
  unless m
69
77
  unless @suppress_parse_error_log
70
- $log.warn "pattern not match: #{text}"
78
+ @log.warn "pattern not match: #{text}"
71
79
  end
72
80
 
73
81
  return nil, nil
@@ -87,7 +95,7 @@ class FluentExt::TextParser
87
95
  return parse_time(record)
88
96
  rescue Yajl::ParseError
89
97
  unless @suppress_parse_error_log
90
- $log.warn "pattern not match(json): #{text.inspect}: #{$!}"
98
+ @log.warn "pattern not match(json): #{text.inspect}: #{$!}"
91
99
  end
92
100
 
93
101
  return nil, nil
@@ -133,16 +141,23 @@ class FluentExt::TextParser
133
141
  end
134
142
  end
135
143
 
136
- class ApacheParser
144
+ class ApacheParser < GenericParser
137
145
  include Fluent::Configurable
138
146
 
139
147
  REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
140
148
 
149
+ def initialize
150
+ super
151
+
152
+ @time_key = "time"
153
+ @time_format = "%d/%b/%Y:%H:%M:%S %z"
154
+ end
155
+
141
156
  def call(text)
142
157
  m = REGEXP.match(text)
143
158
  unless m
144
159
  unless @suppress_parse_error_log
145
- $log.warn "pattern not match: #{text.inspect}"
160
+ @log.warn "pattern not match: #{text.inspect}"
146
161
  end
147
162
 
148
163
  return nil, nil
@@ -155,7 +170,6 @@ class FluentExt::TextParser
155
170
  user = (user == '-') ? nil : user
156
171
 
157
172
  time = m['time']
158
- time = Time.strptime(time, "%d/%b/%Y:%H:%M:%S %z").to_i
159
173
 
160
174
  method = m['method']
161
175
  path = m['path']
@@ -173,6 +187,7 @@ class FluentExt::TextParser
173
187
  agent = (agent == '-') ? nil : agent
174
188
 
175
189
  record = {
190
+ "time" => time,
176
191
  "host" => host,
177
192
  "user" => user,
178
193
  "method" => method,
@@ -183,7 +198,7 @@ class FluentExt::TextParser
183
198
  "agent" => agent,
184
199
  }
185
200
 
186
- return time, record
201
+ parse_time(record)
187
202
  end
188
203
  end
189
204
 
@@ -209,7 +224,11 @@ class FluentExt::TextParser
209
224
  TEMPLATE_FACTORIES[name] = factory
210
225
  end
211
226
 
212
- def initialize
227
+ attr_accessor :log
228
+ attr_reader :parser
229
+
230
+ def initialize(logger)
231
+ @log = logger
213
232
  @parser = nil
214
233
  end
215
234
 
@@ -246,6 +265,8 @@ class FluentExt::TextParser
246
265
 
247
266
  end
248
267
 
268
+ @parser.log = @log
269
+
249
270
  if @parser.respond_to?(:configure)
250
271
  @parser.configure(conf)
251
272
  end
@@ -12,6 +12,8 @@ class Fluent::ParserOutput < Fluent::Output
12
12
  config_param :replace_invalid_sequence, :bool, :default => false
13
13
  config_param :hash_value_field, :string, :default => nil
14
14
 
15
+ attr_reader :parser
16
+
15
17
  def initialize
16
18
  super
17
19
  require 'time'
@@ -39,7 +41,7 @@ class Fluent::ParserOutput < Fluent::Output
39
41
  @added_prefix_string = @add_prefix + '.'
40
42
  end
41
43
 
42
- @parser = FluentExt::TextParser.new
44
+ @parser = FluentExt::TextParser.new(log())
43
45
  @parser.configure(conf)
44
46
  end
45
47
 
@@ -617,51 +617,74 @@ class ParserOutputTest < Test::Unit::TestCase
617
617
  end
618
618
  end
619
619
 
620
+ def swap_logger(instance)
621
+ raise "use with block" unless block_given?
622
+ parser_logger = instance.parser.log
623
+ dummy = DummyLogger.new
624
+ instance.parser.log = dummy
625
+ instance.parser.parser.log = dummy
626
+
627
+ restore = if instance.respond_to?("log=".to_sym)
628
+ saved_logger = instance.log
629
+ instance.log = dummy
630
+ lambda{ instance.log = saved_logger; instance.parser.log = instance.parser.parser.log = parser_logger }
631
+ else
632
+ saved_logger = $log
633
+ $log = dummy
634
+ lambda{ $log = saved_logger; instance.parser.log = instance.parser.parser.log = parser_logger }
635
+ end
636
+
637
+ yield
638
+
639
+ restore.call
640
+ end
641
+
620
642
  def test_suppress_parse_error_log
621
643
  # default(disabled) 'suppress_parse_error_log' is not specify
622
644
  d = create_driver(CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
623
645
 
624
- saved_logger = $log
625
- $log = DummyLogger.new
626
-
627
- assert_raise(DummyLoggerWarnedException) {
628
- d.run do
629
- d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
630
- end
631
- }
632
-
633
- assert_nothing_raised {
634
- d.run do
635
- d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
636
- end
637
- }
646
+ swap_logger(d.instance) do
647
+ assert_raise(DummyLoggerWarnedException) {
648
+ d.run do
649
+ d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
650
+ end
651
+ }
652
+
653
+ assert_nothing_raised {
654
+ d.run do
655
+ d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
656
+ end
657
+ }
658
+ end
638
659
 
639
660
  # disabled 'suppress_parse_error_log'
640
661
  d = create_driver(CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
641
662
 
642
- assert_raise(DummyLoggerWarnedException) {
643
- d.run do
644
- d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
645
- end
646
- }
647
-
648
- assert_nothing_raised {
649
- d.run do
650
- d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
651
- end
652
- }
663
+ swap_logger(d.instance) do
664
+ assert_raise(DummyLoggerWarnedException) {
665
+ d.run do
666
+ d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
667
+ end
668
+ }
669
+
670
+ assert_nothing_raised {
671
+ d.run do
672
+ d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
673
+ end
674
+ }
675
+ end
653
676
 
654
677
  # enabled 'suppress_parse_error_log'
655
678
  d = create_driver(CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
656
679
 
657
- assert_nothing_raised {
658
- d.run do
659
- d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
660
- d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
661
- end
662
- }
663
-
664
- $log = saved_logger
680
+ swap_logger(d.instance) do
681
+ assert_nothing_raised {
682
+ d.run do
683
+ d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
684
+ d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
685
+ end
686
+ }
687
+ end
665
688
  end
666
689
 
667
690
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.3
4
+ version: 0.3.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - TAGOMORI Satoshi