fluent-plugin-json-parser 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +4 -0
- data/Rakefile +11 -0
- data/fluent-plugin-json-parser.gemspec +19 -0
- data/lib/fluent/plugin/out_json_parser.rb +51 -0
- data/lib/fluent/plugin/parser.rb +72 -0
- metadata +6 -1
data/Gemfile
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
Gem::Specification.new do |gem|
|
3
|
+
gem.name = "fluent-plugin-json-parser"
|
4
|
+
gem.version = "0.0.2"
|
5
|
+
gem.authors = ["anarcher"]
|
6
|
+
gem.email = ["anarcher@gmail.com"]
|
7
|
+
gem.description = %q{fluentd plugin to json parse single field, or to combine log structure into single field}
|
8
|
+
gem.summary = %q{plugin to parse/combine fluentd log messages}
|
9
|
+
gem.homepage = "https://github.com/anarcher/fluent-plugin-json-parser"
|
10
|
+
gem.license = "APLv2"
|
11
|
+
|
12
|
+
gem.files = `git ls-files`.split($\)
|
13
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
14
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
|
17
|
+
gem.add_development_dependency "rake"
|
18
|
+
gem.add_runtime_dependency "fluentd"
|
19
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require_relative './parser'
|
2
|
+
|
3
|
+
class Fluent::ParserOutput < Fluent::Output
|
4
|
+
Fluent::Plugin.register_output('json_parser', self)
|
5
|
+
config_param :tag, :string, :default => nil
|
6
|
+
config_param :reserve_data, :bool, :default => false
|
7
|
+
config_param :key_name, :string
|
8
|
+
|
9
|
+
attr_reader :parser
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
super
|
13
|
+
require 'time'
|
14
|
+
end
|
15
|
+
|
16
|
+
# Define `log` method for v0.10.42 or earlier
|
17
|
+
unless method_defined?(:log)
|
18
|
+
define_method("log") { $log }
|
19
|
+
end
|
20
|
+
|
21
|
+
def configure(conf)
|
22
|
+
super
|
23
|
+
if @key_name[0] == ":"
|
24
|
+
@key_name = @key_name[1..-1].to_sym
|
25
|
+
end
|
26
|
+
@parser = FluentExt::JSONParser.new(log())
|
27
|
+
end
|
28
|
+
|
29
|
+
def emit(tag,es,chain)
|
30
|
+
tag = @tag || tag
|
31
|
+
es.each do |time,record|
|
32
|
+
raw_value = record[@key_name]
|
33
|
+
t,values = raw_value ? parse(raw_value) : [nil,nil]
|
34
|
+
t ||= time
|
35
|
+
|
36
|
+
r = @reserve_data ? record.merge(r) : values
|
37
|
+
|
38
|
+
if r
|
39
|
+
Fluent::Engine.emit(tag,t,r)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
chain.next
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def parse(string)
|
49
|
+
return @parser.parse(string)
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
#
|
2
|
+
# This module is copied from fluentd/lib/fluent/parser.rb and
|
3
|
+
# fixed not to overwrite 'time' (reserve nil) when time not found in parsed string.
|
4
|
+
|
5
|
+
module FluentExt
|
6
|
+
class GenericParser
|
7
|
+
include Fluent::Configurable
|
8
|
+
|
9
|
+
config_param :time_key, :string, :default => 'time'
|
10
|
+
config_param :time_format, :string, :default => nil
|
11
|
+
config_param :time_parse, :bool, :default => true
|
12
|
+
|
13
|
+
attr_accessor :log
|
14
|
+
|
15
|
+
def initialize(logger)
|
16
|
+
super()
|
17
|
+
|
18
|
+
@cache1_key = nil
|
19
|
+
@cache1_time = nil
|
20
|
+
@cache2_key = nil
|
21
|
+
@cache2_time = nil
|
22
|
+
|
23
|
+
@log = logger
|
24
|
+
end
|
25
|
+
|
26
|
+
def parse_time(record)
|
27
|
+
time = nil
|
28
|
+
|
29
|
+
unless @time_parse
|
30
|
+
return time, record
|
31
|
+
end
|
32
|
+
|
33
|
+
if value = record.delete(@time_key)
|
34
|
+
if @cache1_key == value
|
35
|
+
time = @cache1_time
|
36
|
+
elsif @cache2_key == value
|
37
|
+
time = @cache2_time
|
38
|
+
else
|
39
|
+
begin
|
40
|
+
time = if @time_format
|
41
|
+
Time.strptime(value, @time_format).to_i
|
42
|
+
else
|
43
|
+
Time.parse(value).to_i
|
44
|
+
end
|
45
|
+
@cache1_key = @cache2_key
|
46
|
+
@cache1_time = @cache2_time
|
47
|
+
@cache2_key = value
|
48
|
+
@cache2_time = time
|
49
|
+
rescue TypeError, ArgumentError => e
|
50
|
+
@log.warn "Failed to parse time", :key => @time_key, :value => value
|
51
|
+
record[@time_key] = value
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
return time, record
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
class JSONParser < GenericParser
|
61
|
+
def parse(text)
|
62
|
+
record = Yajl.load(text)
|
63
|
+
return parse_time(record)
|
64
|
+
rescue Yajl::ParseError
|
65
|
+
unless @suppress_parse_error_log
|
66
|
+
@log.warn "pattern not match(json): #{text.inspect}: #{$!}"
|
67
|
+
end
|
68
|
+
return nil, nil
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-json-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -52,7 +52,12 @@ extensions: []
|
|
52
52
|
extra_rdoc_files: []
|
53
53
|
files:
|
54
54
|
- .gitignore
|
55
|
+
- Gemfile
|
55
56
|
- LICENSE
|
57
|
+
- Rakefile
|
58
|
+
- fluent-plugin-json-parser.gemspec
|
59
|
+
- lib/fluent/plugin/out_json_parser.rb
|
60
|
+
- lib/fluent/plugin/parser.rb
|
56
61
|
homepage: https://github.com/anarcher/fluent-plugin-json-parser
|
57
62
|
licenses:
|
58
63
|
- APLv2
|