fluent-plugin-ltsv-parser 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |gem|
|
3
3
|
gem.name = "fluent-plugin-ltsv-parser"
|
4
|
-
gem.version = "0.0.
|
4
|
+
gem.version = "0.0.2"
|
5
5
|
gem.authors = ["anarcher"]
|
6
6
|
gem.email = ["anarcher@gmail.com"]
|
7
7
|
gem.description = %q{fluentd plugin to ltsv parse single field, or to combine log structure into single field}
|
@@ -1,4 +1,3 @@
|
|
1
|
-
require_relative './parser'
|
2
1
|
|
3
2
|
class Fluent::ParserOutput < Fluent::Output
|
4
3
|
Fluent::Plugin.register_output('ltsv_parser', self)
|
@@ -8,8 +7,6 @@ class Fluent::ParserOutput < Fluent::Output
|
|
8
7
|
config_param :filter_in, :string , :default => ""
|
9
8
|
config_param :add_prefix, :string ,:default => nil
|
10
9
|
|
11
|
-
attr_reader :parser
|
12
|
-
|
13
10
|
def initialize
|
14
11
|
super
|
15
12
|
require 'time'
|
@@ -36,12 +33,11 @@ class Fluent::ParserOutput < Fluent::Output
|
|
36
33
|
end
|
37
34
|
es.each do |time,record|
|
38
35
|
raw_value = record[@key_name]
|
39
|
-
|
40
|
-
t ||= time
|
36
|
+
values = raw_value ? filter(parse(raw_value)) : nil
|
41
37
|
|
42
38
|
r = @reserve_data ? record.merge(values) : values
|
43
39
|
if r
|
44
|
-
Fluent::Engine.emit(tag,
|
40
|
+
Fluent::Engine.emit(tag,time,r)
|
45
41
|
end
|
46
42
|
end
|
47
43
|
|
@@ -54,10 +50,10 @@ class Fluent::ParserOutput < Fluent::Output
|
|
54
50
|
if @filter_in.length > 0 then
|
55
51
|
record.select{ |x| @filter_in.include? x }
|
56
52
|
end
|
57
|
-
|
53
|
+
nil
|
58
54
|
end
|
59
55
|
|
60
|
-
def parse(
|
61
|
-
return
|
56
|
+
def parse(text)
|
57
|
+
return Hash[text.split("\t").map{|p| p.split(":", 2)}]
|
62
58
|
end
|
63
59
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-ltsv-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -58,7 +58,6 @@ files:
|
|
58
58
|
- Rakefile
|
59
59
|
- fluent-plugin-ltsv-parser.gemspec
|
60
60
|
- lib/fluent/plugin/out_ltsv_parser.rb
|
61
|
-
- lib/fluent/plugin/parser.rb
|
62
61
|
homepage: https://github.com/anarcher/fluent-plugin-ltsv-parser
|
63
62
|
licenses:
|
64
63
|
- APLv2
|
data/lib/fluent/plugin/parser.rb
DELETED
@@ -1,67 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# This module is copied from fluentd/lib/fluent/parser.rb and
|
3
|
-
# fixed not to overwrite 'time' (reserve nil) when time not found in parsed string.
|
4
|
-
|
5
|
-
module FluentExt
|
6
|
-
class GenericParser
|
7
|
-
include Fluent::Configurable
|
8
|
-
|
9
|
-
config_param :time_key, :string, :default => 'time'
|
10
|
-
config_param :time_format, :string, :default => nil
|
11
|
-
config_param :time_parse, :bool, :default => true
|
12
|
-
|
13
|
-
attr_accessor :log
|
14
|
-
|
15
|
-
def initialize(logger)
|
16
|
-
super()
|
17
|
-
|
18
|
-
@cache1_key = nil
|
19
|
-
@cache1_time = nil
|
20
|
-
@cache2_key = nil
|
21
|
-
@cache2_time = nil
|
22
|
-
|
23
|
-
@log = logger
|
24
|
-
end
|
25
|
-
|
26
|
-
def parse_time(record)
|
27
|
-
time = nil
|
28
|
-
|
29
|
-
unless @time_parse
|
30
|
-
return time, record
|
31
|
-
end
|
32
|
-
|
33
|
-
if value = record.delete(@time_key)
|
34
|
-
if @cache1_key == value
|
35
|
-
time = @cache1_time
|
36
|
-
elsif @cache2_key == value
|
37
|
-
time = @cache2_time
|
38
|
-
else
|
39
|
-
begin
|
40
|
-
time = if @time_format
|
41
|
-
Time.strptime(value, @time_format).to_i
|
42
|
-
else
|
43
|
-
Time.parse(value).to_i
|
44
|
-
end
|
45
|
-
@cache1_key = @cache2_key
|
46
|
-
@cache1_time = @cache2_time
|
47
|
-
@cache2_key = value
|
48
|
-
@cache2_time = time
|
49
|
-
rescue TypeError, ArgumentError => e
|
50
|
-
@log.warn "Failed to parse time", :key => @time_key, :value => value
|
51
|
-
record[@time_key] = value
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
return time, record
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
class LTSVParser < GenericParser
|
61
|
-
def parse(text)
|
62
|
-
record = Hash[text.split("\t").map{|p| p.split(":", 2)}]
|
63
|
-
return parse_time(record)
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
|
-
end
|