fluent-plugin-time_parser 0.0.6 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.travis.yml +13 -0
- data/LICENSE +1 -1
- data/README.md +7 -5
- data/fluent-plugin-time_parser.gemspec +4 -4
- data/lib/fluent/plugin/out_time_parser.rb +20 -17
- data/test/plugin/test_out_time_parser.rb +39 -24
- data/test/test_helper.rb +3 -0
- metadata +32 -36
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 581b6c1bcc8546fb9c4de7e835100c3c4ccf8325
|
4
|
+
data.tar.gz: 54288fe1474709bbd11a2cdfa70b9b8e3575b8ac
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 58171ffa45dada9b15dd4a2b53c25bff4144e7ff20e785991a3dc55dd7eeb3f44b3fa8760a502c8fce45c5fe47d5ba369c80c2b5481da993f2fe969f9bc3c530
|
7
|
+
data.tar.gz: 476cc0d39402b799ab0ef320e97792bb012f70056ce88b8dbe50c202e8eac3f2c72c0c5e0358ababfd003e9f8835941082d2669381385afcccf405096bc5ff98
|
data/.travis.yml
ADDED
data/LICENSE
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
Copyright (c)
|
1
|
+
Copyright (c) 2017 Carlos Donderis
|
2
2
|
Apache License, Version 2.0
|
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# fluent-plugin-time_parser
|
1
|
+
# fluent-plugin-time_parser, a plugin for [Fluentd](http://fluentd.org)
|
2
2
|
|
3
3
|
## Component
|
4
4
|
TimeParserOutput
|
@@ -13,7 +13,7 @@ Will take a time attribute and will extract the date and the hour for a given ti
|
|
13
13
|
type time_parser
|
14
14
|
|
15
15
|
key time
|
16
|
-
|
16
|
+
tag extracted.${tag}
|
17
17
|
time_zone Asia/Tokyo
|
18
18
|
</match>
|
19
19
|
```
|
@@ -43,11 +43,13 @@ Then you'll get re-emmited tags/records like so:
|
|
43
43
|
|
44
44
|
`key` is used to point a key whose value contains the time you want to parse.
|
45
45
|
|
46
|
-
###
|
46
|
+
### tag
|
47
47
|
|
48
|
-
|
48
|
+
In this param, users can write placeholders.
|
49
49
|
|
50
|
-
|
50
|
+
Please use placeholders ${tag}, ${tag[0]}, ${tag[1]} in configuration.
|
51
|
+
|
52
|
+
Note that `Fluent::HandleTagNameMixin` dependency is removed in v0.14 style of this plugin.
|
51
53
|
|
52
54
|
### time_zone
|
53
55
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |gem|
|
2
2
|
gem.name = 'fluent-plugin-time_parser'
|
3
|
-
gem.version = '0.0
|
4
|
-
gem.authors = ['Carlos Donderis', 'Michael H. Oshita']
|
3
|
+
gem.version = '0.1.0'
|
4
|
+
gem.authors = ['Carlos Donderis', 'Michael H. Oshita', 'Hiroshi Hatake']
|
5
5
|
gem.email = ['cdonderis@gmail.com', 'ijinpublic+github@gmail.com']
|
6
6
|
gem.homepage = 'http://github.com/cads/fluent-plugin-time_parser'
|
7
7
|
gem.description = %q{Fluentd plugin to parse the time parameter.}
|
@@ -13,8 +13,8 @@ Gem::Specification.new do |gem|
|
|
13
13
|
gem.require_paths = ['lib']
|
14
14
|
|
15
15
|
gem.add_development_dependency 'rake'
|
16
|
-
gem.add_development_dependency 'fluentd'
|
17
16
|
gem.add_development_dependency 'tzinfo'
|
17
|
+
gem.add_development_dependency 'test-unit', '~> 3.2'
|
18
18
|
gem.add_runtime_dependency 'tzinfo'
|
19
|
-
gem.add_runtime_dependency 'fluentd'
|
19
|
+
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
|
20
20
|
end
|
@@ -1,27 +1,31 @@
|
|
1
1
|
require 'time'
|
2
2
|
require 'tzinfo'
|
3
|
+
require 'fluent/plugin/output'
|
3
4
|
|
4
|
-
module Fluent
|
5
|
+
module Fluent::Plugin
|
5
6
|
class TimeParserOutput < Output
|
6
|
-
include Fluent::HandleTagNameMixin
|
7
7
|
Fluent::Plugin.register_output('time_parser', self)
|
8
8
|
|
9
|
+
helpers :event_emitter, :compat_parameters
|
10
|
+
|
11
|
+
DEFAULT_BUFFER_TYPE = "memory"
|
12
|
+
|
13
|
+
config_param :tag, :string
|
9
14
|
config_param :key, :string, :default => 'time'
|
10
15
|
config_param :time_zone, :string, :default => ''
|
11
16
|
config_param :parsed_time_tag, :string, :default => 'parsed_time'
|
12
17
|
config_param :parsed_hour_tag, :string, :default => 'parsed_hour'
|
13
18
|
config_param :parsed_date_tag, :string, :default => 'parsed_date'
|
14
19
|
|
20
|
+
config_section :buffer do
|
21
|
+
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
22
|
+
config_set_default :chunk_keys, ['tag']
|
23
|
+
end
|
24
|
+
|
15
25
|
def configure(conf)
|
26
|
+
compat_parameters_convert(conf, :buffer)
|
16
27
|
super
|
17
|
-
if
|
18
|
-
!remove_tag_prefix &&
|
19
|
-
!remove_tag_suffix &&
|
20
|
-
!add_tag_prefix &&
|
21
|
-
!add_tag_suffix
|
22
|
-
)
|
23
|
-
raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
|
24
|
-
end
|
28
|
+
raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
|
25
29
|
end
|
26
30
|
|
27
31
|
def start
|
@@ -32,13 +36,13 @@ module Fluent
|
|
32
36
|
super
|
33
37
|
end
|
34
38
|
|
35
|
-
def
|
36
|
-
|
39
|
+
def write(chunk)
|
40
|
+
tag = extract_placeholders(@tag, chunk.metadata)
|
41
|
+
chunk.msgpack_each {|time,record|
|
37
42
|
t = tag.dup
|
38
43
|
filter_record(t, time, record)
|
39
|
-
|
44
|
+
router.emit(t, time, record)
|
40
45
|
}
|
41
|
-
chain.next
|
42
46
|
end
|
43
47
|
|
44
48
|
def filter_record(tag, time, record)
|
@@ -63,11 +67,10 @@ module Fluent
|
|
63
67
|
record[parsed_hour_tag] = hour
|
64
68
|
|
65
69
|
rescue ArgumentError => error
|
66
|
-
|
70
|
+
log.warn("out_extract_query_params: #{error.message}")
|
67
71
|
rescue TZInfo::InvalidTimezoneIdentifier
|
68
|
-
|
72
|
+
log.warn("Timezone Not Valid, please refer to http://tzinfo.rubyforge.org/doc/classes/TZInfo/Timezone.html for valid timezones")
|
69
73
|
end
|
70
|
-
super(tag, time, record)
|
71
74
|
end
|
72
75
|
end
|
73
76
|
end
|
@@ -12,29 +12,29 @@ class TimeParserOutputTest < Test::Unit::TestCase
|
|
12
12
|
Fluent::Test.setup
|
13
13
|
end
|
14
14
|
|
15
|
-
def create_driver(conf
|
16
|
-
Fluent::Test::
|
17
|
-
Fluent::TimeParserOutput
|
15
|
+
def create_driver(conf)
|
16
|
+
Fluent::Test::Driver::Output.new(
|
17
|
+
Fluent::Plugin::TimeParserOutput
|
18
18
|
).configure(conf)
|
19
19
|
end
|
20
20
|
|
21
21
|
def test_configure
|
22
22
|
d = create_driver(%[
|
23
23
|
key test
|
24
|
-
|
24
|
+
tag extracted.${tag}
|
25
25
|
time_zone Asia/Tokyo
|
26
26
|
])
|
27
27
|
assert_equal 'test', d.instance.key
|
28
|
-
assert_equal 'extracted
|
28
|
+
assert_equal 'extracted.${tag}', d.instance.tag
|
29
29
|
assert_equal 'Asia/Tokyo', d.instance.time_zone
|
30
30
|
|
31
31
|
#Default Key
|
32
32
|
d = create_driver(%[
|
33
|
-
|
33
|
+
tag extracted.${tag}
|
34
34
|
time_zone Asia/Tokyo
|
35
35
|
])
|
36
36
|
assert_equal 'time', d.instance.key
|
37
|
-
assert_equal 'extracted
|
37
|
+
assert_equal 'extracted.${tag}', d.instance.tag
|
38
38
|
assert_equal 'Asia/Tokyo', d.instance.time_zone
|
39
39
|
|
40
40
|
#No Prefix
|
@@ -47,16 +47,16 @@ class TimeParserOutputTest < Test::Unit::TestCase
|
|
47
47
|
#No TimeZone
|
48
48
|
d = create_driver(%[
|
49
49
|
key test
|
50
|
-
|
50
|
+
tag extracted.${tag}
|
51
51
|
])
|
52
52
|
assert_equal 'test', d.instance.key
|
53
|
-
assert_equal 'extracted
|
53
|
+
assert_equal 'extracted.${tag}', d.instance.tag
|
54
54
|
end
|
55
55
|
|
56
56
|
def test_filter_record
|
57
57
|
d = create_driver(%[
|
58
58
|
key time
|
59
|
-
|
59
|
+
tag extracted.${tag}
|
60
60
|
time_zone Asia/Tokyo
|
61
61
|
])
|
62
62
|
tag = 'test'
|
@@ -72,7 +72,7 @@ class TimeParserOutputTest < Test::Unit::TestCase
|
|
72
72
|
def test_girigiri_records
|
73
73
|
d = create_driver(%[
|
74
74
|
key time
|
75
|
-
|
75
|
+
tag extracted.${tag}
|
76
76
|
time_zone Japan
|
77
77
|
])
|
78
78
|
tag = 'test'
|
@@ -88,7 +88,7 @@ class TimeParserOutputTest < Test::Unit::TestCase
|
|
88
88
|
def test_filter_record_bad_parameters
|
89
89
|
d = create_driver(%[
|
90
90
|
key time
|
91
|
-
|
91
|
+
tag extracted.${tag}
|
92
92
|
time_zone myPlace
|
93
93
|
])
|
94
94
|
tag = 'test'
|
@@ -109,31 +109,46 @@ class TimeParserOutputTest < Test::Unit::TestCase
|
|
109
109
|
def test_emit
|
110
110
|
d = create_driver(%[
|
111
111
|
key time
|
112
|
-
|
112
|
+
tag extracted.${tag}
|
113
113
|
time_zone Asia/Tokyo
|
114
114
|
])
|
115
115
|
|
116
|
-
d.run { d.
|
117
|
-
emits = d.
|
116
|
+
d.run(default_tag: 'test') { d.feed('time' => TIME) }
|
117
|
+
emits = d.events
|
118
118
|
|
119
119
|
assert_equal 1, emits.count
|
120
120
|
assert_equal 'extracted.test', emits[0][0]
|
121
121
|
assert_equal TIME, emits[0][2]['time']
|
122
122
|
end
|
123
123
|
|
124
|
+
def test_emit_without_tag_prefix
|
125
|
+
d = create_driver(%[
|
126
|
+
key time
|
127
|
+
tag ${tag[1]}
|
128
|
+
time_zone Asia/Tokyo
|
129
|
+
])
|
130
|
+
|
131
|
+
d.run(default_tag: 'removed.test') { d.feed('time' => TIME) }
|
132
|
+
emits = d.events
|
133
|
+
|
134
|
+
assert_equal 1, emits.count
|
135
|
+
assert_equal 'test', emits[0][0]
|
136
|
+
assert_equal TIME, emits[0][2]['time']
|
137
|
+
end
|
138
|
+
|
124
139
|
def test_emit_multi
|
125
140
|
d = create_driver(%[
|
126
141
|
key time
|
127
|
-
|
142
|
+
tag extracted.${tag}
|
128
143
|
time_zone Asia/Tokyo
|
129
144
|
])
|
130
145
|
|
131
|
-
d.run do
|
132
|
-
d.
|
133
|
-
d.
|
134
|
-
d.
|
146
|
+
d.run(default_tag: 'test') do
|
147
|
+
d.feed('time' => TIME)
|
148
|
+
d.feed('time' => TIME)
|
149
|
+
d.feed('time' => TIME)
|
135
150
|
end
|
136
|
-
emits = d.
|
151
|
+
emits = d.events
|
137
152
|
|
138
153
|
assert_equal 3, emits.count
|
139
154
|
0.upto(2) do |i|
|
@@ -145,12 +160,12 @@ class TimeParserOutputTest < Test::Unit::TestCase
|
|
145
160
|
def test_emit_with_invalid_time
|
146
161
|
d = create_driver(%[
|
147
162
|
key time
|
148
|
-
|
163
|
+
tag extracted.${tag}
|
149
164
|
time_zone Asia/Tokyo
|
150
165
|
])
|
151
166
|
wrong_time = 'wrong time'
|
152
|
-
d.run { d.
|
153
|
-
emits = d.
|
167
|
+
d.run(default_tag: 'test') { d.feed('time' => wrong_time) }
|
168
|
+
emits = d.events
|
154
169
|
|
155
170
|
assert_equal 1, emits.count
|
156
171
|
assert_equal 'extracted.test', emits[0][0]
|
data/test/test_helper.rb
CHANGED
@@ -4,6 +4,8 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
|
4
4
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
5
5
|
|
6
6
|
require 'fluent/test'
|
7
|
+
require 'fluent/test/helpers'
|
8
|
+
require 'fluent/test/driver/output'
|
7
9
|
|
8
10
|
unless ENV.has_key?('VERBOSE')
|
9
11
|
nulllogger = Object.new
|
@@ -18,4 +20,5 @@ end
|
|
18
20
|
require 'fluent/plugin/out_time_parser'
|
19
21
|
|
20
22
|
class Test::Unit::TestCase
|
23
|
+
include Fluent::Test::Helpers
|
21
24
|
end
|
metadata
CHANGED
@@ -1,97 +1,93 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-time_parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
5
|
-
prerelease:
|
4
|
+
version: 0.1.0
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Carlos Donderis
|
9
8
|
- Michael H. Oshita
|
9
|
+
- Hiroshi Hatake
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2017-09-20 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: rake
|
17
17
|
requirement: !ruby/object:Gem::Requirement
|
18
|
-
none: false
|
19
18
|
requirements:
|
20
|
-
- -
|
19
|
+
- - ">="
|
21
20
|
- !ruby/object:Gem::Version
|
22
21
|
version: '0'
|
23
22
|
type: :development
|
24
23
|
prerelease: false
|
25
24
|
version_requirements: !ruby/object:Gem::Requirement
|
26
|
-
none: false
|
27
25
|
requirements:
|
28
|
-
- -
|
26
|
+
- - ">="
|
29
27
|
- !ruby/object:Gem::Version
|
30
28
|
version: '0'
|
31
29
|
- !ruby/object:Gem::Dependency
|
32
|
-
name:
|
30
|
+
name: tzinfo
|
33
31
|
requirement: !ruby/object:Gem::Requirement
|
34
|
-
none: false
|
35
32
|
requirements:
|
36
|
-
- -
|
33
|
+
- - ">="
|
37
34
|
- !ruby/object:Gem::Version
|
38
35
|
version: '0'
|
39
36
|
type: :development
|
40
37
|
prerelease: false
|
41
38
|
version_requirements: !ruby/object:Gem::Requirement
|
42
|
-
none: false
|
43
39
|
requirements:
|
44
|
-
- -
|
40
|
+
- - ">="
|
45
41
|
- !ruby/object:Gem::Version
|
46
42
|
version: '0'
|
47
43
|
- !ruby/object:Gem::Dependency
|
48
|
-
name:
|
44
|
+
name: test-unit
|
49
45
|
requirement: !ruby/object:Gem::Requirement
|
50
|
-
none: false
|
51
46
|
requirements:
|
52
|
-
- -
|
47
|
+
- - "~>"
|
53
48
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
49
|
+
version: '3.2'
|
55
50
|
type: :development
|
56
51
|
prerelease: false
|
57
52
|
version_requirements: !ruby/object:Gem::Requirement
|
58
|
-
none: false
|
59
53
|
requirements:
|
60
|
-
- -
|
54
|
+
- - "~>"
|
61
55
|
- !ruby/object:Gem::Version
|
62
|
-
version: '
|
56
|
+
version: '3.2'
|
63
57
|
- !ruby/object:Gem::Dependency
|
64
58
|
name: tzinfo
|
65
59
|
requirement: !ruby/object:Gem::Requirement
|
66
|
-
none: false
|
67
60
|
requirements:
|
68
|
-
- -
|
61
|
+
- - ">="
|
69
62
|
- !ruby/object:Gem::Version
|
70
63
|
version: '0'
|
71
64
|
type: :runtime
|
72
65
|
prerelease: false
|
73
66
|
version_requirements: !ruby/object:Gem::Requirement
|
74
|
-
none: false
|
75
67
|
requirements:
|
76
|
-
- -
|
68
|
+
- - ">="
|
77
69
|
- !ruby/object:Gem::Version
|
78
70
|
version: '0'
|
79
71
|
- !ruby/object:Gem::Dependency
|
80
72
|
name: fluentd
|
81
73
|
requirement: !ruby/object:Gem::Requirement
|
82
|
-
none: false
|
83
74
|
requirements:
|
84
|
-
- -
|
75
|
+
- - ">="
|
85
76
|
- !ruby/object:Gem::Version
|
86
|
-
version:
|
77
|
+
version: 0.14.0
|
78
|
+
- - "<"
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '2'
|
87
81
|
type: :runtime
|
88
82
|
prerelease: false
|
89
83
|
version_requirements: !ruby/object:Gem::Requirement
|
90
|
-
none: false
|
91
84
|
requirements:
|
92
|
-
- -
|
85
|
+
- - ">="
|
93
86
|
- !ruby/object:Gem::Version
|
94
|
-
version:
|
87
|
+
version: 0.14.0
|
88
|
+
- - "<"
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '2'
|
95
91
|
description: Fluentd plugin to parse the time parameter.
|
96
92
|
email:
|
97
93
|
- cdonderis@gmail.com
|
@@ -100,7 +96,8 @@ executables: []
|
|
100
96
|
extensions: []
|
101
97
|
extra_rdoc_files: []
|
102
98
|
files:
|
103
|
-
- .gitignore
|
99
|
+
- ".gitignore"
|
100
|
+
- ".travis.yml"
|
104
101
|
- Gemfile
|
105
102
|
- LICENSE
|
106
103
|
- README.md
|
@@ -111,27 +108,26 @@ files:
|
|
111
108
|
- test/test_helper.rb
|
112
109
|
homepage: http://github.com/cads/fluent-plugin-time_parser
|
113
110
|
licenses: []
|
111
|
+
metadata: {}
|
114
112
|
post_install_message:
|
115
113
|
rdoc_options: []
|
116
114
|
require_paths:
|
117
115
|
- lib
|
118
116
|
required_ruby_version: !ruby/object:Gem::Requirement
|
119
|
-
none: false
|
120
117
|
requirements:
|
121
|
-
- -
|
118
|
+
- - ">="
|
122
119
|
- !ruby/object:Gem::Version
|
123
120
|
version: '0'
|
124
121
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
125
|
-
none: false
|
126
122
|
requirements:
|
127
|
-
- -
|
123
|
+
- - ">="
|
128
124
|
- !ruby/object:Gem::Version
|
129
125
|
version: '0'
|
130
126
|
requirements: []
|
131
127
|
rubyforge_project:
|
132
|
-
rubygems_version:
|
128
|
+
rubygems_version: 2.6.13
|
133
129
|
signing_key:
|
134
|
-
specification_version:
|
130
|
+
specification_version: 4
|
135
131
|
summary: Fluentd plugin to parse the time parameter.
|
136
132
|
test_files:
|
137
133
|
- test/plugin/test_out_time_parser.rb
|