fluent-plugin-time_parser 0.0.6 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 581b6c1bcc8546fb9c4de7e835100c3c4ccf8325
4
+ data.tar.gz: 54288fe1474709bbd11a2cdfa70b9b8e3575b8ac
5
+ SHA512:
6
+ metadata.gz: 58171ffa45dada9b15dd4a2b53c25bff4144e7ff20e785991a3dc55dd7eeb3f44b3fa8760a502c8fce45c5fe47d5ba369c80c2b5481da993f2fe969f9bc3c530
7
+ data.tar.gz: 476cc0d39402b799ab0ef320e97792bb012f70056ce88b8dbe50c202e8eac3f2c72c0c5e0358ababfd003e9f8835941082d2669381385afcccf405096bc5ff98
@@ -0,0 +1,13 @@
1
+ language: ruby
2
+
3
+ rvm:
4
+ - 2.1
5
+ - 2.2
6
+ - 2.3.5
7
+ - 2.4.2
8
+
9
+ gemfile:
10
+ - Gemfile
11
+
12
+ script: bundle exec rake test
13
+ sudo: false
data/LICENSE CHANGED
@@ -1,2 +1,2 @@
1
- Copyright (c) 2012 Kentaro Kuribayashi
1
+ Copyright (c) 2017 Carlos Donderis
2
2
  Apache License, Version 2.0
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # fluent-plugin-time_parser
1
+ # fluent-plugin-time_parser, a plugin for [Fluentd](http://fluentd.org)
2
2
 
3
3
  ## Component
4
4
  TimeParserOutput
@@ -13,7 +13,7 @@ Will take a time attribute and will extract the date and the hour for a given ti
13
13
  type time_parser
14
14
 
15
15
  key time
16
- add_tag_prefix extracted.
16
+ tag extracted.${tag}
17
17
  time_zone Asia/Tokyo
18
18
  </match>
19
19
  ```
@@ -43,11 +43,13 @@ Then you'll get re-emmited tags/records like so:
43
43
 
44
44
  `key` is used to point a key whose value contains the time you want to parse.
45
45
 
46
- ### remove_tag_prefix, remove_tag_suffix, add_tag_prefix, add_tag_suffix
46
+ ### tag
47
47
 
48
- These params are included from `Fluent::HandleTagNameMixin`. See the code for details.
48
+ In this param, users can write placeholders.
49
49
 
50
- You must add at least one of these params.
50
+ Please use placeholders ${tag}, ${tag[0]}, ${tag[1]} in configuration.
51
+
52
+ Note that `Fluent::HandleTagNameMixin` dependency is removed in v0.14 style of this plugin.
51
53
 
52
54
  ### time_zone
53
55
 
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |gem|
2
2
  gem.name = 'fluent-plugin-time_parser'
3
- gem.version = '0.0.6'
4
- gem.authors = ['Carlos Donderis', 'Michael H. Oshita']
3
+ gem.version = '0.1.0'
4
+ gem.authors = ['Carlos Donderis', 'Michael H. Oshita', 'Hiroshi Hatake']
5
5
  gem.email = ['cdonderis@gmail.com', 'ijinpublic+github@gmail.com']
6
6
  gem.homepage = 'http://github.com/cads/fluent-plugin-time_parser'
7
7
  gem.description = %q{Fluentd plugin to parse the time parameter.}
@@ -13,8 +13,8 @@ Gem::Specification.new do |gem|
13
13
  gem.require_paths = ['lib']
14
14
 
15
15
  gem.add_development_dependency 'rake'
16
- gem.add_development_dependency 'fluentd'
17
16
  gem.add_development_dependency 'tzinfo'
17
+ gem.add_development_dependency 'test-unit', '~> 3.2'
18
18
  gem.add_runtime_dependency 'tzinfo'
19
- gem.add_runtime_dependency 'fluentd'
19
+ gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
20
20
  end
@@ -1,27 +1,31 @@
1
1
  require 'time'
2
2
  require 'tzinfo'
3
+ require 'fluent/plugin/output'
3
4
 
4
- module Fluent
5
+ module Fluent::Plugin
5
6
  class TimeParserOutput < Output
6
- include Fluent::HandleTagNameMixin
7
7
  Fluent::Plugin.register_output('time_parser', self)
8
8
 
9
+ helpers :event_emitter, :compat_parameters
10
+
11
+ DEFAULT_BUFFER_TYPE = "memory"
12
+
13
+ config_param :tag, :string
9
14
  config_param :key, :string, :default => 'time'
10
15
  config_param :time_zone, :string, :default => ''
11
16
  config_param :parsed_time_tag, :string, :default => 'parsed_time'
12
17
  config_param :parsed_hour_tag, :string, :default => 'parsed_hour'
13
18
  config_param :parsed_date_tag, :string, :default => 'parsed_date'
14
19
 
20
+ config_section :buffer do
21
+ config_set_default :@type, DEFAULT_BUFFER_TYPE
22
+ config_set_default :chunk_keys, ['tag']
23
+ end
24
+
15
25
  def configure(conf)
26
+ compat_parameters_convert(conf, :buffer)
16
27
  super
17
- if (
18
- !remove_tag_prefix &&
19
- !remove_tag_suffix &&
20
- !add_tag_prefix &&
21
- !add_tag_suffix
22
- )
23
- raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
24
- end
28
+ raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
25
29
  end
26
30
 
27
31
  def start
@@ -32,13 +36,13 @@ module Fluent
32
36
  super
33
37
  end
34
38
 
35
- def emit(tag, es, chain)
36
- es.each {|time,record|
39
+ def write(chunk)
40
+ tag = extract_placeholders(@tag, chunk.metadata)
41
+ chunk.msgpack_each {|time,record|
37
42
  t = tag.dup
38
43
  filter_record(t, time, record)
39
- Engine.emit(t, time, record)
44
+ router.emit(t, time, record)
40
45
  }
41
- chain.next
42
46
  end
43
47
 
44
48
  def filter_record(tag, time, record)
@@ -63,11 +67,10 @@ module Fluent
63
67
  record[parsed_hour_tag] = hour
64
68
 
65
69
  rescue ArgumentError => error
66
- $log.warn("out_extract_query_params: #{error.message}")
70
+ log.warn("out_extract_query_params: #{error.message}")
67
71
  rescue TZInfo::InvalidTimezoneIdentifier
68
- $log.warn("Timezone Not Valid, please refer to http://tzinfo.rubyforge.org/doc/classes/TZInfo/Timezone.html for valid timezones")
72
+ log.warn("Timezone Not Valid, please refer to http://tzinfo.rubyforge.org/doc/classes/TZInfo/Timezone.html for valid timezones")
69
73
  end
70
- super(tag, time, record)
71
74
  end
72
75
  end
73
76
  end
@@ -12,29 +12,29 @@ class TimeParserOutputTest < Test::Unit::TestCase
12
12
  Fluent::Test.setup
13
13
  end
14
14
 
15
- def create_driver(conf, tag = 'test')
16
- Fluent::Test::OutputTestDriver.new(
17
- Fluent::TimeParserOutput, tag
15
+ def create_driver(conf)
16
+ Fluent::Test::Driver::Output.new(
17
+ Fluent::Plugin::TimeParserOutput
18
18
  ).configure(conf)
19
19
  end
20
20
 
21
21
  def test_configure
22
22
  d = create_driver(%[
23
23
  key test
24
- add_tag_prefix extracted.
24
+ tag extracted.${tag}
25
25
  time_zone Asia/Tokyo
26
26
  ])
27
27
  assert_equal 'test', d.instance.key
28
- assert_equal 'extracted.', d.instance.add_tag_prefix
28
+ assert_equal 'extracted.${tag}', d.instance.tag
29
29
  assert_equal 'Asia/Tokyo', d.instance.time_zone
30
30
 
31
31
  #Default Key
32
32
  d = create_driver(%[
33
- add_tag_prefix extracted.
33
+ tag extracted.${tag}
34
34
  time_zone Asia/Tokyo
35
35
  ])
36
36
  assert_equal 'time', d.instance.key
37
- assert_equal 'extracted.', d.instance.add_tag_prefix
37
+ assert_equal 'extracted.${tag}', d.instance.tag
38
38
  assert_equal 'Asia/Tokyo', d.instance.time_zone
39
39
 
40
40
  #No Prefix
@@ -47,16 +47,16 @@ class TimeParserOutputTest < Test::Unit::TestCase
47
47
  #No TimeZone
48
48
  d = create_driver(%[
49
49
  key test
50
- add_tag_prefix extracted.
50
+ tag extracted.${tag}
51
51
  ])
52
52
  assert_equal 'test', d.instance.key
53
- assert_equal 'extracted.', d.instance.add_tag_prefix
53
+ assert_equal 'extracted.${tag}', d.instance.tag
54
54
  end
55
55
 
56
56
  def test_filter_record
57
57
  d = create_driver(%[
58
58
  key time
59
- add_tag_prefix extracted.
59
+ tag extracted.${tag}
60
60
  time_zone Asia/Tokyo
61
61
  ])
62
62
  tag = 'test'
@@ -72,7 +72,7 @@ class TimeParserOutputTest < Test::Unit::TestCase
72
72
  def test_girigiri_records
73
73
  d = create_driver(%[
74
74
  key time
75
- add_tag_prefix extracted.
75
+ tag extracted.${tag}
76
76
  time_zone Japan
77
77
  ])
78
78
  tag = 'test'
@@ -88,7 +88,7 @@ class TimeParserOutputTest < Test::Unit::TestCase
88
88
  def test_filter_record_bad_parameters
89
89
  d = create_driver(%[
90
90
  key time
91
- add_tag_prefix extracted.
91
+ tag extracted.${tag}
92
92
  time_zone myPlace
93
93
  ])
94
94
  tag = 'test'
@@ -109,31 +109,46 @@ class TimeParserOutputTest < Test::Unit::TestCase
109
109
  def test_emit
110
110
  d = create_driver(%[
111
111
  key time
112
- add_tag_prefix extracted.
112
+ tag extracted.${tag}
113
113
  time_zone Asia/Tokyo
114
114
  ])
115
115
 
116
- d.run { d.emit('time' => TIME) }
117
- emits = d.emits
116
+ d.run(default_tag: 'test') { d.feed('time' => TIME) }
117
+ emits = d.events
118
118
 
119
119
  assert_equal 1, emits.count
120
120
  assert_equal 'extracted.test', emits[0][0]
121
121
  assert_equal TIME, emits[0][2]['time']
122
122
  end
123
123
 
124
+ def test_emit_without_tag_prefix
125
+ d = create_driver(%[
126
+ key time
127
+ tag ${tag[1]}
128
+ time_zone Asia/Tokyo
129
+ ])
130
+
131
+ d.run(default_tag: 'removed.test') { d.feed('time' => TIME) }
132
+ emits = d.events
133
+
134
+ assert_equal 1, emits.count
135
+ assert_equal 'test', emits[0][0]
136
+ assert_equal TIME, emits[0][2]['time']
137
+ end
138
+
124
139
  def test_emit_multi
125
140
  d = create_driver(%[
126
141
  key time
127
- add_tag_prefix extracted.
142
+ tag extracted.${tag}
128
143
  time_zone Asia/Tokyo
129
144
  ])
130
145
 
131
- d.run do
132
- d.emit('time' => TIME)
133
- d.emit('time' => TIME)
134
- d.emit('time' => TIME)
146
+ d.run(default_tag: 'test') do
147
+ d.feed('time' => TIME)
148
+ d.feed('time' => TIME)
149
+ d.feed('time' => TIME)
135
150
  end
136
- emits = d.emits
151
+ emits = d.events
137
152
 
138
153
  assert_equal 3, emits.count
139
154
  0.upto(2) do |i|
@@ -145,12 +160,12 @@ class TimeParserOutputTest < Test::Unit::TestCase
145
160
  def test_emit_with_invalid_time
146
161
  d = create_driver(%[
147
162
  key time
148
- add_tag_prefix extracted.
163
+ tag extracted.${tag}
149
164
  time_zone Asia/Tokyo
150
165
  ])
151
166
  wrong_time = 'wrong time'
152
- d.run { d.emit('time' => wrong_time) }
153
- emits = d.emits
167
+ d.run(default_tag: 'test') { d.feed('time' => wrong_time) }
168
+ emits = d.events
154
169
 
155
170
  assert_equal 1, emits.count
156
171
  assert_equal 'extracted.test', emits[0][0]
@@ -4,6 +4,8 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
4
4
  $LOAD_PATH.unshift(File.dirname(__FILE__))
5
5
 
6
6
  require 'fluent/test'
7
+ require 'fluent/test/helpers'
8
+ require 'fluent/test/driver/output'
7
9
 
8
10
  unless ENV.has_key?('VERBOSE')
9
11
  nulllogger = Object.new
@@ -18,4 +20,5 @@ end
18
20
  require 'fluent/plugin/out_time_parser'
19
21
 
20
22
  class Test::Unit::TestCase
23
+ include Fluent::Test::Helpers
21
24
  end
metadata CHANGED
@@ -1,97 +1,93 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-time_parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.6
5
- prerelease:
4
+ version: 0.1.0
6
5
  platform: ruby
7
6
  authors:
8
7
  - Carlos Donderis
9
8
  - Michael H. Oshita
9
+ - Hiroshi Hatake
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2013-05-15 00:00:00.000000000 Z
13
+ date: 2017-09-20 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: rake
17
17
  requirement: !ruby/object:Gem::Requirement
18
- none: false
19
18
  requirements:
20
- - - ! '>='
19
+ - - ">="
21
20
  - !ruby/object:Gem::Version
22
21
  version: '0'
23
22
  type: :development
24
23
  prerelease: false
25
24
  version_requirements: !ruby/object:Gem::Requirement
26
- none: false
27
25
  requirements:
28
- - - ! '>='
26
+ - - ">="
29
27
  - !ruby/object:Gem::Version
30
28
  version: '0'
31
29
  - !ruby/object:Gem::Dependency
32
- name: fluentd
30
+ name: tzinfo
33
31
  requirement: !ruby/object:Gem::Requirement
34
- none: false
35
32
  requirements:
36
- - - ! '>='
33
+ - - ">="
37
34
  - !ruby/object:Gem::Version
38
35
  version: '0'
39
36
  type: :development
40
37
  prerelease: false
41
38
  version_requirements: !ruby/object:Gem::Requirement
42
- none: false
43
39
  requirements:
44
- - - ! '>='
40
+ - - ">="
45
41
  - !ruby/object:Gem::Version
46
42
  version: '0'
47
43
  - !ruby/object:Gem::Dependency
48
- name: tzinfo
44
+ name: test-unit
49
45
  requirement: !ruby/object:Gem::Requirement
50
- none: false
51
46
  requirements:
52
- - - ! '>='
47
+ - - "~>"
53
48
  - !ruby/object:Gem::Version
54
- version: '0'
49
+ version: '3.2'
55
50
  type: :development
56
51
  prerelease: false
57
52
  version_requirements: !ruby/object:Gem::Requirement
58
- none: false
59
53
  requirements:
60
- - - ! '>='
54
+ - - "~>"
61
55
  - !ruby/object:Gem::Version
62
- version: '0'
56
+ version: '3.2'
63
57
  - !ruby/object:Gem::Dependency
64
58
  name: tzinfo
65
59
  requirement: !ruby/object:Gem::Requirement
66
- none: false
67
60
  requirements:
68
- - - ! '>='
61
+ - - ">="
69
62
  - !ruby/object:Gem::Version
70
63
  version: '0'
71
64
  type: :runtime
72
65
  prerelease: false
73
66
  version_requirements: !ruby/object:Gem::Requirement
74
- none: false
75
67
  requirements:
76
- - - ! '>='
68
+ - - ">="
77
69
  - !ruby/object:Gem::Version
78
70
  version: '0'
79
71
  - !ruby/object:Gem::Dependency
80
72
  name: fluentd
81
73
  requirement: !ruby/object:Gem::Requirement
82
- none: false
83
74
  requirements:
84
- - - ! '>='
75
+ - - ">="
85
76
  - !ruby/object:Gem::Version
86
- version: '0'
77
+ version: 0.14.0
78
+ - - "<"
79
+ - !ruby/object:Gem::Version
80
+ version: '2'
87
81
  type: :runtime
88
82
  prerelease: false
89
83
  version_requirements: !ruby/object:Gem::Requirement
90
- none: false
91
84
  requirements:
92
- - - ! '>='
85
+ - - ">="
93
86
  - !ruby/object:Gem::Version
94
- version: '0'
87
+ version: 0.14.0
88
+ - - "<"
89
+ - !ruby/object:Gem::Version
90
+ version: '2'
95
91
  description: Fluentd plugin to parse the time parameter.
96
92
  email:
97
93
  - cdonderis@gmail.com
@@ -100,7 +96,8 @@ executables: []
100
96
  extensions: []
101
97
  extra_rdoc_files: []
102
98
  files:
103
- - .gitignore
99
+ - ".gitignore"
100
+ - ".travis.yml"
104
101
  - Gemfile
105
102
  - LICENSE
106
103
  - README.md
@@ -111,27 +108,26 @@ files:
111
108
  - test/test_helper.rb
112
109
  homepage: http://github.com/cads/fluent-plugin-time_parser
113
110
  licenses: []
111
+ metadata: {}
114
112
  post_install_message:
115
113
  rdoc_options: []
116
114
  require_paths:
117
115
  - lib
118
116
  required_ruby_version: !ruby/object:Gem::Requirement
119
- none: false
120
117
  requirements:
121
- - - ! '>='
118
+ - - ">="
122
119
  - !ruby/object:Gem::Version
123
120
  version: '0'
124
121
  required_rubygems_version: !ruby/object:Gem::Requirement
125
- none: false
126
122
  requirements:
127
- - - ! '>='
123
+ - - ">="
128
124
  - !ruby/object:Gem::Version
129
125
  version: '0'
130
126
  requirements: []
131
127
  rubyforge_project:
132
- rubygems_version: 1.8.24
128
+ rubygems_version: 2.6.13
133
129
  signing_key:
134
- specification_version: 3
130
+ specification_version: 4
135
131
  summary: Fluentd plugin to parse the time parameter.
136
132
  test_files:
137
133
  - test/plugin/test_out_time_parser.rb