fluent-plugin-cat-sweep 0.1.5 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 461b6372ab9b8ea7ca9baa69443793350c580d3883cb69b8c5299269665f6c33
4
- data.tar.gz: 6d80c1a473201ad1fd914c01f61d6015886051f059319e0d10cf5ed1f8fea2ee
3
+ metadata.gz: 06fae11b730b391025104c3d387d68e81a1cf3f1f2911587442c0423caec24ba
4
+ data.tar.gz: c5df3a3823d0121a5191e678667a3bcec2e59bca75b3cab6f0a10d4cc8e0d340
5
5
  SHA512:
6
- metadata.gz: 27d7345c99a88dac1d1bde54d0ea0198b8a74eec8e9cf27a5d742bcf55f9de00a2c7a6697c125aaa55a8723cab18be33d8c6a0a9e5eb1e03588be5e5a40d037c
7
- data.tar.gz: 71172be7137a25e36c752a7b4b02fee14cf9b0bdce0ae7eb4227b1a3a0b9a137d57cbcec5019bc181e11b707a1d7fa18d5a34dd50b7e78eefeb5942dc0cfa3a2
6
+ metadata.gz: dde9cb7885b2d4706391f278de60d0f8f74bbbf4f6dc7c40421a1584607b0cf702d3177525984a51dfda09a03dfa9b93da49d668497d56fb555f410a05f78b3b
7
+ data.tar.gz: 1cc78bdd2b38ee16857a87b03f477d1e3c056a83bab443a455bc1c93ecb4e036309cfa71ebff2f2f983bac0a1a7d0fb509f5cecbe3cb5f71fb4cbac8ba821dac
@@ -0,0 +1 @@
1
+ github: civitaspo
@@ -7,22 +7,13 @@ rvm:
7
7
  - '2.5'
8
8
  - '2.6'
9
9
  gemfile:
10
+ - gemfiles/Gemfile.fluentd.1.8
11
+ - gemfiles/Gemfile.fluentd.1.7
12
+ - gemfiles/Gemfile.fluentd.1.6
13
+ - gemfiles/Gemfile.fluentd.1.5
14
+ - gemfiles/Gemfile.fluentd.1.4
15
+ - gemfiles/Gemfile.fluentd.1.3
16
+ - gemfiles/Gemfile.fluentd.1.2
17
+ - gemfiles/Gemfile.fluentd.1.1
18
+ - gemfiles/Gemfile.fluentd.1.0
10
19
  - gemfiles/Gemfile.fluentd.0.14
11
- - gemfiles/Gemfile.fluentd.0.12
12
- - gemfiles/Gemfile.fluentd.0.10
13
- - gemfiles/Gemfile.fluentd.0.10.45
14
-
15
- matrix:
16
- exclude:
17
- - rvm: '2.4'
18
- gemfile: gemfiles/Gemfile.fluentd.0.10
19
- - rvm: '2.4'
20
- gemfile: gemfiles/Gemfile.fluentd.0.10.45
21
- - rvm: '2.5'
22
- gemfile: gemfiles/Gemfile.fluentd.0.10
23
- - rvm: '2.5'
24
- gemfile: gemfiles/Gemfile.fluentd.0.10.45
25
- - rvm: '2.6'
26
- gemfile: gemfiles/Gemfile.fluentd.0.10
27
- - rvm: '2.6'
28
- gemfile: gemfiles/Gemfile.fluentd.0.10.45
@@ -1,3 +1,11 @@
1
+ # 0.2.0
2
+
3
+ Breaking Changes:
4
+
5
+ * Use v1.0 API.
6
+ * Drop support for Fluentd versions smaller than v0.14.
7
+ * https://github.com/civitaspo/fluent-plugin-cat-sweep/pull/21 ( by @karupanerura )
8
+
1
9
  # 0.1.5
2
10
 
3
11
  Enhancements:
data/README.md CHANGED
@@ -111,14 +111,21 @@ tmp/test
111
111
 
112
112
  ```
113
113
  <source>
114
- type cat_sweep
114
+ @type cat_sweep
115
115
 
116
116
  # Required. process files that match this pattern using glob.
117
117
  file_path_with_glob /tmp/test/file_*
118
118
 
119
- # Input pattern. It depends on Parser plugin
120
- format tsv
121
- keys xpath,access_time,label,payload
119
+ # Parser Plugin Setting
120
+ # You can use the old style instead. (Not recommended)
121
+ # ===
122
+ # format tsv
123
+ # keys xpath,access_time,label,payload
124
+ # ===
125
+ <parse>
126
+ @type tsv
127
+ keys xpath,access_time,label,payload
128
+ </parse>
122
129
 
123
130
  # Required. process files that are older than this parameter(seconds).
124
131
  # [WARNING!!]: this plugin moves or removes files even if the files are still open.
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-cat-sweep"
7
- spec.version = "0.1.5"
7
+ spec.version = "0.2.0"
8
8
  spec.authors = ["Civitaspo(takahiro.nakayama)", "Naotoshi Seo"]
9
9
  spec.email = ["civitaspo@gmail.com", "sonots@gmail.com"]
10
10
 
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
18
18
  spec.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
19
19
  spec.require_paths = ["lib"]
20
20
 
21
- spec.add_runtime_dependency "fluentd", ">= 0.10.45"
21
+ spec.add_runtime_dependency "fluentd", ">= 0.14.0"
22
22
  spec.add_development_dependency "bundler"
23
23
  spec.add_development_dependency "rake"
24
24
  spec.add_development_dependency "test-unit"
@@ -1,4 +1,4 @@
1
1
  source "http://rubygems.org"
2
2
 
3
- gem 'fluentd', '= 0.10.45'
3
+ gem 'fluentd', '~> 1.0.0'
4
4
  gemspec :path => "../"
@@ -1,4 +1,4 @@
1
1
  source "http://rubygems.org"
2
2
 
3
- gem 'fluentd', '~> 0.12.0'
3
+ gem 'fluentd', '~> 1.1.0'
4
4
  gemspec :path => "../"
@@ -1,4 +1,4 @@
1
1
  source "http://rubygems.org"
2
2
 
3
- gem 'fluentd', '~> 0.10.45'
3
+ gem 'fluentd', '~> 1.2.0'
4
4
  gemspec :path => "../"
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ gem 'fluentd', '~> 1.3.0'
4
+ gemspec :path => "../"
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ gem 'fluentd', '~> 1.4.0'
4
+ gemspec :path => "../"
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ gem 'fluentd', '~> 1.5.0'
4
+ gemspec :path => "../"
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ gem 'fluentd', '~> 1.6.0'
4
+ gemspec :path => "../"
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ gem 'fluentd', '~> 1.7.0'
4
+ gemspec :path => "../"
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ gem 'fluentd', '~> 1.8.0'
4
+ gemspec :path => "../"
@@ -1,8 +1,12 @@
1
- require 'fluent/input'
1
+ require 'fluent/engine'
2
+ require 'fluent/plugin/input'
3
+ require 'fluent/compat/parser'
2
4
 
3
- module Fluent
4
- class CatSweepInput < Input
5
- Plugin.register_input('cat_sweep', self)
5
+ module Fluent::Plugin
6
+ class CatSweepInput < Fluent::Plugin::Input
7
+ Fluent::Plugin.register_input('cat_sweep', self)
8
+
9
+ helpers :compat_parameters, :parser
6
10
 
7
11
  class OneLineMaxBytesOverError < StandardError
8
12
  end
@@ -11,7 +15,6 @@ module Fluent
11
15
  end
12
16
 
13
17
  config_param :file_path_with_glob, :string
14
- config_param :format, :string
15
18
  config_param :waiting_seconds, :integer # seconds
16
19
  config_param :tag, :string, :default => 'file.cat_sweep'
17
20
  config_param :processing_file_suffix, :string, :default => '.processing'
@@ -24,22 +27,10 @@ module Fluent
24
27
  config_param :file_event_stream, :bool, :default => false
25
28
  config_param :flock_with_rw_mode, :bool, :default => false
26
29
 
27
- # To support log_level option implemented by Fluentd v0.10.43
28
- unless method_defined?(:log)
29
- define_method("log") { $log }
30
- end
31
-
32
- # Define `router` method of v0.12 to support v0.10 or earlier
33
- unless method_defined?(:router)
34
- define_method("router") { Fluent::Engine }
35
- end
36
-
37
30
  def configure(conf)
31
+ compat_parameters_convert(conf, :parser, :buffer, default_chunk_key: "time")
38
32
  super
39
33
 
40
- # Message for users about supported fluentd versions
41
- supported_versions_information
42
-
43
34
  configure_parser(conf)
44
35
 
45
36
  if @processing_file_suffix.empty?
@@ -64,7 +55,7 @@ module Fluent
64
55
  else
65
56
  begin
66
57
  FileUtils.mkdir_p(dirname)
67
- rescue => e
58
+ rescue
68
59
  raise Fluent::ConfigError, "in_cat_sweep: `move_to` directory (#{dirname}) must be writable."
69
60
  end
70
61
  end
@@ -75,6 +66,8 @@ module Fluent
75
66
  end
76
67
 
77
68
  def start
69
+ super
70
+
78
71
  @processing = true
79
72
  @thread = Thread.new(&method(:run_periodic))
80
73
  end
@@ -82,6 +75,8 @@ module Fluent
82
75
  def shutdown
83
76
  @processing = false
84
77
  @thread.join
78
+
79
+ super
85
80
  end
86
81
 
87
82
  def run_periodic
@@ -109,54 +104,8 @@ module Fluent
109
104
  private
110
105
 
111
106
  def configure_parser(conf)
112
- if Plugin.respond_to?(:new_parser)
113
- @parser = Plugin.new_parser(@format)
114
- @parser.configure(conf)
115
- else # For supporting fluentd lower than v0.10.58
116
- @parser = TextParser.new
117
- @parser.configure(conf)
118
- # In lower version of fluentd than v0.10.50,
119
- # `Fluent::Parser#parse` does not support block based API.
120
- # cf. https://github.com/fluent/fluentd/blob/v0.10.49/lib/fluent/parser.rb#L270
121
- # On the other hand, in newer version(like v0.14) of fluentd,
122
- # `Fluent::Parser#parse` only supports block based API.
123
- # cf. https://github.com/fluent/fluentd/blob/v0.14.0.rc.3/lib/fluent/plugin/parser_tsv.rb#L33
124
- # So, lower version of `Fluent::Parser#parse` extends the way to call by block based API.
125
- @parser.extend(Module.new {
126
- def parse(line)
127
- time, record = super
128
- yield(time, record)
129
- return
130
- end
131
- })
132
- end
133
- end
134
-
135
- def supported_versions_information
136
- if current_fluent_version < fluent_version('0.12.0')
137
- log.warn "in_cat_sweep: the support for fluentd v0.10 will end near future. Please upgrade your fluentd or fix this plugin version."
138
- end
139
- if current_fluent_version < fluent_version('0.10.58')
140
- log.warn "in_cat_sweep: fluentd officially supports Plugin.new_parser/Plugin.register_parser APIs from v0.10.58." \
141
- " The support for v0.10.58 will end near future." \
142
- " Please upgrade your fluentd or fix this plugin version."
143
- end
144
- if current_fluent_version < fluent_version('0.10.46')
145
- log.warn "in_cat_sweep: fluentd officially supports parser plugin from v0.10.46." \
146
- " If you use `time_key` parameter and fluentd v0.10.45, doesn't work properly." \
147
- " The support for v0.10.45 will end near future." \
148
- " Please upgrade your fluentd or fix this plugin version."
149
- end
150
- end
151
-
152
- def current_fluent_version
153
- parse_version_comparable(Fluent::VERSION)
154
- end
155
-
156
- def parse_version_comparable(v)
157
- Gem::Version.new(v)
107
+ @parser = parser_create()
158
108
  end
159
- alias :fluent_version :parse_version_comparable # For the readability
160
109
 
161
110
  def will_process?(filename)
162
111
  !(processing?(filename) or error_file?(filename) or sufficient_waiting?(filename))
@@ -171,13 +120,13 @@ module Fluent
171
120
  end
172
121
 
173
122
  def sufficient_waiting?(filename)
174
- (Time.now - File.mtime(filename)).to_i < @waiting_seconds
123
+ (Time.at(Fluent::EventTime.now.to_r) - File.mtime(filename)).to_i < @waiting_seconds
175
124
  end
176
125
 
177
126
  def get_processing_filename(filename)
178
127
  tmpfile = String.new
179
128
  tmpfile << filename << '.' << Process.pid.to_s << '.'
180
- tmpfile << Time.now.to_i.to_s << @processing_file_suffix
129
+ tmpfile << Fluent::EventTime.now.to_s << @processing_file_suffix
181
130
  end
182
131
 
183
132
  def revert_processing_filename(processing_filename)
@@ -256,7 +205,7 @@ module Fluent
256
205
  end
257
206
  end
258
207
  unless entries.empty?
259
- es = ArrayEventStream.new(entries)
208
+ es = Fluent::ArrayEventStream.new(entries)
260
209
  router.emit_stream(@tag, es)
261
210
  end
262
211
  end
@@ -1,33 +1,8 @@
1
1
  require 'test/unit'
2
2
  require 'fluent/version'
3
- require 'fluent/log'
4
3
  require 'fluent/test'
5
-
6
- unless defined?(Test::Unit::AssertionFailedError)
7
- class Test::Unit::AssertionFailedError < StandardError
8
- end
9
- end
10
-
11
- # Reduce sleep period at
12
- # https://github.com/fluent/fluentd/blob/a271b3ec76ab7cf89ebe4012aa5b3912333dbdb7/lib/fluent/test/base.rb#L81
13
- module Fluent
14
- module Test
15
- class TestDriver
16
- def run(num_waits = 10, &block)
17
- @instance.start
18
- begin
19
- # wait until thread starts
20
- # num_waits.times { sleep 0.05 }
21
- sleep 0.05
22
- return yield
23
- ensure
24
- @instance.shutdown
25
- end
26
- end
27
- end
28
- end
29
- end
30
-
4
+ require 'fluent/test/helpers'
5
+ require 'fluent/test/driver/input'
31
6
 
32
7
  def current_fluent_version
33
8
  fluent_version(Fluent::VERSION)
@@ -1,9 +1,10 @@
1
1
  require_relative 'helper'
2
2
  require 'rr'
3
- require 'fluent/input'
4
3
  require 'fluent/plugin/in_cat_sweep'
5
4
 
6
5
  class CatSweepInputTest < Test::Unit::TestCase
6
+ include Fluent::Test::Helpers
7
+
7
8
  def setup
8
9
  Fluent::Test.setup
9
10
  FileUtils.mkdir_p(TMP_DIR_FROM)
@@ -24,48 +25,54 @@ class CatSweepInputTest < Test::Unit::TestCase
24
25
  ]
25
26
 
26
27
  CONFIG_MINIMUM_REQUIRED =
27
- if current_fluent_version < fluent_version('0.12.0') ||
28
- current_fluent_version >= fluent_version('0.14.0')
29
- CONFIG_BASE + %[
30
- format tsv
28
+ CONFIG_BASE + %[
29
+ <parse>
30
+ @type tsv
31
31
  keys ""
32
- waiting_seconds 5
33
- ]
34
- else
35
- CONFIG_BASE + %[
36
- format tsv
37
- waiting_seconds 5
38
- ]
39
- end
40
-
41
- def create_driver(conf, use_v1 = true)
42
- driver = Fluent::Test::InputTestDriver.new(Fluent::CatSweepInput)
43
- if current_fluent_version < fluent_version('0.10.51')
44
- driver.configure(conf)
45
- else
46
- driver.configure(conf, use_v1)
47
- end
48
- driver
32
+ </parse>
33
+ waiting_seconds 3
34
+ ]
35
+
36
+ CONFIG_MINIMUM_REQUIRED_IN_OLD_STYLE =
37
+ CONFIG_BASE + %[
38
+ format tsv
39
+ keys ""
40
+ waiting_seconds 4
41
+ ]
42
+
43
+ def create_driver(conf)
44
+ Fluent::Test::Driver::Input.new(Fluent::Plugin::CatSweepInput).configure(conf)
49
45
  end
50
46
 
51
47
  def test_required_configure
52
48
  assert_raise(Fluent::ConfigError) do
53
- d = create_driver(%[])
49
+ create_driver(%[])
54
50
  end
55
51
 
56
52
  assert_raise(Fluent::ConfigError) do
57
- d = create_driver(CONFIG_BASE)
53
+ create_driver(CONFIG_BASE)
58
54
  end
59
55
 
60
56
  assert_raise(Fluent::ConfigError) do
61
- d = create_driver(CONFIG_BASE + %[format tsv])
57
+ create_driver(CONFIG_BASE + %[
58
+ <parse>
59
+ @type tsv
60
+ keys ""
61
+ </parse>
62
+ ])
62
63
  end
63
64
 
64
65
  d = create_driver(CONFIG_MINIMUM_REQUIRED)
65
66
 
66
67
  assert_equal "#{TMP_DIR_FROM}/*", d.instance.instance_variable_get(:@file_path_with_glob)
67
- assert_equal 'tsv', d.instance.instance_variable_get(:@format)
68
- assert_equal 5, d.instance.instance_variable_get(:@waiting_seconds)
68
+ assert_equal Fluent::Plugin::TSVParser, d.instance.instance_variable_get(:@parser).class
69
+ assert_equal 3, d.instance.instance_variable_get(:@waiting_seconds)
70
+
71
+ d = create_driver(CONFIG_MINIMUM_REQUIRED_IN_OLD_STYLE)
72
+
73
+ assert_equal "#{TMP_DIR_FROM}/*", d.instance.instance_variable_get(:@file_path_with_glob)
74
+ assert_equal Fluent::Plugin::TSVParser, d.instance.instance_variable_get(:@parser).class
75
+ assert_equal 4, d.instance.instance_variable_get(:@waiting_seconds)
69
76
  end
70
77
 
71
78
  def test_configure_file_event_stream
@@ -76,9 +83,9 @@ class CatSweepInputTest < Test::Unit::TestCase
76
83
  assert { true == d.instance.file_event_stream }
77
84
  end
78
85
 
79
- def compare_test_result(emits, tests)
80
- emits.each_index do |i|
81
- assert { tests[i]['expected'] == emits[i][2]['message'] }
86
+ def compare_test_result(events, tests)
87
+ events.each_index do |i|
88
+ assert { tests[i]['expected'] == events[i][2]['message'] }
82
89
  end
83
90
  end
84
91
 
@@ -114,9 +121,9 @@ class CatSweepInputTest < Test::Unit::TestCase
114
121
  waiting_seconds 0
115
122
  keys hdfs_path,unixtimestamp,label,message
116
123
  ])
117
- d.run
124
+ d.run(timeout: 1, expect_records: 2) # timeout = waiting_seconds + 1
118
125
 
119
- compare_test_result(d.emits, test_cases)
126
+ compare_test_result(d.events, test_cases)
120
127
  assert { Dir.glob("#{TMP_DIR_FROM}/#{test_case_name}*").empty? }
121
128
  end
122
129
  end
@@ -142,9 +149,9 @@ class CatSweepInputTest < Test::Unit::TestCase
142
149
  keys hdfs_path,unixtimestamp,label,message
143
150
  move_to #{TMP_DIR_TO}
144
151
  ])
145
- d.run
152
+ d.run(timeout: 1, expect_records: 2) # timeout = waiting_seconds + 1
146
153
 
147
- compare_test_result(d.emits, test_cases)
154
+ compare_test_result(d.events, test_cases)
148
155
 
149
156
  assert(Dir.glob("#{TMP_DIR_FROM}/test_move_file*").empty?)
150
157
  assert_match(
@@ -177,7 +184,7 @@ class CatSweepInputTest < Test::Unit::TestCase
177
184
  oneline_max_bytes 1
178
185
  ])
179
186
 
180
- d.run
187
+ d.run(timeout: 1, expect_records: 2) # timeout = waiting_seconds + 1
181
188
 
182
189
  assert_match(
183
190
  %r{\A#{TMP_DIR_FROM}/test_oneline_max_bytes.*\.error},
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cat-sweep
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.5
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Civitaspo(takahiro.nakayama)
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2019-04-05 00:00:00.000000000 Z
12
+ date: 2020-01-09 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -17,14 +17,14 @@ dependencies:
17
17
  requirements:
18
18
  - - ">="
19
19
  - !ruby/object:Gem::Version
20
- version: 0.10.45
20
+ version: 0.14.0
21
21
  type: :runtime
22
22
  prerelease: false
23
23
  version_requirements: !ruby/object:Gem::Requirement
24
24
  requirements:
25
25
  - - ">="
26
26
  - !ruby/object:Gem::Version
27
- version: 0.10.45
27
+ version: 0.14.0
28
28
  - !ruby/object:Gem::Dependency
29
29
  name: bundler
30
30
  requirement: !ruby/object:Gem::Requirement
@@ -117,6 +117,7 @@ executables: []
117
117
  extensions: []
118
118
  extra_rdoc_files: []
119
119
  files:
120
+ - ".github/FUNDING.yml"
120
121
  - ".gitignore"
121
122
  - ".travis.yml"
122
123
  - CHANGELOG.md
@@ -127,10 +128,16 @@ files:
127
128
  - Rakefile
128
129
  - example.conf
129
130
  - fluent-plugin-cat-sweep.gemspec
130
- - gemfiles/Gemfile.fluentd.0.10
131
- - gemfiles/Gemfile.fluentd.0.10.45
132
- - gemfiles/Gemfile.fluentd.0.12
133
131
  - gemfiles/Gemfile.fluentd.0.14
132
+ - gemfiles/Gemfile.fluentd.1.0
133
+ - gemfiles/Gemfile.fluentd.1.1
134
+ - gemfiles/Gemfile.fluentd.1.2
135
+ - gemfiles/Gemfile.fluentd.1.3
136
+ - gemfiles/Gemfile.fluentd.1.4
137
+ - gemfiles/Gemfile.fluentd.1.5
138
+ - gemfiles/Gemfile.fluentd.1.6
139
+ - gemfiles/Gemfile.fluentd.1.7
140
+ - gemfiles/Gemfile.fluentd.1.8
134
141
  - lib/fluent/plugin/in_cat_sweep.rb
135
142
  - test/helper.rb
136
143
  - test/test_in_cat_sweep.rb
@@ -153,8 +160,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
153
160
  - !ruby/object:Gem::Version
154
161
  version: '0'
155
162
  requirements: []
156
- rubyforge_project:
157
- rubygems_version: 2.7.3
163
+ rubygems_version: 3.0.1
158
164
  signing_key:
159
165
  specification_version: 4
160
166
  summary: Fluentd plugin to cat files and move them.