fluent-plugin-cat-sweep 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 461b6372ab9b8ea7ca9baa69443793350c580d3883cb69b8c5299269665f6c33
-  data.tar.gz: 6d80c1a473201ad1fd914c01f61d6015886051f059319e0d10cf5ed1f8fea2ee
+  metadata.gz: 06fae11b730b391025104c3d387d68e81a1cf3f1f2911587442c0423caec24ba
+  data.tar.gz: c5df3a3823d0121a5191e678667a3bcec2e59bca75b3cab6f0a10d4cc8e0d340
 SHA512:
-  metadata.gz: 27d7345c99a88dac1d1bde54d0ea0198b8a74eec8e9cf27a5d742bcf55f9de00a2c7a6697c125aaa55a8723cab18be33d8c6a0a9e5eb1e03588be5e5a40d037c
-  data.tar.gz: 71172be7137a25e36c752a7b4b02fee14cf9b0bdce0ae7eb4227b1a3a0b9a137d57cbcec5019bc181e11b707a1d7fa18d5a34dd50b7e78eefeb5942dc0cfa3a2
+  metadata.gz: dde9cb7885b2d4706391f278de60d0f8f74bbbf4f6dc7c40421a1584607b0cf702d3177525984a51dfda09a03dfa9b93da49d668497d56fb555f410a05f78b3b
+  data.tar.gz: 1cc78bdd2b38ee16857a87b03f477d1e3c056a83bab443a455bc1c93ecb4e036309cfa71ebff2f2f983bac0a1a7d0fb509f5cecbe3cb5f71fb4cbac8ba821dac

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+github: civitaspo

data/.travis.yml

@@ -7,22 +7,13 @@ rvm:
   - '2.5'
   - '2.6'
 gemfile:
+  - gemfiles/Gemfile.fluentd.1.8
+  - gemfiles/Gemfile.fluentd.1.7
+  - gemfiles/Gemfile.fluentd.1.6
+  - gemfiles/Gemfile.fluentd.1.5
+  - gemfiles/Gemfile.fluentd.1.4
+  - gemfiles/Gemfile.fluentd.1.3
+  - gemfiles/Gemfile.fluentd.1.2
+  - gemfiles/Gemfile.fluentd.1.1
+  - gemfiles/Gemfile.fluentd.1.0
   - gemfiles/Gemfile.fluentd.0.14
-  - gemfiles/Gemfile.fluentd.0.12
-  - gemfiles/Gemfile.fluentd.0.10
-  - gemfiles/Gemfile.fluentd.0.10.45
-
-matrix:
-  exclude:
-    - rvm: '2.4'
-      gemfile: gemfiles/Gemfile.fluentd.0.10
-    - rvm: '2.4'
-      gemfile: gemfiles/Gemfile.fluentd.0.10.45
-    - rvm: '2.5'
-      gemfile: gemfiles/Gemfile.fluentd.0.10
-    - rvm: '2.5'
-      gemfile: gemfiles/Gemfile.fluentd.0.10.45
-    - rvm: '2.6'
-      gemfile: gemfiles/Gemfile.fluentd.0.10
-    - rvm: '2.6'
-      gemfile: gemfiles/Gemfile.fluentd.0.10.45

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+# 0.2.0
+
+Breaking Changes:
+
+* Use v1.0 API.
+  * Drop support for Fluentd versions smaller than v0.14.
+  * https://github.com/civitaspo/fluent-plugin-cat-sweep/pull/21  ( by @karupanerura )
+
 # 0.1.5
 
 Enhancements:

data/README.md

@@ -111,14 +111,21 @@ tmp/test
 
 ```
 <source>
-  type cat_sweep
+  @type cat_sweep
 
   # Required. process files that match this pattern using glob.
   file_path_with_glob /tmp/test/file_*
 
-  # Input pattern. It depends on Parser plugin
-  format tsv
-  keys xpath,access_time,label,payload
+  # Parser Plugin Setting
+  # You can use the old style instead. (Not recommended)
+  # ===
+  # format tsv
+  # keys xpath,access_time,label,payload
+  # ===
+  <parse>
+    @type tsv
+    keys xpath,access_time,label,payload
+  </parse>
 
   # Required. process files that are older than this parameter(seconds).
   # [WARNING!!]: this plugin moves or removes files even if the files are still open.

data/fluent-plugin-cat-sweep.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-cat-sweep"
-  spec.version       = "0.1.5"
+  spec.version       = "0.2.0"
   spec.authors       = ["Civitaspo(takahiro.nakayama)", "Naotoshi Seo"]
   spec.email         = ["civitaspo@gmail.com", "sonots@gmail.com"]
 
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "fluentd", ">= 0.10.45"
+  spec.add_runtime_dependency "fluentd", ">= 0.14.0"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit"

data/gemfiles/{Gemfile.fluentd.0.10.45 → Gemfile.fluentd.1.0}

@@ -1,4 +1,4 @@
 source "http://rubygems.org"
 
-gem 'fluentd', '= 0.10.45'
+gem 'fluentd', '~> 1.0.0'
 gemspec :path => "../"

data/gemfiles/{Gemfile.fluentd.0.12 → Gemfile.fluentd.1.1}

@@ -1,4 +1,4 @@
 source "http://rubygems.org"
 
-gem 'fluentd', '~> 0.12.0'
+gem 'fluentd', '~> 1.1.0'
 gemspec :path => "../"

data/gemfiles/{Gemfile.fluentd.0.10 → Gemfile.fluentd.1.2}

@@ -1,4 +1,4 @@
 source "http://rubygems.org"
 
-gem 'fluentd', '~> 0.10.45'
+gem 'fluentd', '~> 1.2.0'
 gemspec :path => "../"

data/gemfiles/Gemfile.fluentd.1.3

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 1.3.0'
+gemspec :path => "../"

data/gemfiles/Gemfile.fluentd.1.4

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 1.4.0'
+gemspec :path => "../"

data/gemfiles/Gemfile.fluentd.1.5

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 1.5.0'
+gemspec :path => "../"

data/gemfiles/Gemfile.fluentd.1.6

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 1.6.0'
+gemspec :path => "../"

data/gemfiles/Gemfile.fluentd.1.7

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 1.7.0'
+gemspec :path => "../"

data/gemfiles/Gemfile.fluentd.1.8

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 1.8.0'
+gemspec :path => "../"

data/lib/fluent/plugin/in_cat_sweep.rb

@@ -1,8 +1,12 @@
-require 'fluent/input'
+require 'fluent/engine'
+require 'fluent/plugin/input'
+require 'fluent/compat/parser'
 
-module Fluent
-  class CatSweepInput < Input
-    Plugin.register_input('cat_sweep', self)
+module Fluent::Plugin
+  class CatSweepInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('cat_sweep', self)
+
+    helpers :compat_parameters, :parser
 
     class OneLineMaxBytesOverError < StandardError
     end
@@ -11,7 +15,6 @@ module Fluent
     end
 
     config_param :file_path_with_glob,     :string
-    config_param :format,                  :string
     config_param :waiting_seconds,         :integer  # seconds
     config_param :tag,                     :string,  :default => 'file.cat_sweep'
     config_param :processing_file_suffix,  :string,  :default => '.processing'
@@ -24,22 +27,10 @@ module Fluent
     config_param :file_event_stream,       :bool,    :default => false
     config_param :flock_with_rw_mode,      :bool,    :default => false
 
-    # To support log_level option implemented by Fluentd v0.10.43
-    unless method_defined?(:log)
-      define_method("log") { $log }
-    end
-
-    # Define `router` method of v0.12 to support v0.10 or earlier
-    unless method_defined?(:router)
-      define_method("router") { Fluent::Engine }
-    end
-
     def configure(conf)
+      compat_parameters_convert(conf, :parser, :buffer, default_chunk_key: "time")
       super
 
-      # Message for users about supported fluentd versions
-      supported_versions_information
-
       configure_parser(conf)
 
       if @processing_file_suffix.empty?
@@ -64,7 +55,7 @@ module Fluent
         else
           begin
             FileUtils.mkdir_p(dirname)
-          rescue => e
+          rescue
             raise Fluent::ConfigError, "in_cat_sweep: `move_to` directory (#{dirname}) must be writable."
           end
         end
@@ -75,6 +66,8 @@ module Fluent
     end
 
     def start
+      super
+
       @processing = true
       @thread = Thread.new(&method(:run_periodic))
     end
@@ -82,6 +75,8 @@ module Fluent
     def shutdown
       @processing = false
       @thread.join
+
+      super
     end
 
     def run_periodic
@@ -109,54 +104,8 @@ module Fluent
     private
 
     def configure_parser(conf)
-      if Plugin.respond_to?(:new_parser)
-        @parser = Plugin.new_parser(@format)
-        @parser.configure(conf)
-      else # For supporting fluentd lower than v0.10.58
-        @parser = TextParser.new
-        @parser.configure(conf)
-        # In lower version of fluentd than v0.10.50,
-        # `Fluent::Parser#parse` does not support block based API.
-        # cf. https://github.com/fluent/fluentd/blob/v0.10.49/lib/fluent/parser.rb#L270
-        # On the other hand, in newer version(like v0.14) of fluentd,
-        # `Fluent::Parser#parse` only supports block based API.
-        # cf. https://github.com/fluent/fluentd/blob/v0.14.0.rc.3/lib/fluent/plugin/parser_tsv.rb#L33
-        # So, lower version of `Fluent::Parser#parse` extends the way to call by block based API.
-        @parser.extend(Module.new {
-          def parse(line)
-            time, record = super
-            yield(time, record)
-            return
-          end
-        })
-      end
-    end
-
-    def supported_versions_information
-      if current_fluent_version < fluent_version('0.12.0')
-        log.warn "in_cat_sweep: the support for fluentd v0.10 will end near future. Please upgrade your fluentd or fix this plugin version."
-      end
-      if current_fluent_version < fluent_version('0.10.58')
-        log.warn "in_cat_sweep: fluentd officially supports Plugin.new_parser/Plugin.register_parser APIs from v0.10.58." \
-          " The support for v0.10.58 will end near future." \
-          " Please upgrade your fluentd or fix this plugin version."
-      end
-      if current_fluent_version < fluent_version('0.10.46')
-        log.warn "in_cat_sweep: fluentd officially supports parser plugin from v0.10.46." \
-          " If you use `time_key` parameter and fluentd v0.10.45, doesn't work properly." \
-          " The support for v0.10.45 will end near future." \
-          " Please upgrade your fluentd or fix this plugin version."
-      end
-    end
-
-    def current_fluent_version
-      parse_version_comparable(Fluent::VERSION)
-    end
-    
-    def parse_version_comparable(v)
-      Gem::Version.new(v)
+      @parser = parser_create()
     end
-    alias :fluent_version :parse_version_comparable # For the readability
 
     def will_process?(filename)
       !(processing?(filename) or error_file?(filename) or sufficient_waiting?(filename))
@@ -171,13 +120,13 @@ module Fluent
     end
 
     def sufficient_waiting?(filename)
-      (Time.now - File.mtime(filename)).to_i < @waiting_seconds
+      (Time.at(Fluent::EventTime.now.to_r) - File.mtime(filename)).to_i < @waiting_seconds
     end
 
     def get_processing_filename(filename)
       tmpfile = String.new
       tmpfile << filename << '.' << Process.pid.to_s << '.'
-      tmpfile << Time.now.to_i.to_s << @processing_file_suffix
+      tmpfile << Fluent::EventTime.now.to_s << @processing_file_suffix
     end
 
     def revert_processing_filename(processing_filename)
@@ -256,7 +205,7 @@ module Fluent
         end
       end
       unless entries.empty?
-        es = ArrayEventStream.new(entries)
+        es = Fluent::ArrayEventStream.new(entries)
         router.emit_stream(@tag, es)
       end
     end

data/test/helper.rb

@@ -1,33 +1,8 @@
 require 'test/unit'
 require 'fluent/version'
-require 'fluent/log'
 require 'fluent/test'
-
-unless defined?(Test::Unit::AssertionFailedError)
-  class Test::Unit::AssertionFailedError < StandardError
-  end
-end
-
-# Reduce sleep period at
-# https://github.com/fluent/fluentd/blob/a271b3ec76ab7cf89ebe4012aa5b3912333dbdb7/lib/fluent/test/base.rb#L81
-module Fluent
-  module Test
-    class TestDriver
-      def run(num_waits = 10, &block)
-        @instance.start
-        begin
-          # wait until thread starts
-          # num_waits.times { sleep 0.05 }
-          sleep 0.05
-          return yield
-        ensure
-          @instance.shutdown
-        end
-      end
-    end
-  end
-end
-
+require 'fluent/test/helpers'
+require 'fluent/test/driver/input'
 
 def current_fluent_version
   fluent_version(Fluent::VERSION)

data/test/test_in_cat_sweep.rb

@@ -1,9 +1,10 @@
 require_relative 'helper'
 require 'rr'
-require 'fluent/input'
 require 'fluent/plugin/in_cat_sweep'
 
 class CatSweepInputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
   def setup
     Fluent::Test.setup
     FileUtils.mkdir_p(TMP_DIR_FROM)
@@ -24,48 +25,54 @@ class CatSweepInputTest < Test::Unit::TestCase
   ]
 
   CONFIG_MINIMUM_REQUIRED =
-    if current_fluent_version < fluent_version('0.12.0') ||
-       current_fluent_version >= fluent_version('0.14.0')
-      CONFIG_BASE + %[
-        format tsv
+    CONFIG_BASE + %[
+      <parse>
+        @type tsv
         keys ""
-        waiting_seconds 5
-      ]
-    else
-      CONFIG_BASE + %[
-        format tsv
-        waiting_seconds 5
-      ]
-    end
-
-  def create_driver(conf, use_v1 = true)
-    driver = Fluent::Test::InputTestDriver.new(Fluent::CatSweepInput)
-    if current_fluent_version < fluent_version('0.10.51')
-      driver.configure(conf)
-    else
-      driver.configure(conf, use_v1)
-    end
-    driver
+      </parse>
+      waiting_seconds 3
+    ]
+
+  CONFIG_MINIMUM_REQUIRED_IN_OLD_STYLE =
+    CONFIG_BASE + %[
+      format tsv
+      keys ""
+      waiting_seconds 4
+    ]
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::CatSweepInput).configure(conf)
   end
 
   def test_required_configure
     assert_raise(Fluent::ConfigError) do
-      d = create_driver(%[])
+      create_driver(%[])
     end
 
     assert_raise(Fluent::ConfigError) do
-      d = create_driver(CONFIG_BASE)
+      create_driver(CONFIG_BASE)
     end
 
     assert_raise(Fluent::ConfigError) do
-      d = create_driver(CONFIG_BASE + %[format tsv])
+      create_driver(CONFIG_BASE + %[
+        <parse>
+          @type tsv
+          keys ""
+        </parse>
+      ])
     end
 
     d = create_driver(CONFIG_MINIMUM_REQUIRED)
 
     assert_equal "#{TMP_DIR_FROM}/*", d.instance.instance_variable_get(:@file_path_with_glob)
-    assert_equal 'tsv', d.instance.instance_variable_get(:@format)
-    assert_equal 5, d.instance.instance_variable_get(:@waiting_seconds)
+    assert_equal Fluent::Plugin::TSVParser, d.instance.instance_variable_get(:@parser).class
+    assert_equal 3, d.instance.instance_variable_get(:@waiting_seconds)
+
+    d = create_driver(CONFIG_MINIMUM_REQUIRED_IN_OLD_STYLE)
+
+    assert_equal "#{TMP_DIR_FROM}/*", d.instance.instance_variable_get(:@file_path_with_glob)
+    assert_equal Fluent::Plugin::TSVParser, d.instance.instance_variable_get(:@parser).class
+    assert_equal 4, d.instance.instance_variable_get(:@waiting_seconds)
   end
 
   def test_configure_file_event_stream
@@ -76,9 +83,9 @@ class CatSweepInputTest < Test::Unit::TestCase
     assert { true == d.instance.file_event_stream }
   end
 
-  def compare_test_result(emits, tests)
-    emits.each_index do |i|
-      assert { tests[i]['expected'] == emits[i][2]['message'] }
+  def compare_test_result(events, tests)
+    events.each_index do |i|
+      assert { tests[i]['expected'] == events[i][2]['message'] }
     end
   end
 
@@ -114,9 +121,9 @@ class CatSweepInputTest < Test::Unit::TestCase
           waiting_seconds 0
           keys hdfs_path,unixtimestamp,label,message
           ])
-        d.run
+        d.run(timeout: 1, expect_records: 2) # timeout = waiting_seconds + 1
 
-        compare_test_result(d.emits, test_cases)
+        compare_test_result(d.events, test_cases)
         assert { Dir.glob("#{TMP_DIR_FROM}/#{test_case_name}*").empty? }
       end
     end
@@ -142,9 +149,9 @@ class CatSweepInputTest < Test::Unit::TestCase
       keys hdfs_path,unixtimestamp,label,message
       move_to #{TMP_DIR_TO}
       ])
-    d.run
+    d.run(timeout: 1, expect_records: 2) # timeout = waiting_seconds + 1
 
-    compare_test_result(d.emits, test_cases)
+    compare_test_result(d.events, test_cases)
 
     assert(Dir.glob("#{TMP_DIR_FROM}/test_move_file*").empty?)
     assert_match(
@@ -177,7 +184,7 @@ class CatSweepInputTest < Test::Unit::TestCase
       oneline_max_bytes 1
       ])
 
-    d.run
+    d.run(timeout: 1, expect_records: 2) # timeout = waiting_seconds + 1
 
     assert_match(
       %r{\A#{TMP_DIR_FROM}/test_oneline_max_bytes.*\.error},

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cat-sweep
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Civitaspo(takahiro.nakayama)
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-05 00:00:00.000000000 Z
+date: 2020-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.45
+        version: 0.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.45
+        version: 0.14.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -117,6 +117,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/FUNDING.yml"
 - ".gitignore"
 - ".travis.yml"
 - CHANGELOG.md
@@ -127,10 +128,16 @@ files:
 - Rakefile
 - example.conf
 - fluent-plugin-cat-sweep.gemspec
-- gemfiles/Gemfile.fluentd.0.10
-- gemfiles/Gemfile.fluentd.0.10.45
-- gemfiles/Gemfile.fluentd.0.12
 - gemfiles/Gemfile.fluentd.0.14
+- gemfiles/Gemfile.fluentd.1.0
+- gemfiles/Gemfile.fluentd.1.1
+- gemfiles/Gemfile.fluentd.1.2
+- gemfiles/Gemfile.fluentd.1.3
+- gemfiles/Gemfile.fluentd.1.4
+- gemfiles/Gemfile.fluentd.1.5
+- gemfiles/Gemfile.fluentd.1.6
+- gemfiles/Gemfile.fluentd.1.7
+- gemfiles/Gemfile.fluentd.1.8
 - lib/fluent/plugin/in_cat_sweep.rb
 - test/helper.rb
 - test/test_in_cat_sweep.rb
@@ -153,8 +160,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to cat files and move them.