fluent-plugin-cat-sweep 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a1a1bc310740299a8f62e3f801ba1715b5bbe7b6
-  data.tar.gz: 3d9b318ff11fc7f90ab805d7d10101dd93800555
+  metadata.gz: 544f11f87dd14ada6a99e6bf6c8e8afb91c815c3
+  data.tar.gz: a8738fa50b24923cf7594ea3d30a45e1c1c5435b
 SHA512:
-  metadata.gz: 8f74d5c1faf9b9f5190b58c219ed2d3c982a3779cb7d7d5fccc86637b1e000b718a420b97120bf37eaa61d2664bf1706cea693c469d99621229da170625802f8
-  data.tar.gz: 175666731ebeb4c8f7b1115b46f0437f7ccfafcca820d47beacd8cfa1e20bc5b20d4024eae3b6af471678192ef7c652d08390598e2b51c966fe36172ca9c7cd6
+  metadata.gz: d8c2d5ffb0b448fc28e84910387f888c578fbcee264b5fd3bdc32132998a01e4ee35bc4c0741a850e0a8518cc91e98b5b5aebdb806858c62822f339c5711e1a1
+  data.tar.gz: b0840fb54a3957463bb21243e5f84974514e9024d57f1b6570aa9bdf32dca000e24573aaa00e0128dfa6a139f79f07c0b3ebe0462141eccb99dc5476ff9d0676

data/.travis.yml

@@ -1,10 +1,18 @@
 language: ruby
 rvm:
-  - 1.9.3
-  - 2.0.0
   - 2.1.*
   - 2.2.*
+  - '2.3'
+  - '2.4'
 gemfile:
+  - gemfiles/Gemfile.fluentd.0.14
   - gemfiles/Gemfile.fluentd.0.12
   - gemfiles/Gemfile.fluentd.0.10
   - gemfiles/Gemfile.fluentd.0.10.45
+
+matrix:
+  exclude:
+    - rvm: '2.4'
+      gemfile: gemfiles/Gemfile.fluentd.0.10
+    - rvm: '2.4'
+      gemfile: gemfiles/Gemfile.fluentd.0.10.45

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 0.1.4
+
+Enhancements:
+
+* Support Fluentd 0.14 with compatibility API.
+  * https://github.com/civitaspo/fluent-plugin-cat-sweep/pull/17 ( by @cosmo0920 )
+
 # 0.1.3
 
 Enhancements:

data/README.md

@@ -37,6 +37,76 @@ This plugin watches the directory (`file_path_with_glob tmp/test/access.log.*`),
 
 Our assumption is that this mechanism should provide more durability than `in_tail` (batch read overcomes than streaming read). 
 
+## Potential problem of in_tail
+
+Assume that an application outputs logs into `/tmp/test/access.log` and rotates it in every one minute interval as
+
+(initial state)
+
+```
+tmp/test
+└── accesss.log (i-node 4478316)
+```
+
+(one minute later)
+
+```
+tmp/test
+├── accesss.log (i-node 4478319)
+└── accesss.log.1 (i-node 4478316)
+```
+
+(two minutes later)
+
+```
+tmp/test
+├── accesss.log (i-node 4478322)
+├── accesss.log.1 (i-node 4478319)
+└── accesss.log.2 (i-node 4478316)
+```
+
+Your configuration of `in_tail` may become as followings:
+
+```apache
+<source>
+  @type tail
+  path tmp/test/access.log
+  pos_file /var/log/td-agent/access.log.pos
+  tag access
+  format none
+</source>
+```
+
+Now, imagine that the fluentd process dies (or manually stops for maintenance) just before the 2nd file of i-node 4478319 is generated, and you restart the fluentd process after two minutes passed. Then, you miss the 2nd file of i-node 4478319.
+
+(initial state)
+
+```
+tmp/test
+└── accesss.log (i-node 4478316) <= catch
+```
+
+(fluentd dies)
+
+(one minute later)
+
+```
+tmp/test
+├── accesss.log (i-node 4478319) <= miss
+└── accesss.log.1 (i-node 4478316)
+```
+
+(two minutes later)
+
+(fluentd restarts)
+
+```
+tmp/test
+├── accesss.log (i-node 4478322) <= catch
+├── accesss.log.1 (i-node 4478319) <= miss
+└── accesss.log.2 (i-node 4478316)
+```
+
 ## Configuration
 
 ```
@@ -51,31 +121,31 @@ Our assumption is that this mechanism should provide more durability than `in_ta
   keys xpath,access_time,label,payload
 
   # Required. process files that are older than this parameter(seconds).
-  # [WARNING!!]: this plugin move or remove files even if the files open,
-  # so this parameter is set as seconds that the application close files definitely.  
+  # [WARNING!!]: this plugin moves or removes files even if the files are still open.
+  # make sure to set this parameter for seconds that the application closes files definitely.
   waiting_seconds 60
 
   # Optional. default is file.cat_sweep
   tag test.input
 
-  # Optional. processing files is renamed with this suffix. default is .processing
+  # Optional. processing files are renamed with this suffix. default is .processing
   processing_file_suffix .processing
 
-  # Optional. error files is renamed with this suffix. default is .error
+  # Optional. error files are renamed with this suffix. default is .error
   error_file_suffix .err
 
   # Optional. line terminater. default is "\n"
   line_terminated_by ,
 
-  # Optional. max bytes oneline can has. default 536870912 (512MB)
+  # Optional. max bytes oneline can have. default 536870912 (512MB)
   oneline_max_bytes 128000
 
-  # Optional. processed files are move to this directory.
+  # Optional. processed files are moved to this directory.
   # default '/tmp'
   move_to /tmp/test_processed
 
-  # Optional. this parameter indicated, `move_to` is ignored.
-  # files that is processed are removed.
+  # Optional. if this parameter is specified, `move_to` option is ignored.
+  # processed files are removed instead of being moved to `move_to` directory.
   # default is false.
   remove_after_processing true
 
@@ -97,7 +167,6 @@ Our assumption is that this mechanism should provide more durability than `in_ta
 
 * This plugin supports fluentd from v0.10.45
 * The support for fluentd v0.10 will end near future
-* The support for fluentd v0.14 is not yet
 
 ## Contributing
 

data/fluent-plugin-cat-sweep.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-cat-sweep"
-  spec.version       = "0.1.3"
+  spec.version       = "0.1.4"
   spec.authors       = ["Civitaspo(takahiro.nakayama)", "Naotoshi Seo"]
   spec.email         = ["civitaspo@gmail.com", "sonots@gmail.com"]
 

data/gemfiles/Gemfile.fluentd.0.14

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 0.14.0'
+gemspec :path => "../"

data/lib/fluent/plugin/in_cat_sweep.rb

@@ -1,3 +1,4 @@
+require 'fluent/input'
 
 module Fluent
   class CatSweepInput < Input

data/test/test_in_cat_sweep.rb

@@ -24,7 +24,8 @@ class CatSweepInputTest < Test::Unit::TestCase
   ]
 
   CONFIG_MINIMUM_REQUIRED =
-    if current_fluent_version < fluent_version('0.12.0')
+    if current_fluent_version < fluent_version('0.12.0') ||
+       current_fluent_version >= fluent_version('0.14.0')
       CONFIG_BASE + %[
         format tsv
         keys ""

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cat-sweep
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Civitaspo(takahiro.nakayama)
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-21 00:00:00.000000000 Z
+date: 2017-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -130,6 +130,7 @@ files:
 - gemfiles/Gemfile.fluentd.0.10
 - gemfiles/Gemfile.fluentd.0.10.45
 - gemfiles/Gemfile.fluentd.0.12
+- gemfiles/Gemfile.fluentd.0.14
 - lib/fluent/plugin/in_cat_sweep.rb
 - test/helper.rb
 - test/test_in_cat_sweep.rb
@@ -153,7 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to cat files and move them.