fluent-plugin-cat-sweep 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c359caaffa952bd5d22a4e7a446bd817b9431c9
-  data.tar.gz: b21ad62093a0737d4d67fe2a4e760d85a6593c68
+  metadata.gz: 9d379d06372f06e8fc8930c01c1cb35126344f56
+  data.tar.gz: 12bc664edebb79f7d0db631234b2cd2077b86bcc
 SHA512:
-  metadata.gz: 8bdd6442226a93d646122f4b2c42399b3fd2388e40b20f483830275b858256619123acdf6853b3e34d90ade69dcfe8714d2da16243b9e83a98d28f46bbc7bc23
-  data.tar.gz: d76f94192d97fcdfa559b0e66fec2efc6f3e68f97f79c5c89cf51190bd895807891430ccfe2207aa366e0cda5c1b95d60fd9074fb40859a23bbcd728d9c88d43
+  metadata.gz: 9c09d4acc0ecc381aa599b15995275b637f5c55def2df29ec2ede02e5c296d45923527bb39b7449875ed9515f94991136ffff520f44936411e9d05a7630a6b9c
+  data.tar.gz: 317727c30f4d959895db9e60ce2f995456000df1524c4907bfb4c50354aba22e904c51761c062d9be03727bda1cb3d4c934f7a957f56cb826a728388f46fe4a1

data/README.md

@@ -1,6 +1,6 @@
 # fluent-plugin-cat-sweep
 
-[![Build Status](https://secure.travis-ci.org/civitaspo/fluent-plugin-cat-sweep.png?branch=master)](http://travis-ci.org/civitaspo/fluent-plugin-cat-sweep)
+[![Circle CI](https://circleci.com/gh/civitaspo/fluent-plugin-cat-sweep.svg?style=svg)](https://circleci.com/gh/civitaspo/fluent-plugin-cat-sweep) [![Build Status](https://secure.travis-ci.org/civitaspo/fluent-plugin-cat-sweep.png?branch=master)](http://travis-ci.org/civitaspo/fluent-plugin-cat-sweep)
 
 Fluentd plugin to read data from files and to remove or move after processing.
 
@@ -20,6 +20,23 @@ Or install it yourself as:
 
     $ gem install fluent-plugin-cat-sweep
 
+## Basic Behavior
+
+Assume that an application outputs logs into `/tmp/test` directory as
+
+```
+tmp/test
+├── accesss.log.201509151611
+├── accesss.log.201509151612
+└── accesss.log.201509151613
+```
+
+in every one minute interval. 
+
+This plugin watches the directory (`file_path_with_glob tmp/test/access.log.*`), and reads the contents and sweep (deafault: remove) for files whose mtime are passed in 60 seconds (can be configured with `waiting_seconds`).
+
+Our assumption is that this mechanism should provide more durability than `in_tail` (batch read overcomes than streaming read). 
+
 ## Configuration
 
 ```

data/fluent-plugin-cat-sweep.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-cat-sweep"
-  spec.version       = "0.0.1"
+  spec.version       = "0.1.0"
   spec.authors       = ["Civitaspo(takahiro.nakayama)"]
   spec.email         = ["civitaspo@gmail.com"]
 

data/lib/fluent/plugin/in_cat_sweep.rb

@@ -50,8 +50,20 @@ module Fluent
         raise Fluent::ConfigError, "in_cat_sweep: `line_terminated_by` must has some letters."
       end
 
-      if !remove_file? and !Dir.exists?(@move_to)
-        raise Fluent::ConfigError, "in_cat_sweep: `move_to` directory must be existed."
+      if !remove_file?
+        first_filename = Dir.glob(@file_path_with_glob).first
+        dirname = first_filename ? move_dirname(first_filename) : @move_to
+        if Dir.exist?(dirname)
+          if !File.writable?(dirname)
+            raise Fluent::ConfigError, "in_cat_sweep: `move_to` directory (#{dirname}) must be writable."
+          end
+        else
+          begin
+            FileUtils.mkdir_p(dirname)
+          rescue => e
+            raise Fluent::ConfigError, "in_cat_sweep: `move_to` directory (#{dirname}) must be writable."
+          end
+        end
       end
 
       @read_bytes_once = 262144 # 256 KB
@@ -82,10 +94,9 @@ module Fluent
               after_processing(processing_filename)
             end
           rescue => e
-            log.error "in_cat_sweep: processing error: #{e}, file: #{processing_filename}",
-              :error => e, :error_class => e.class
+            log.error "in_cat_sweep: processing: #{processing_filename}", :error => e, :error_class => e.class
             log.error_backtrace
-            safe_fail(processing_filename)
+            safe_fail(e, processing_filename)
           end
         end
       end
@@ -112,19 +123,26 @@ module Fluent
     def get_processing_filename(filename)
       tmpfile = String.new
       tmpfile << filename << '.' << Process.pid.to_s << '.'
-      tmpfile << Time.now.to_i.to_s << '.' << @processing_file_suffix
+      tmpfile << Time.now.to_i.to_s << @processing_file_suffix
+    end
+
+    def revert_processing_filename(processing_filename)
+      tmpfile = processing_filename.dup
+      tmpfile.chomp!(@processing_file_suffix)
+      tmpfile.gsub!(/\.\d+\.\d+$/, '')
     end
 
-    def get_error_filename(filename)
+    def get_error_filename(e, filename)
       errfile = String.new
-      errfile << filename << '.' << @error_file_suffix
+      errfile << filename << "." << e.class.to_s << @error_file_suffix
     end
 
-    def safe_fail(filename)
+    def safe_fail(e, filename)
       begin
-        lock_with_renaming(filename, get_error_filename(filename))
+        error_filename = get_error_filename(e, filename)
+        lock_with_renaming(filename, error_filename)
       rescue => e
-        log.error "in_cat_sweep: rename #{filename} to error name. message: #{e}",
+        log.error "in_cat_sweep: rename #{filename} to error filename #{error_filename}",
           :error => e, :error_class => e.class
         log.error_backtrace
       end
@@ -201,11 +219,18 @@ module Fluent
       end
     end
 
-    def after_processing(filename)
+    def move_dirname(filename)
+      File.join(@move_to, File.dirname(File.expand_path(filename)))
+    end
+
+    def after_processing(processing_filename)
       if remove_file?
-        FileUtils.rm(filename)
+        FileUtils.rm(processing_filename)
       else
-        FileUtils.mv(filename, @move_to)
+        dirname = move_dirname(processing_filename)
+        FileUtils.mkdir_p(dirname)
+        filename = revert_processing_filename(File.basename(processing_filename))
+        FileUtils.mv(processing_filename, File.join(dirname, filename))
       end
     end
   end

data/test/test_in_cat_sweep.rb

@@ -117,11 +117,11 @@ class CatSweepInputTest < Test::Unit::TestCase
 
     assert(Dir.glob("#{TMP_DIR_FROM}/test_move_file*").empty?)
     assert_match(
-      %r{\A#{TMP_DIR_TO}/test_move_file.*\.processing},
-      Dir.glob("#{TMP_DIR_TO}/test_move_file*").first)
+      %r{\A#{TMP_DIR_TO}#{TMP_DIR_FROM}/test_move_file},
+      Dir.glob("#{TMP_DIR_TO}#{TMP_DIR_FROM}/test_move_file*").first)
     assert_equal(
       test_cases.map{|t|t['msg']}.join.to_s,
-      File.read(Dir.glob("#{TMP_DIR_TO}/test_move_file*").first))
+      File.read(Dir.glob("#{TMP_DIR_TO}#{TMP_DIR_FROM}/test_move_file*").first))
   end
 
   def test_oneline_max_bytes

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cat-sweep
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Civitaspo(takahiro.nakayama)
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-16 00:00:00.000000000 Z
+date: 2015-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd