fluent-plugin-file-sprintf 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9d163ac14dd07900df0593a1f4af13c36c0e8f90
-  data.tar.gz: 3dd2e2847944b971e4282f5fd677e078338c164c
+  metadata.gz: 1ba817afaccb6b510811ffb6855dd56d498dd967
+  data.tar.gz: 4528ecde8a9397a30d29075aed0e32285638d50e
 SHA512:
-  metadata.gz: c81ea6d448be12f508575718c6984e7a745bdeb447ecfd8336fb8795e6e5467f98caa4dba7be9ab6c4d1ae577ff5f7824516b629c6ca79f0dad581a9a88d57f1
-  data.tar.gz: e4b654ae0bd9045b9d38a7014136880c934df7772ae1e64c5e83b28c80d80ef98f5319e988108726e3703f40cd9322685c47559052c14c90f75982ebda453f98
+  metadata.gz: 2a992d7e22d56f32a0f0f1f04ca0d149a6ebfa8ceee74be29d28281d0e2cf0fcfff4d0f39556daefbc7bc296c82d80fa203c8577e9e15efc36d91f0efa42d09b
+  data.tar.gz: 23211010a1fec9519c423ce62b8cfe582826abfc7e0d22fff0bd31a9f62007bd6cebb89e98298297b260ba7b047451bca43adc91c5cd34a1c227c5e06b708abd

data/README.md

@@ -26,11 +26,11 @@ path|output file path(require)|/tmp/test.ltsv
 format|sprintf format(require)|%s
 key_names|key names comma separator(require)|ltsv
 time_format|time value output format(default:%Y-%m-%d %H:%M:%S)|%Y-%m-%d %H:%M:%S
-file_size_limit|log rotate file size limit byte(default 8388608)|31457280
 include_tag_key|tag key in record|true
 tag_key_name|tag key name(default:tag)|tag_name
 include_time_key|time key in record|true
 time_key_name|time key name(default:time)|timestamp
+rotate_format|file rotate format(default:%Y%m%d)|%Y%m%d
 
 ## key_names reserved words
 
@@ -52,10 +52,8 @@ json|output json string
 		path /tmp/apache.json
 		format %s
 		key_names json
-		file_size_limit 31457280
 	</match>
 
-
 ## Configuration Exsample(labeled tsv format)
 
 	<match apache.ltsv>
@@ -65,7 +63,6 @@ json|output json string
 		path /tmp/apache.ltsv
 		format %s
 		key_names ltsv
-		file_size_limit 31457280
 	</match>
 
 ## Configuration Exsample(message pack format)
@@ -77,7 +74,6 @@ json|output json string
 		path /tmp/apache.msgpack
 		format %s
 		key_names msgpack
-		file_size_limit 31457280
 	</match>
 
 ## Configuration Exsample(custom json format)
@@ -89,7 +85,6 @@ json|output json string
 		path /tmp/apache.json
 		format {"method":"%s","agent":"%s","referer":"%s","path":"%s","host":"%s","time":"%s","tag":"%s"}
 		key_names method,agent,referer,path,host,time,tag
-		file_size_limit 31457280
 	</match>
 
 ## Configuration Exsample(tsv format)
@@ -101,7 +96,6 @@ json|output json string
 		path /tmp/apache.tsv
 		format %s\t%s\t%s\t%s\t%s\t%s\t%s
 		key_names method,agent,referer,path,host,time,tag
-		file_size_limit 31457280
 	</match>
 
 ## Configuration Exsample(elasticsearch bulk import format. multiline format)
@@ -113,7 +107,6 @@ json|output json string
 		path /tmp/es.json
 		format { "index" : { "_index" : "test_index", "_type" : "test_type" } }\n%s
 		key_names json
-		file_size_limit 31457280
 	</store>
 
 

data/fluent-plugin-file-sprintf.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-file-sprintf"
-  spec.version       = "0.0.3"
+  spec.version       = "0.0.4"
   spec.authors       = ["Hiroshi Toyama"]
   spec.email         = ["toyama0919@gmail.com"]
   spec.description   = %q{sprintf output file plugin for Fluentd.}

data/lib/fluent/plugin/out_file_sprintf.rb

@@ -4,7 +4,6 @@ module Fluent
     Fluent::Plugin.register_output('file_sprintf', self)
 
     config_param :path, :string
-    config_param :file_size_limit, :integer ,:default => 8388608
     config_param :format, :string
     config_param :compress, :bool, :default => true
     config_param :include_tag_key, :bool, :default => false
@@ -13,6 +12,8 @@ module Fluent
     config_param :time_key_name, :string, :default => "time"
     config_param :key_names, :string
     config_param :time_format, :string, :default => "%Y-%m-%d %H:%M:%S"
+    config_param :rotate_format, :string, :default => "%Y%m%d"
+    config_param :file_prefix_key, :string, :default => "Time.at(time).strftime(@rotate_format)"
 
     def initialize
       super
@@ -51,6 +52,7 @@ module Fluent
       @key_names = @key_names.join(',')
       @eval_string = "%Q{#{@format}} % [#{@key_names}]"
       $log.info "format => #{@eval_string}"
+      $log.info "flush_interval => #{@flush_interval}"
     end
 
     def format(tag, time, record)
@@ -64,28 +66,45 @@ module Fluent
     end
 
     def write(chunk)
-      filepath = @path
-      file_size = 0
-      File.open(filepath,'a') { |file|
-        chunk.msgpack_each do |tag, time, record|
-          result = eval(@eval_string)
-          file.puts result
-        end
-        file_size = file.size
+      write_file(chunk)
+      compress_file
+    end
+
+    private
+    def write_file(chunk)
+      set = Set.new
+      chunk.msgpack_each do |tag, time, record|
+        set.add(eval(@file_prefix_key))
+      end
+
+      filename_hash = {}
+      set.each{|prefix|
+        filename_hash[prefix] = File.open(@path + '.' + prefix,'a')
+      }
+
+      chunk.msgpack_each do |tag, time, record|
+        result = eval(@eval_string)
+        file = filename_hash[eval(@file_prefix_key)]
+        file.puts result
+      end
+
+      filename_hash.each{|k,v|
+        v.close
       }
+    end
 
-      if file_size > @file_size_limit
-        output_path = @path + "." + "#{Time.now.strftime('%Y%m%d%H%M%S')}"
-        if @compress
-          Zlib::GzipWriter.open(output_path + ".gz") do |gz|
-            gz.mtime = File.mtime(filepath)
-            gz.orig_name = filepath
-            gz.write IO.binread(filepath)
+    def compress_file
+      if @compress
+        Dir.glob( "#{@path}.*[^gz]" ).each{ |output_path|
+          if Time.now > File.mtime(output_path) + (@flush_interval * 5)
+            Zlib::GzipWriter.open(output_path + ".gz") do |gz|
+              gz.mtime = File.mtime(output_path)
+              gz.orig_name = output_path
+              gz.write IO.binread(output_path)
+            end
+            FileUtils.remove_file(output_path, force = true)
           end
-        else
-          FileUtils.cp filepath, output_path
-        end
-        FileUtils.remove_file(filepath, force = true)
+        }
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-file-sprintf
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Hiroshi Toyama
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-15 00:00:00.000000000 Z
+date: 2014-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ltsv
@@ -86,7 +86,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0.rc.1
+rubygems_version: 2.2.0
 signing_key: 
 specification_version: 4
 summary: sprintf output file plugin for Fluentd.