fluent-plugin-file-alternative 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 717fa824f86b1d9e7efd9a2c72bc90745886bd85
-  data.tar.gz: 53299516fbc595bc4083fa1859f76764759ab003
+  metadata.gz: ea52057a062ad50131a6e1cf865c06c7bb8b69d2
+  data.tar.gz: efd08dcdc758d6f6ffab1b1767ecf613ea9a7cc9
 SHA512:
-  metadata.gz: 5bb40a237167cb760d77ef7c9d50b8ee390f12a541d633f79d656e7c64efb88ec4f12e1ef3d5513dcf32fc0b62a45fd69d1642e8e178cb61eb35812351050295
-  data.tar.gz: fd7c06f49331ce0b890bc2e38a2b6db7ad387cc2ce7bfe3746f39b82c78526277938a02259384b6eb77bf4dc04520567b316bb032640ff2cabf95b5c677eec2f
+  metadata.gz: c78ef8f78cb7cbaf29ae78287beaba67d6267624599b3116bf9aa69050a158175e5fba4ec50c125ed9adb1a959f398dd20e55aa0e32d6b231b651023e9bb6f30
+  data.tar.gz: 3d618956f885a15c80984cec1da9425783e8332fa3912bc0218e56e3b6c5d8a30a705865e271cc9c02055b1279d1affe0819d2dd0e116616bbd9f2590ed67f3f

data/README.md

@@ -1,7 +1,6 @@
 # fluent-plugin-file-alternative 
 
-File output plugin alternative implementation, **is 100% compatible with
-[Fluentd](http://fluentd.org) built-in 'out_file'**, and added many options to format output as you want.
+File output plugin alternative implementation.
 
 FileAlternativeOutput slices data by time (for specified units), and store these data as plain text on local file. You can specify to:
 
@@ -14,19 +13,19 @@ FileAlternativeOutput slices data by time (for specified units), and store these
 And you can specify output file path as:
 
 * Standard out_file way
-  * configure 'path /path/to/dir/access'
-  * and 'time\_slice\_format %Y%m%d'
-  * got '/path/to/dir/access.20120316.log'
+  * configure `path /path/to/dir/access`
+  * and `time_slice_format %Y%m%d`
+  * got `/path/to/dir/access.20120316.log`
+* Standard out_file way (alternative)
+  * configure `path /path/to/dir/access.*.log`
+  * and `time_slice_format %Y%m%d`
+  * got `/path/to/dir/access.20120316.log`
 * Alternative style
-  * configure 'path /path/to/dir/access.%Y%m%d.log' only
-  * got '/path/to/dir/access.20120316.log'
+  * configure `path /path/to/dir/access.%Y%m%d.log` only (don't include `*`)
+  * got `/path/to/dir/access.20120316.log`
 
 And, gzip compression is also supported.
 
--### Why this is not a patch for out_file?
--
--`fluent-plugin-file-alternative` has optimized buffer data structure to make faster to write data on disk. But that buffer structure is not compatible with `out_file`'s one. That's tha reason why this plugin is an another plugin from `out_file`.
-
 ## Configuration
 
 ### FileAlternativeOutput
@@ -36,7 +35,7 @@ Standard out_file way (hourly log, compression, time-tag-json):
     <match out.**>
       type file_alternative
       path /var/log/service/access.*.log
-      time_slice_output %Y%m%d_%H
+      time_slice_format %Y%m%d_%H
       compress gzip
     </match>
 
@@ -57,6 +56,8 @@ If you don't want fluentd-time and tag in written file, and messages with single
       output_include_tag false
       output_data_type attr:message
       add_newline false
+      symlink_path /var/log/service/access_latest.log
+      dir_mode 0777
     </match>
 
 Then, you will get:

data/fluent-plugin-file-alternative.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-file-alternative"
-  gem.version       = "0.1.5"
+  gem.version       = "0.2.0"
 
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
@@ -19,4 +19,5 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "fluentd", ">= 0.10.39" # This version for @buffer.symlink_path
   gem.add_runtime_dependency "fluent-mixin-plaintextformatter"
   gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit"
 end

data/lib/fluent/plugin/out_file_alternative.rb

@@ -27,6 +27,8 @@ class Fluent::FileAlternativeOutput < Fluent::TimeSlicedOutput
 
   config_param :symlink_path, :string, :default => nil
 
+  config_param :dir_mode, :string, :default => '0777'
+
   include Fluent::Mixin::PlainTextFormatter
 
   def initialize
@@ -133,7 +135,12 @@ class Fluent::FileAlternativeOutput < Fluent::TimeSlicedOutput
     path = path_format(chunk.key)
 
     begin
-      FileUtils.mkdir_p File.dirname(path)
+      require 'pathname'
+
+      Pathname.new(path).descend {|p|
+        FileUtils.mkdir_p( File.dirname(p)) unless File.directory?(p)
+        FileUtils.chmod @dir_mode.to_i(8), File.dirname(p) unless File.directory?(p)
+      }
 
       case @compress
       when :gz

data/test/plugin/test_out_file_alternative.rb

@@ -31,6 +31,7 @@ class FileAlternativeOutputTest < Test::Unit::TestCase
 
     d = create_driver %[
       path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
+      dir_mode 0700
     ]
     assert_equal '%Y%m%d%H%M%S', d.instance.time_slice_format
     assert_nil d.instance.compress
@@ -39,6 +40,7 @@ class FileAlternativeOutputTest < Test::Unit::TestCase
     assert_equal 'json', d.instance.output_data_type
     assert_equal true, d.instance.add_newline
     assert_equal "TAB", d.instance.field_separator
+    assert_equal '0700', d.instance.dir_mode
     assert_nil d.instance.remove_prefix
   end
 
@@ -109,7 +111,7 @@ class FileAlternativeOutputTest < Test::Unit::TestCase
     d2.expect_format %[123-456-789\n]
     path = d2.run
     assert_equal "#{TMP_DIR}/accesslog.2011-01-02-13-14-15", path[0]
-    
+
     d3 = create_driver %[
       path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
       compress gzip
@@ -129,6 +131,20 @@ class FileAlternativeOutputTest < Test::Unit::TestCase
     d3.expect_format %[app01,info,Send response\n]
     path = d3.run
     assert_equal "#{TMP_DIR}/accesslog.2011-01-02-13-14-15.gz", path[0]
+
+    d4 = create_driver %[
+      path #{TMP_DIR}/path_to_test/%Y/%m/%d/accesslog.%Y-%m-%d-%H-%M-%S
+      dir_mode 0700
+    ]
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d4.emit({"server" => "www01", "level" => "warn", "log" => "Exception\n"}, time)
+    path = d4.run
+    assert_equal '40700', File.stat(File.dirname(File.dirname(File.dirname(path[0])))).mode.to_s(8)
+    assert_equal '40700', File.stat(File.dirname(File.dirname(path[0]))).mode.to_s(8)
+    assert_equal '40700', File.stat(File.dirname(path[0])).mode.to_s(8)
+    assert_equal "#{TMP_DIR}/path_to_test/2011/01/02/accesslog.2011-01-02-13-14-15", path[0]
+
   end
 
   def test_write_with_symlink
@@ -143,16 +159,17 @@ class FileAlternativeOutputTest < Test::Unit::TestCase
     d.run
     assert File.exists?(symlink_path)
     assert File.symlink?(symlink_path)
-    
+
     d.instance.start
     begin
       10.times { sleep 0.05 }
       d.instance.enqueue_buffer
-      
+
       assert !File.exists?(symlink_path)
       assert File.symlink?(symlink_path)
     ensure
       d.instance.shutdown
     end
   end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-file-alternative
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-07 00:00:00.000000000 Z
+date: 2015-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: alternative implementation of out_file, with various configurations
 email:
 - tagomoris@gmail.com
@@ -69,7 +83,6 @@ files:
 - lib/fluent/plugin/out_file_alternative.rb
 - test/helper.rb
 - test/plugin/test_out_file_alternative.rb
-- test/plugin/test_out_file_original.rb
 homepage: https://github.com/tagomoris/fluent-plugin-file-alternative
 licenses:
 - APLv2
@@ -90,11 +103,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: alternative implementation of out_file
 test_files:
 - test/helper.rb
 - test/plugin/test_out_file_alternative.rb
-- test/plugin/test_out_file_original.rb
+has_rdoc: 

data/test/plugin/test_out_file_original.rb

@@ -1,81 +0,0 @@
-require 'fluent/test'
-require 'fileutils'
-require 'time'
-
-class FileOutputTest < Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-    FileUtils.rm_rf(TMP_DIR)
-    FileUtils.mkdir_p(TMP_DIR)
-  end
-
-  TMP_DIR = File.dirname(__FILE__) + "/../tmp"
-
-  CONFIG = %[
-    path #{TMP_DIR}/out_file_test
-    compress gz
-    utc
-  ]
-
-  def create_driver(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::FileOutput).configure(conf)
-  end
-
-  def test_configure
-    d = create_driver %[
-      path test_path
-      compress gz
-    ]
-    assert_equal 'test_path', d.instance.path
-    assert_equal :gz, d.instance.compress
-  end
-
-  def test_format
-    d = create_driver
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
-    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
-
-    d.run
-  end
-
-  def test_write
-    d = create_driver
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    # FileOutput#write returns path
-    path = d.run
-    expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
-    assert_equal expect_path, path
-
-    data = Zlib::GzipReader.open(expect_path) {|f| f.read }
-    assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
-                    %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
-                 data
-  end
-
-  def test_write_path_increment
-    d = create_driver
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    # FileOutput#write returns path
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test._0.log.gz", path
-
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test._1.log.gz", path
-
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test._2.log.gz", path
-  end
-end