fluent-plugin-rackspace-cloud-files 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 86f2e257bff2142ebccdffad4b9f54c8fa162c2c
-  data.tar.gz: 311e01976a6ae1787b47614705f0ca200fe52c6d
+  metadata.gz: e8adc0ac521e3f0645f1e790a383bdb5b25cff93
+  data.tar.gz: ad67f03f2cd45d669c5bca4a6cd6a9ea8e288b39
 SHA512:
-  metadata.gz: 0163df2a53171a56a30d19345b0945466482ac278915ecd9e75af19c8c672ba1ba8c8b91e2d31456595da0ac550950808fe8d8a4c16ad9400a2c74465312c54f
-  data.tar.gz: 9601891414db1399dc2ee7fb8fa7c84ff64875f8a51fe77696071825d10f05d389510343b10ae5bc5fc9158794c9116336224991db8da4f2d8f20e5fabb7e19c
+  metadata.gz: 57a75f07a66991d52d32e8f814ff4c9f9d4a9d2779cf2598d5c4d09f9fad1ab7178b982cfa46e410358950b010036fc73dff3d5461aa5b34f67f0cb8286cf3a9
+  data.tar.gz: 1e3f74b993b87be01e675ff60c5863f299f60f0252b2a8fac4d35cc369eb1f934924edb3b8c053caaec905e215a5fc0c5eac88ae0d744e40eeaf2d8e4ee76d6c

data/.rubocop.yml

@@ -0,0 +1,11 @@
+Lint/Loop:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Style/RegexpLiteral:
+  Enabled: false

data/.travis.yml

@@ -4,5 +4,5 @@ rvm:
   - 2.1.5  # Version of Ruby that ships with fluentd
   - 2.2.3
 script:
-  # - bundle exec rake style  # Not currently passing style checks
+  - bundle exec rake style
   - bundle exec rake build

data/README.rdoc

@@ -4,6 +4,7 @@
 
 {<img src="https://travis-ci.org/brint/fluent-plugin-rackspace-cloud-files.svg" alt="Build Status" />}[https://travis-ci.org/brint/fluent-plugin-rackspace-cloud-files]
 {<img src="https://badge.fury.io/rb/fluent-plugin-rackspace-cloud-files.svg" alt="Gem Version" />}[http://badge.fury.io/rb/fluent-plugin-rackspace-cloud-files]
+{<img src="https://gemnasium.com/brint/fluent-plugin-rackspace-cloud-files.svg" alt="Dependency Status" />}[https://gemnasium.com/brint/fluent-plugin-rackspace-cloud-files]
 
 This has only been tested on fluentd version 0.12.12.
 

data/VERSION

@@ -1 +1 @@
-0.0.3
+0.0.4

data/fluent-plugin-rackspace-cloud-files.gemspec

@@ -26,5 +26,5 @@ Gem::Specification.new do |gem|
                              '>= 0.3.0'
   gem.add_development_dependency 'rake', '~> 10.1', '>= 10.1.0'
   gem.add_development_dependency 'flexmock', '~> 1.2', '>= 1.2.0'
-  gem.add_development_dependency 'rubocop', '~> 0.33.0', '>= 0.33.0'
+  gem.add_development_dependency 'rubocop', '~> 0.33', '>= 0.33'
 end

data/lib/fluent/plugin/out_rackspace_cloud_files.rb

@@ -1,8 +1,7 @@
-module Fluent
-
+module Fluent #:nodoc: all
   require 'fluent/mixin/config_placeholders'
 
-  class RackspaceCloudFilesOutput < TimeSlicedOutput
+  class RackspaceCloudFilesOutput < Fluent::TimeSlicedOutput
     Fluent::Plugin.register_output('rackspace_cloud_files', self)
 
     def initialize
@@ -11,11 +10,10 @@ module Fluent
       require 'zlib'
       require 'time'
       require 'tempfile'
-      require 'open3'
     end
 
-    config_param :path, :string, :default => ""
-    config_param :time_format, :string, :default => nil
+    config_param :path, :string, default: ''
+    config_param :time_format, :string, default: nil
 
     include SetTagKeyMixin
     config_set_default :include_tag_key, false
@@ -23,18 +21,21 @@ module Fluent
     include SetTimeKeyMixin
     config_set_default :include_time_key, false
 
-    config_param :rackspace_auth_url, :string, :default => "https://identity.api.rackspacecloud.com/v2.0"
+    config_param :rackspace_auth_url, :string,
+                 default: 'https://identity.api.rackspacecloud.com/v2.0'
     config_param :rackspace_username, :string
     config_param :rackspace_api_key, :string
     config_param :rackspace_container, :string
     config_param :rackspace_region, :string
 
-    config_param :object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
-    config_param :store_as, :string, :default => "gzip"
-    config_param :auto_create_container, :bool, :default => true
-    config_param :check_apikey_on_start, :bool, :default => true
-    config_param :proxy_uri, :string, :default => nil
-    config_param :ssl_verify, :bool, :default => true
+    config_param :object_key_format, :string,
+                 default: '%{path}%{time_slice}_%{index}.%{file_extension}'
+    config_param :store_as, :string, default: 'gzip'
+    config_param :auto_create_container, :bool, default: true
+    config_param :check_apikey_on_start, :bool, default: true
+    config_param :proxy_uri, :string, default: nil
+    config_param :ssl_verify, :bool, default: true
+    config_param :format_json, :bool, default: false
 
     # attr_reader :container
 
@@ -46,58 +47,30 @@ module Fluent
 
     def configure(conf)
       super
-
-      if format_json = conf['format_json']
-        @format_json = true
-      else
-        @format_json = false
-      end
-
-      @ext, @mime_type = case @store_as
-        when 'gzip' then ['gz', 'application/x-gzip']
-        when 'json' then ['json', 'application/json']
-        else ['txt', 'text/plain']
-      end
-
+      @ext, @mime_type = storage_method
       @timef = TimeFormatter.new(@time_format, @localtime)
-
-      if @localtime
-        @path_slicer = Proc.new {|path|
-          Time.now.strftime(path)
-        }
-      else
-        @path_slicer = Proc.new {|path|
-          Time.now.utc.strftime(path)
-        }
-      end
+      @path_slicer = time_slicer
     end
 
     def start
       super
-
       Excon.defaults[:ssl_verify_peer] = @ssl_verify
-
-      @storage = Fog::Storage.new :provider => 'Rackspace',
-                        :rackspace_auth_url => @rackspace_auth_url,
-                        :rackspace_username => @rackspace_username,
-                        :rackspace_api_key => @rackspace_api_key,
-                        :rackspace_region => @rackspace_region
+      @storage = Fog::Storage.new provider: 'Rackspace',
+                                  rackspace_auth_url: @rackspace_auth_url,
+                                  rackspace_username: @rackspace_username,
+                                  rackspace_api_key: @rackspace_api_key,
+                                  rackspace_region: @rackspace_region
 
       check_container
     end
 
     def format(tag, time, record)
-      if @include_time_key || !@format_json
-        time_str = @timef.format(time)
-      end
+      time_str = @timef.format(time) if @include_time_key || !@format_json
 
-      # copied from each mixin because current TimeSlicedOutput can't support mixins.
-      if @include_tag_key
-        record[@tag_key] = tag
-      end
-      if @include_time_key
-        record[@time_key] = time_str
-      end
+      # copied from each mixin because current TimeSlicedOutput can't support
+      # mixins.
+      record[@tag_key] = tag if @include_tag_key
+      record[@time_key] = time_str if @include_time_key
 
       if @format_json
         Yajl.dump(record) + "\n"
@@ -108,65 +81,103 @@ module Fluent
 
     def write(chunk)
       i = 0
+      previous_path = nil
 
       begin
         path = @path_slicer.call(@path)
         values_for_swift_object_key = {
-          "path" => path,
-          "time_slice" => chunk.key,
-          "file_extension" => @ext,
-          "index" => i
-        }
-        swift_path = @object_key_format.gsub(%r(%{[^}]+})) { |expr|
-          values_for_swift_object_key[expr[2...expr.size-1]]
+          'path' => path,
+          'time_slice' => chunk.key,
+          'file_extension' => @ext,
+          'index' => i
         }
+        swift_path = @object_key_format.gsub(%r(%{[^}]+})) do |expr|
+          values_for_swift_object_key[expr[2...expr.size - 1]]
+        end
+        if (i > 0) && (swift_path == previous_path)
+          fail 'duplicated path is generated. use %{index} in '\
+          "object_key_format: path = #{swift_path}"
+        end
         i += 1
+        previous_path = swift_path
       end while check_object_exists(@rackspace_container, swift_path)
+      write_temp_file(chunk, swift_path)
+    end
 
-      tmp = Tempfile.new("rackspace-cloud-files-")
-      begin
-        if @store_as == "gzip"
-          w = Zlib::GzipWriter.new(tmp)
-          chunk.write_to(w)
-          w.close
-        else
-          chunk.write_to(tmp)
-          tmp.close
+    private
+
+    # Configure methods
+    def storage_method
+      case @store_as
+      when 'gzip' then return ['gz', 'application/x-gzip']
+      when 'json' then return ['json', 'application/json']
+      else return ['txt', 'text/plain']
+      end
+    end
+
+    def time_slicer
+      if @localtime
+        return proc do |path|
+          Time.now.strftime(path)
         end
-        File.open(tmp.path) do |file|
-          @storage.put_object(@rackspace_container, swift_path, file, {:content_type => @mime_type})
+      else
+        return proc do |path|
+          Time.now.utc.strftime(path)
         end
-        $log.info "Put Log to Rackspace Cloud Files. container=#{@rackspace_container} object=#{swift_path}"
-      ensure
-        tmp.close(true) rescue nil
-        w.close rescue nil
-        w.unlink rescue nil
       end
     end
 
-    private
-
+    # Start methods
     def check_container
-      begin
-        @storage.get_container(@rackspace_container)
-      rescue Fog::Storage::Rackspace::NotFound
-        if @auto_create_container
-          $log.info "Creating container #{@rackspace_container} in region #{@rackspace_region}"
-          @storage.put_container(@rackspace_container)
-        else
-          raise "The specified container does not exist: container = #{rackspace_container}"
-        end
+      @storage.get_container(@rackspace_container)
+    rescue Fog::Storage::Rackspace::NotFound
+      if @auto_create_container
+        log.info 'Creating container #{@rackspace_container} in region '\
+                  "#{@rackspace_region}"
+        @storage.put_container(@rackspace_container)
+      else
+        raise 'The specified container does not exist: container = '\
+              "#{rackspace_container}"
       end
     end
 
+    # Write methods
     def check_object_exists(container, object)
+      @storage.head_object(container, object)
+      return true
+    rescue Fog::Storage::Rackspace::NotFound
+      return false
+    end
+
+    def write_temp_file(chunk, swift_path)
+      tmp = Tempfile.new('rackspace-cloud-files-')
       begin
-        @storage.head_object(container, object)
-      rescue Fog::Storage::Rackspace::NotFound
-        return false
+        if @store_as == 'gzip'
+          w = write_gzip_temp_file(chunk, tmp)
+        else
+          chunk.write_to(tmp)
+          tmp.close
+        end
+        write_temp_file_to_cloud_files(tmp, swift_path)
+      ensure
+        tmp.close(true)
+        w.close(true)
       end
-      return true
     end
 
+    def write_gzip_temp_file(chunk, tmp)
+      w = Zlib::GzipWriter.new(tmp)
+      chunk.write_to(w)
+      w.close
+    end
+
+    def write_temp_file_to_cloud_files(tmp, swift_path)
+      File.open(tmp.path) do |file|
+        @storage.put_object(@rackspace_container, swift_path, file,
+                            content_type: @mime_type)
+      end
+      log.info 'Put Log to Rackspace Cloud Files. container='\
+                "#{@rackspace_container} object=#{swift_path}"
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rackspace-cloud-files
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Brint O'Hearn
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-05 00:00:00.000000000 Z
+date: 2015-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -136,20 +136,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.33.0
+        version: '0.33'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.33.0
+        version: '0.33'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.33.0
+        version: '0.33'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.33.0
+        version: '0.33'
 description: Rackspace Cloud Files output plugin for Fluent event collector
 email: brintly@gmail.com
 executables: []
@@ -157,6 +157,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rubocop.yml"
 - ".travis.yml"
 - Gemfile
 - LICENSE.md