fluent-plugin-http_file_upload 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 51dbdedb4b700afa5e852a48d118711c44bbc4a8
-  data.tar.gz: fba90692103cab6e258f80b195528f695a619023
+  metadata.gz: 2620b143deb8788ff60dc04a35121e3d46784bca
+  data.tar.gz: f7fc807e21aaae3a55389957d0dab94ff948f0e5
 SHA512:
-  metadata.gz: 0a85f8c168a2da0abaade94c77b4e498070c19f0aedec0efb0a71406223b310defc5646be09c8e07c0472bc30ac9da207f7d9c7394b8b37d860149b5d38323c5
-  data.tar.gz: 84bef46d13d46574b4c19689a8837da566a6c8f106abae7dfc1920da00f244cf091634cfa940e92ca7cfc8bb4350e937f5cf2b9241a37a0edbc21ad53c06fbf0
+  metadata.gz: 2bb710641f615280bbf7169ac3d9d43a207814b6403918c2c0a9853f35ee60854102287a292b3f43cadb7750924e7d0b957e0a6bda37f88e7b029cf4ce7ac2f0
+  data.tar.gz: c7b88c29596196b1cb6fd3ad2112db370a27d743d7efff2a86130330bdcee2435e365b893b00699ca3d3c74b94aa0dd0cdda6eb774b593d836529fcf794ed32b

data/fluent-plugin-http_file_upload.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-http_file_upload"
-  spec.version       = "0.1.2"
+  spec.version       = "0.1.3"
   spec.authors       = ["TAGOMORI Satoshi"]
   spec.email         = ["tagomoris@gmail.com"]
 

data/lib/fluent/plugin/out_http_file_upload.rb

@@ -1,6 +1,7 @@
 require 'fluent/output'
 require 'fluent/mixin'
 
+require 'tempfile'
 require 'openssl'
 require 'uri'
 require 'httpclient'
@@ -21,6 +22,7 @@ module Fluent
     config_set_default :buffer_type, "file"
 
     config_param :uri, :string, desc: "Full URI for http upload endpoint for POST requests"
+
     config_param :param_name, :string, default: "file", desc: "Parameter name which contains uploaded file content"
     config_param :user_agent, :string, default: "fluent-plugin-http_file_upload", desc: "User-Agent header content"
     config_param :headers,    :hash, default: {}, desc: "Additional header fields for requests"
@@ -40,8 +42,13 @@ module Fluent
 
     config_param :format,   :string, default: "json", desc: "How to format records in uploaded files"
 
-    # TODO: support compression
-    # TODO: support gzipped transferring
+    SUPPORTED_COMPRESSION_TYPES = ['gzip']
+    config_param :compress, default: nil do |val|
+      unless SUPPORTED_COMPRESSION_TYPES.include?(val)
+        raise Fluent::ConfigError, "unsupported compression type: #{val}"
+      end
+      val
+    end
 
     def configure(conf)
       super
@@ -53,6 +60,11 @@ module Fluent
       if @uri.start_with?("https://")
         @client.ssl_config.verify_mode = @ssl_verify_mode
       end
+
+      case @compress
+      when 'gzip'
+        raise Fluent::ConfigError, "gzip command unavailable" unless system('gzip -h > /dev/null 2>&1')
+      end
     end
 
     def format(tag, time, record)
@@ -60,14 +72,62 @@ module Fluent
     end
 
     def write(chunk)
+      case @compress
+      when 'gzip'
+        write_gzip(chunk)
+      else
+        write_plain(chunk)
+      end
+    end
+
+    def write_plain(chunk)
       filename = Time.now.strftime(@filename)
       chunk.open do |io|
-        io.singleton_class.class_eval{ define_method(:path){ filename } }
-        postdata = { @param_name => io }
-        unless @parameters.empty?
-          postdata = @parameters.merge(postdata)
+        upload(io, filename)
+      end
+    end
+
+    def write_gzip(chunk)
+      filename = Time.now.strftime(@filename) + '.gz'
+      path = if chunk.respond_to?(:path)
+               chunk.path
+             else
+               w = Tempfile.new('chunk-gzip-temp-http_file_upload')
+               chunk.write_to(w)
+               w.close
+               w.path
+             end
+      tmp = Tempfile.new('gzip-temp-http_file_upload')
+      tmp.close # file will be removed after GC
+      res = system "gzip -c #{path} > #{tmp.path}"
+      unless res
+        log.warn "failed to execute gzip command: exit code '#{$?}'"
+      end
+      tmp.open
+      upload(tmp, filename)
+      tmp.close
+    end
+
+    StatDummy = Struct.new(:size)
+
+    def upload(io, filename)
+      stat_dummy = StatDummy.new(io.size)
+      io.singleton_class.class_eval{
+        define_method(:path){ filename }    # override path to feed specified filename to httpclient
+        define_method(:lstat){ stat_dummy } # override lstat to return chunk size only (lstat doesn't work for chunk buffer file)
+      }
+      postdata = { @param_name => io }
+      unless @parameters.empty?
+        postdata = @parameters.merge(postdata)
+      end
+      res = @client.post(@uri, postdata)
+      if res.status == 200
+        log.info "upload success with code 200" # TODO: make this `debug`
+      else
+        log.error "failed to upload", uri: @uri, code: res.status, content: res.content
+        if res.status >= 500 && res.status < 600
+          raise "failed to upload with ServerError. retrying."
         end
-        @client.post(@uri, postdata)
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-http_file_upload
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-26 00:00:00.000000000 Z
+date: 2016-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd