fluent-plugin-elb-log 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 71fea708525f12dea801f2e95871eae3e3361faa
-  data.tar.gz: b43655a000f3509012cfb5530839ca88c5201fcc
+  metadata.gz: 1072c05095bf58f28fcc18c5d2c2dd74de3b761f
+  data.tar.gz: a6584249f0bef727d7495870d86f70db2deb8f78
 SHA512:
-  metadata.gz: a6e9a47ae3fd4cc4bc25e2c6b702bdcf6524b998f8844d7afa22f86fbb22e3220e984f4e44092a333112f64f5b081fcd8f42f174ccdbe99e3bf0770864292f15
-  data.tar.gz: c2654c3fc22772b407fe9ca0d06022373544e0011782b5d29f32c662b4b30a344d0fe8b8b991b0a3c0a2e0285842ba87609f3f8e075b2e9e415ca2304c32042f
+  metadata.gz: a5e800e7091a55b7144ff28b99263666fcee7980f6663dcda08cd411a49d35fee6508ea64dd4b4509db351e0fc5faf35d95b0a11dd609f6257c23b07ab41f116
+  data.tar.gz: fc32e6358c926b332b7796b8d636a4b153dafbfab3b0f9ee40be11a75f490ae0bfa7d232e9ac62e66cf08bbfa82d2eecaa592d03c1c95bf8a816cf4ea62a8cb7

data/fluent-plugin-elb-log.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-elb-log"
-  spec.version       = "0.4.2"
+  spec.version       = "0.4.3"
   spec.authors       = ["shinsaka"]
   spec.email         = ["shinx1265@gmail.com"]
   spec.summary       = "Amazon ELB log input plugin"

data/lib/fluent/plugin/in_elb_log.rb

@@ -137,26 +137,29 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
         "key" => object_key[:key],
         "prefix" => object_key[:prefix],
         "elb_timestamp_unixtime" => object_key[:elb_timestamp_unixtime],
+        "s3_last_modified_unixtime" => object_key[:s3_last_modified_unixtime],
       }
 
       get_file_from_s3(object_key[:key])
       emit_lines_from_buffer_file(record_common)
 
-      put_timestamp_file(object_key[:elb_timestamp_unixtime])
+      put_timestamp_file(object_key[:s3_last_modified_unixtime])
     end
   end
 
   def get_object_keys(timestamp)
     begin
       object_keys = []
-      get_object_keys_from_s3.each do |object_key|
+      get_object_contents.each do |content|
+        # snip old items
+        s3_last_modified_unixtime = content.last_modified.to_i
+        next if s3_last_modified_unixtime <= timestamp
+
+        object_key = content.key
         matches = LOGFILE_REGEXP.match(object_key)
         next unless matches
 
-        # snip old items
-        elb_timestamp_unixtime = Time.parse(matches[:elb_timestamp]).to_i
-        next if elb_timestamp_unixtime <= timestamp
-
+        log.debug "matched"
         log.debug object_key
         object_keys << {
           key: object_key,
@@ -168,7 +171,8 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
           elb_timestamp: matches[:elb_timestamp],
           elb_ip_address: matches[:elb_ip_address],
           logfile_hash: matches[:logfile_hash],
-          elb_timestamp_unixtime: elb_timestamp_unixtime,
+          elb_timestamp_unixtime: Time.parse(matches[:elb_timestamp]).to_i,
+          s3_last_modified_unixtime: s3_last_modified_unixtime,
         }
       end
       return object_keys
@@ -177,22 +181,10 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     end
   end
 
-  def get_object_keys_from_s3
-    begin
-      object_keys = []
-      get_object_contents().each do |content|
-        object_keys << content.key
-      end
-      return object_keys
-    rescue => e
-      log.warn "error occurred: #{e.message}"
-    end
-  end
-
   def sort_object_key(src_object_keys)
     begin
       src_object_keys.sort do |a, b|
-        a[:elb_timestamp_unixtime] <=> b[:elb_timestamp_unixtime]
+        a[:s3_last_modified_unixtime] <=> b[:s3_last_modified_unixtime]
       end
     rescue => e
       log.warn "error occurred: #{e.message}"
@@ -220,7 +212,7 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
         contents << content
       end
 
-      if !resp.is_truncated 
+      if !resp.is_truncated
           return contents
       end
 
@@ -234,25 +226,35 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     return contents
   end
 
+  def inflate(srcfile, dstfile)
+    File.open(dstfile, File::WRONLY|File::CREAT|File::TRUNC) do |bfile|
+      File.open(srcfile) do |file|
+        zio = file
+        loop do
+          io = Zlib::GzipReader.new zio
+          bfile.write io.read
+          unused = io.unused
+          io.finish
+          break if unused.nil?
+          zio.pos -= unused.length
+        end
+      end
+    end
+  end
+
   def get_file_from_s3(object_name)
     begin
       log.debug "getting object from s3 name is #{object_name}"
 
-      Tempfile.create('fluent-elblog') do |tfile|
-        s3_client.get_object(bucket: @s3_bucketname, key: object_name) do |chunk|
-          tfile.write(chunk)
-        end
-        tfile.close
-
-        if File.extname(object_name) != '.gz'
-          FileUtils.cp(tfile.path, @buf_file)
-        else
-          File.open(@buf_file, File::WRONLY|File::CREAT|File::TRUNC) do |bfile|
-            Zlib::GzipReader.open(tfile.path) do |gz|
-              bfile.write gz.read
-            end
-          end
-        end
+      tfile = Tempfile.create('fluent-elblog')
+      tfile.close
+
+      s3_client.get_object(bucket: @s3_bucketname, key: object_name, response_target: tfile.path)
+
+      if File.extname(object_name) != '.gz'
+        FileUtils.cp(tfile.path, @buf_file)
+      else
+        inflate(tfile.path, @buf_file)
       end
     rescue => e
       log.warn "error occurred: #{e.message}, #{e.backtrace}"

data/test/plugin/in_elb_log.rb

@@ -9,7 +9,7 @@ class Elb_LogInputTest < Test::Unit::TestCase
   DEFAULT_CONFIG = {
     access_key_id: 'dummy_access_key_id',
     secret_access_key: 'dummy_secret_access_key',
-    s3_endpoint: 's3-ap-northeast-1.amazonaws.com',
+    s3_endpoint: 's3.ap-northeast-1.amazonaws.com',
     s3_bucketname: 'dummy_bucket',
     s3_prefix: 'test',
     region: 'ap-northeast-1',
@@ -51,11 +51,15 @@ class Elb_LogInputTest < Test::Unit::TestCase
   end
 
   def s3bucket_ok
+    stub_request(:get, 'https://s3.ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test')
+      .to_return(status: 200, body: "", headers: {})
     stub_request(:get, 'https://s3-ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test')
       .to_return(status: 200, body: "", headers: {})
   end
 
   def s3bucket_not_found
+    stub_request(:get, 'https://s3.ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test')
+      .to_return(status: 404, body: "", headers: {})
     stub_request(:get, 'https://s3-ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test')
       .to_return(status: 404, body: "", headers: {})
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-log
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - shinsaka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-16 00:00:00.000000000 Z
+date: 2017-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd