fluent-plugin-elb-log 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f565fbbe387bad73d53cee18557d89ae0acd9f4f
-  data.tar.gz: 6a37e1c172a8d1c5b20394dbaa38923b9b82b0e4
+  metadata.gz: 71fea708525f12dea801f2e95871eae3e3361faa
+  data.tar.gz: b43655a000f3509012cfb5530839ca88c5201fcc
 SHA512:
-  metadata.gz: 58c8fd4c5ba84514c783afb2867438c4646a171f41d3d8fcb052db6624f807578b7bec83061f357b04791662b5a42d96cf973e0426aa5d091a5690b62625a5ba
-  data.tar.gz: 85870e3681ddc11eb40ad22feed503aacfddd970dfc108917af87f86fd06fadba084f1f09ff4bd0d3d38a2c88d436e463bfef1403fde1c690735ddb4b8accb04
+  metadata.gz: a6e9a47ae3fd4cc4bc25e2c6b702bdcf6524b998f8844d7afa22f86fbb22e3220e984f4e44092a333112f64f5b081fcd8f42f174ccdbe99e3bf0770864292f15
+  data.tar.gz: c2654c3fc22772b407fe9ca0d06022373544e0011782b5d29f32c662b4b30a344d0fe8b8b991b0a3c0a2e0285842ba87609f3f8e075b2e9e415ca2304c32042f

data/fluent-plugin-elb-log.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-elb-log"
-  spec.version       = "0.4.1"
+  spec.version       = "0.4.2"
   spec.authors       = ["shinsaka"]
   spec.email         = ["shinx1265@gmail.com"]
   spec.summary       = "Amazon ELB log input plugin"

data/lib/fluent/plugin/in_elb_log.rb

@@ -180,10 +180,8 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
   def get_object_keys_from_s3
     begin
       object_keys = []
-      get_object_list(100).each do |object|
-        object.contents.each do |content|
-          object_keys << content.key
-        end
+      get_object_contents().each do |content|
+        object_keys << content.key
       end
       return object_keys
     rescue => e
@@ -209,6 +207,33 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     )
   end
 
+  def get_object_contents()
+    contents = []
+
+    resp = s3_client.list_objects_v2(
+      bucket: @s3_bucketname,
+      prefix: @s3_prefix
+    )
+
+    loop do
+      resp.contents.each do |content|
+        contents << content
+      end
+
+      if !resp.is_truncated 
+          return contents
+      end
+
+      resp = s3_client.list_objects_v2(
+        bucket: @s3_bucketname,
+        prefix: @s3_prefix,
+        continuation_token: resp.next_continuation_token
+      )
+    end
+
+    return contents
+  end
+
   def get_file_from_s3(object_name)
     begin
       log.debug "getting object from s3 name is #{object_name}"

data/test/helper.rb

@@ -3,7 +3,9 @@ require 'test/unit'
 require 'webmock/test_unit'
 require 'simplecov'
 
-SimpleCov.start
+SimpleCov.start do
+  add_filter '/test/'
+end
 
 $LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
 $LOAD_PATH.unshift(__dir__)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-log
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - shinsaka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-06 00:00:00.000000000 Z
+date: 2017-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd