fluent-plugin-cloudfront-log 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4d3261d6109af600ee45feae079ad583fb586435
-  data.tar.gz: 2c237fc30d3ac111a3d06fb15271fc9d44268903
+  metadata.gz: 5a826fe802133f58af53f3c1d168af22c6a6014a
+  data.tar.gz: 6b58d43be522a4f7937c44b42a6e8cb614fc7744
 SHA512:
-  metadata.gz: 24cb6dbbb51e4f6a736eaf6b30adf12a3ffa48654828581c219bc5fabad49f7340c5029b578a389db471b5b079087d26e31e2f7aa982e76a5e4a2b9df69403f5
-  data.tar.gz: bfb0c00d90770537cc8b9510f9d85bc0474fb7fc6f481d8982989b4767080e7d6252585ac005c7c2212702621529858aba050032065779b61403a1b428c278e6
+  metadata.gz: b201b556e0d72ce801a1911b7a341c444767b0f98c05e0ccb0db2a3e850f0e75fd904db09aa2c8a87b32ca61a686dc63ea7d521d91a06a7e96d195d4ea45e0aa
+  data.tar.gz: 158fba1b70f4079bf545bbff5fd51bbda13dfc01606c7281e21dcaf8abbbecc394391868cc52d93dc7c4b92bba2701ef3e03780be0bda710af2feca2a5c6dd8f

data/README.md

@@ -48,6 +48,9 @@ This is a FluentD builtin.
 #### thread_num
 The number of threads to create to concurrently process the S3 objects. Defaults to 4.
 
+#### s3_get_max
+Control the size of the S3 fetched list on each iteration. Default to 200.
+
 #### delimiter
 You shouldn't have to specify delimiter at all but this option is provided and passed to the S3 client in the event that you have a weird delimiter in your log file names. Defaults to `nil`.
 

data/fluent-plugin-cloudfront-log.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-cloudfront-log"
-  spec.version       = "0.0.4"
+  spec.version       = "0.0.5"
   spec.authors       = ["kubihie"]
   spec.email         = ["kubihie@gmail.com"]
 

data/lib/fluent/plugin/in_cloudfront_log.rb

@@ -13,6 +13,7 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
   config_param :delimiter,         :string,  :default => nil
   config_param :verbose,           :string,  :default => false
   config_param :thread_num,        :integer, :default => 4
+  config_param :s3_get_max,        :integer, :default => 200
 
   def initialize
     super
@@ -146,27 +147,33 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
 
   def input
     log.info("CloudFront Begining input going to list S3")
-    client.list_objects(:bucket => @log_bucket, :prefix => @log_prefix , :delimiter => @delimiter).each do |list|
+    begin
+      s3_list = client.list_objects(:bucket => @log_bucket, :prefix => @log_prefix , :delimiter => @delimiter, :max_keys => @s3_get_max)
+    rescue => e
+      log.warn("S3 GET list error. #{e.message}")
+      return
+    end
+    log.info("Finished S3 get list")
     queue = Queue.new
     threads = []
-      list.contents.each do |content|
-        queue << content
-      end
-        # BEGINS THREADS
-        @thread_num.times do
-          threads << Thread.new do
-            until queue.empty?
-              work_unit = queue.pop(true) rescue nil
-              if work_unit
-                 process_content(work_unit)
-              end
-            end
+    log.debug("S3 List size: #{s3_list.contents.length}")
+    s3_list.contents.each do |content|
+      queue << content
+    end
+    # BEGINS THREADS
+    @thread_num.times do
+      threads << Thread.new do
+        until queue.empty?
+          work_unit = queue.pop(true) rescue nil
+          if work_unit
+            process_content(work_unit)
           end
         end
-        log.debug("CloudFront Waiting for Threads to finish...")
-        threads.each { |t| t.join }
-        log.debug("CloudFront Finished")
-    end
+       end
+     end
+     log.debug("CloudFront Waiting for Threads to finish...")
+     threads.each { |t| t.join }
+     log.debug("CloudFront Finished")
   end
 
   class TimerWatcher < Coolio::TimerWatcher

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudfront-log
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - kubihie
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-21 00:00:00.000000000 Z
+date: 2016-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd