fluent-plugin-cloudfront-log 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9d0e94873fb17f7ba12461603e56f157a1b826a5
-  data.tar.gz: b53f30d6a03a6afc2d3f2e51635007bdc5457e24
+  metadata.gz: 4d3261d6109af600ee45feae079ad583fb586435
+  data.tar.gz: 2c237fc30d3ac111a3d06fb15271fc9d44268903
 SHA512:
-  metadata.gz: ca690918de97d284715582b8a04a0661c13d00859bfe0b84458a2537c90f59d53306bc401e964a8017bb65f6d9266e43164cc99eae606a0a646f0eb2e7f8ebd3
-  data.tar.gz: 6259a40d205cd1a359159e806e98d2e53d1b903b4f3ce5cd1bd158c3eea62b22239dc0c4bf09cbaa42b7f90c66fdcf9407f1202f11b4661c61105b2b1690c8d6
+  metadata.gz: 24cb6dbbb51e4f6a736eaf6b30adf12a3ffa48654828581c219bc5fabad49f7340c5029b578a389db471b5b079087d26e31e2f7aa982e76a5e4a2b9df69403f5
+  data.tar.gz: bfb0c00d90770537cc8b9510f9d85bc0474fb7fc6f481d8982989b4767080e7d6252585ac005c7c2212702621529858aba050032065779b61403a1b428c278e6

data/README.md

@@ -35,8 +35,7 @@ This specifices what the log files will be named once they're processed. This de
 The region where your cloudfront logs are stored.
 
 #### interval
-This is the rate in seconds at which we check the bucket for updated logs. It's recommended not to put this lower than 300(The default), cloudfront delivers logs every 20~ minutes to s3, so shortening this interval won't deliver your logs faster.
-
+This is the rate in seconds at which we check the bucket for updated logs. This defaults to 300.
 #### aws_sec_id
 The ID of your AWS keypair. Note: Since this plugin uses aws-sdk under the hood you can leave these two aws fields blank if you have an IAM role applied to your FluentD instance.
 
@@ -46,6 +45,9 @@ The secret key portion of your AWS keypair
 #### tag
 This is a FluentD builtin.
 
+#### thread_num
+The number of threads to create to concurrently process the S3 objects. Defaults to 4.
+
 #### delimiter
 You shouldn't have to specify delimiter at all but this option is provided and passed to the S3 client in the event that you have a weird delimiter in your log file names. Defaults to `nil`.
 

data/fluent-plugin-cloudfront-log.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-cloudfront-log"
-  spec.version       = "0.0.3"
+  spec.version       = "0.0.4"
   spec.authors       = ["kubihie"]
   spec.email         = ["kubihie@gmail.com"]
 

data/lib/fluent/plugin/in_cloudfront_log.rb

@@ -12,6 +12,7 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
   config_param :interval,          :integer, :default => 300
   config_param :delimiter,         :string,  :default => nil
   config_param :verbose,           :string,  :default => false
+  config_param :thread_num,        :integer, :default => 4
 
   def initialize
     super
@@ -37,6 +38,7 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
       log.info("@moved_log_bucket: #{@moved_log_bucket}")
       log.info("@log_prefix: #{@log_prefix}")
       log.info("@moved_log_prefix: #{@moved_log_prefix}")
+      log.info("@thread_num: #{@thread_num}")
     end
   end
 
@@ -46,7 +48,7 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
     client
 
     @loop = Coolio::Loop.new
-    timer = TimerWatcher.new(@interval, true, log, &method(:input)) 
+    timer = TimerWatcher.new(@interval, true, log, &method(:input))
 
     @loop.attach(timer)
     @thread = Thread.new(&method(:run))
@@ -80,7 +82,7 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
       @version = line.sub(/^#Version:/i, '').strip
     when /^#Fields:.+/i then
       @fields = line.sub(/^#Fields:/i, '').strip.split("\s")
-    end  
+    end
   end
 
   def purge(filename)
@@ -94,37 +96,76 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
     dest_object_full_path   = [@moved_log_bucket, dest_object_key].join('/')
 
     log.info("Copying object: #{source_object_full_path} to #{dest_object_full_path}") if @verbose
-    client.copy_object(:bucket => @moved_log_bucket, :copy_source => source_object_full_path, :key => dest_object_key)
+
+    begin
+      client.copy_object(:bucket => @moved_log_bucket, :copy_source => source_object_full_path, :key => dest_object_key)
+    rescue => e
+      log.warn("S3 Copy client error. #{e.message}")
+      return
+    end
+
 
     log.info("Deleting object: #{source_object_key} from #{@log_bucket}") if @verbose
-    client.delete_object(:bucket => @log_bucket, :key => source_object_key)
+    begin
+      client.delete_object(:bucket => @log_bucket, :key => source_object_key)
+    rescue => e
+      log.warn("S3 Delete client error. #{e.message}")
+      return
+    end
+  end
+
+
+  def process_content(content)
+    filename = content.key.sub(/^#{@log_prefix}\//, "")
+    log.info("CloudFront Currently processing: #{filename}") if @verbose
+    return if filename[-1] == '/'  #skip directory/
+    return unless filename[-2, 2] == 'gz'  #skip without gz file
+
+    begin
+      access_log_gz = client.get_object(:bucket => @log_bucket, :key => content.key).body
+      access_log = Zlib::GzipReader.new(access_log_gz).read
+    rescue => e
+      log.warn("S3 GET client error. #{e.message}")
+      return
+    end
+
+    access_log.split("\n").each do |line|
+      if line[0.1] == '#'
+        parse_header(line)
+        next
+      end
+      line = URI.unescape(line)  #hoge%2520fuga -> hoge%20fuga
+      line = URI.unescape(line)  #hoge%20fuga   -> hoge fuga
+      line = line.split("\t")
+      record = Hash[@fields.collect.zip(line)]
+      timestamp = Time.parse("#{record['date']}T#{record['time']}+00:00").to_i
+      router.emit(@tag, timestamp, record)
+    end
+    purge(filename)
   end
 
   def input
+    log.info("CloudFront Begining input going to list S3")
     client.list_objects(:bucket => @log_bucket, :prefix => @log_prefix , :delimiter => @delimiter).each do |list|
+    queue = Queue.new
+    threads = []
       list.contents.each do |content|
-        filename = content.key.sub(/^#{@log_prefix}\//, "")
-        log.info("Currently processing: #{filename}") if @verbose
-        next if filename[-1] == '/'  #skip directory/
-        next unless filename[-2, 2] == 'gz'  #skip without gz file
-
-        access_log_gz = client.get_object(:bucket => @log_bucket, :key => content.key).body
-        access_log = Zlib::GzipReader.new(access_log_gz).read
-
-        access_log.split("\n").each do |line|
-          if line[0.1] == '#'
-            parse_header(line)
-            next
+        queue << content
+      end
+        # BEGINS THREADS
+        @thread_num.times do
+          threads << Thread.new do
+            until queue.empty?
+              work_unit = queue.pop(true) rescue nil
+              if work_unit
+                 process_content(work_unit)
+              end
+            end
           end
-          line = URI.unescape(line)  #hoge%2520fuga -> hoge%20fuga
-          line = URI.unescape(line)  #hoge%20fuga   -> hoge fuga
-          line = line.split("\t")
-          record = Hash[@fields.collect.zip(line)]
-          timestamp = Time.parse("#{record['date']}T#{record['time']}+00:00").to_i
-          router.emit(@tag, timestamp, record)
         end
-        purge(filename)
-      end
+        log.debug("CloudFront Waiting for Threads to finish...")
+        threads.each { |t| t.join }
+        log.debug("CloudFront Finished")
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudfront-log
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - kubihie
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-16 00:00:00.000000000 Z
+date: 2016-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -114,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.7
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: AWS CloudFront log input plugin.