fluent-plugin-elb-log 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '098789aaeb055a519eb134eacc836b136871a0ff'
-  data.tar.gz: 19c18c2c0b1e40b9910d96b82f775e411456224b
+  metadata.gz: 45e03965c1d8f3c0c81cac0793a2fd9dd08d5c06
+  data.tar.gz: c23f025998cb293ae4bc9472d6f4e97456532192
 SHA512:
-  metadata.gz: bc08af197a894d4029acad72fc8130ec420978b2dee02d394b81de5392368c043d3c7caf7b24339bf305aa302a586b538158cbc82eddf99e29961246e6e8c2d9
-  data.tar.gz: b449eaa8f96fa886db6de387d365de1d6896e88d60ccf721c67b93f096a7da6a8e60c06ed43e0e4cc1955c30d457196d4a3bd6fafc8f62084be5e7164956ff81
+  metadata.gz: 41381c1563f713ef5561a984c6790f9870f6bfa97cffd154a37f28508cc5114e554200ccaa9ff495a1e321c5c38421de165c3bd1904d1e855a1af56c3d00b489
+  data.tar.gz: d9d6f310ee099df38efdd65cf76e04b167bcd03877fc994460491bd65d60cc599542b33762e8da62fcf570e0650195b077c685ad2c1e58e04aee375858dff05e

data/fluent-plugin-elb-log.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-elb-log"
-  spec.version       = "0.7.0"
+  spec.version       = "0.8.0"
   spec.authors       = ["shinsaka"]
   spec.email         = ["shinx1265@gmail.com"]
   spec.summary       = "Amazon ELB log input plugin"

data/lib/fluent/plugin/in_elb_log.rb

@@ -4,6 +4,7 @@ require 'fileutils'
 require 'aws-sdk-s3'
 require 'aws-sdk-ec2'
 require 'fluent/input'
+require 'digest/sha1'
 
 class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
   Fluent::Plugin.register_input('elb_log', self)
@@ -25,6 +26,8 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
   config_param :http_proxy, :string, default: nil
   config_param :start_time, :string, default: nil
   config_param :delete, :bool, default: false
+  config_param :num_nodes, :integer, default: 1
+  config_param :node_no, :integer, default: 0
 
   def configure(conf)
     super
@@ -66,7 +69,12 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
       timestamp = start_time.to_i
       log.debug "timestamp file #{@timestamp_file} read"
       File.open(@timestamp_file, File::RDONLY) do |file|
-        timestamp = file.read.to_i if file.size > 0
+        if file.size > 0
+          timestamp_from_file = file.read.to_i 
+          if timestamp_from_file > timestamp 
+            timestamp = timestamp_from_file
+          end
+        end
       end
       log.debug "timestamp start at:" + Time.at(timestamp).to_s
       return timestamp
@@ -118,70 +126,39 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
   end
 
   def input
-    log.debug "start"
-
-    timestamp = get_timestamp_file()
-
-    object_keys = get_object_keys(timestamp)
-    object_keys = sort_object_key(object_keys)
-
-    log.info "processing #{object_keys.count} object(s)."
-
-    object_keys.each do |object_key|
-      record_common = {
-        "account_id" => object_key[:account_id],
-        "region" => object_key[:region],
-        "logfile_date" => object_key[:logfile_date],
-        "logfile_elb_name" => object_key[:logfile_elb_name],
-        "elb_ip_address" => object_key[:elb_ip_address],
-        "logfile_hash" => object_key[:logfile_hash],
-        "elb_timestamp" => object_key[:elb_timestamp],
-        "key" => object_key[:key],
-        "prefix" => object_key[:prefix],
-        "elb_timestamp_unixtime" => object_key[:elb_timestamp_unixtime],
-        "s3_last_modified_unixtime" => object_key[:s3_last_modified_unixtime],
-      }
-
-      get_file_from_s3(object_key[:key])
-      emit_lines_from_buffer_file(record_common)
-
-      put_timestamp_file(object_key[:s3_last_modified_unixtime])
+    begin
+      log.debug "start"
+      timestamp = get_timestamp_file()
+
+      object_keys = get_object_keys(timestamp)
+      object_keys = sort_object_key(object_keys)
+
+      log.info "processing #{object_keys.count} object(s)."
+
+      object_keys.each do |object_key|
+        record_common = {
+          "account_id" => object_key[:account_id],
+          "region" => object_key[:region],
+          "logfile_date" => object_key[:logfile_date],
+          "logfile_elb_name" => object_key[:logfile_elb_name],
+          "elb_ip_address" => object_key[:elb_ip_address],
+          "logfile_hash" => object_key[:logfile_hash],
+          "elb_timestamp" => object_key[:elb_timestamp],
+          "key" => object_key[:key],
+          "prefix" => object_key[:prefix],
+          "elb_timestamp_unixtime" => object_key[:elb_timestamp_unixtime],
+          "s3_last_modified_unixtime" => object_key[:s3_last_modified_unixtime],
+        }
 
-      if @delete
-        delete_file_from_s3(object_key[:key])
-      end
-    end
-  end
+        get_file_from_s3(object_key[:key])
+        emit_lines_from_buffer_file(record_common)
 
-  def get_object_keys(timestamp)
-    begin
-      object_keys = []
-      get_object_contents.each do |content|
-        # snip old items
-        s3_last_modified_unixtime = content.last_modified.to_i
-        next if s3_last_modified_unixtime <= timestamp
+        put_timestamp_file(object_key[:s3_last_modified_unixtime])
 
-        object_key = content.key
-        matches = LOGFILE_REGEXP.match(object_key)
-        next unless matches
-
-        log.debug "matched"
-        log.debug object_key
-        object_keys << {
-          key: object_key,
-          prefix: matches[:prefix],
-          account_id: matches[:account_id],
-          region: matches[:region],
-          logfile_date: matches[:logfile_date],
-          logfile_elb_name: matches[:logfile_elb_name],
-          elb_timestamp: matches[:elb_timestamp],
-          elb_ip_address: matches[:elb_ip_address],
-          logfile_hash: matches[:logfile_hash],
-          elb_timestamp_unixtime: Time.parse(matches[:elb_timestamp]).to_i,
-          s3_last_modified_unixtime: s3_last_modified_unixtime,
-        }
+        if @delete
+          delete_file_from_s3(object_key[:key])
+        end
       end
-      return object_keys
     rescue => e
       log.warn "error occurred: #{e.message}"
     end
@@ -205,8 +182,8 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     )
   end
 
-  def get_object_contents()
-    contents = []
+  def get_object_keys(timestamp)
+    object_keys = []
 
     resp = s3_client.list_objects_v2(
       bucket: @s3_bucketname,
@@ -215,11 +192,32 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
 
     loop do
       resp.contents.each do |content|
-        contents << content
+        s3_last_modified_unixtime = content.last_modified.to_i
+
+        object_key = content.key
+        node_no = Digest::SHA1.hexdigest(object_key).to_i(16) % @num_nodes
+        next unless node_no == @node_no 
+
+        matches = LOGFILE_REGEXP.match(object_key)
+        if s3_last_modified_unixtime > timestamp and matches
+          object_keys << {
+            key: object_key,
+            prefix: matches[:prefix],
+            account_id: matches[:account_id],
+            region: matches[:region],
+            logfile_date: matches[:logfile_date],
+            logfile_elb_name: matches[:logfile_elb_name],
+            elb_timestamp: matches[:elb_timestamp],
+            elb_ip_address: matches[:elb_ip_address],
+            logfile_hash: matches[:logfile_hash],
+            elb_timestamp_unixtime: Time.parse(matches[:elb_timestamp]).to_i,
+            s3_last_modified_unixtime: s3_last_modified_unixtime,
+          }
+        end
       end
 
       if !resp.is_truncated
-          return contents
+        return object_keys
       end
 
       resp = s3_client.list_objects_v2(
@@ -229,7 +227,7 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
       )
     end
 
-    return contents
+    return object_keys
   end
 
   def inflate(srcfile, dstfile)
@@ -322,4 +320,4 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
       "option3" => item[:option3]
     }
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-log
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - shinsaka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-22 00:00:00.000000000 Z
+date: 2018-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd