fluent-plugin-elb-access-log 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aa61c99c73fe15d923591e28868b58ac63537fbe
-  data.tar.gz: 8fd24f9385d67df78c65b1396f7007e835544d50
+  metadata.gz: 797ad8f847603d045b5eca038c0f5afed8730332
+  data.tar.gz: cf272aa08b78a320f4b5845d2be210ad7df91272
 SHA512:
-  metadata.gz: bfdb323b0785cc474b96f6019f6932a80658d3ba4c33713fea5d294e7582f607018ee04cc1640d9eb8eac306cc907bdf612b7498f030a2725eaa6d7f1ebd8ee3
-  data.tar.gz: 5d755876a4d9c871751e66fa0a51d9ec2d2ed97000ed658cbb4fbd16ffb38ad7ac012aee0cda94692116e95cdf5ec9e37981b4cb73d309b824f432ae2eecdfd6
+  metadata.gz: 059252da8f5468d7e579f9edf8beab4cd14b20f55a99a8bce0e9aede7cbf30e5253ba66ac67b2872b145b678bc9762bb0cfdc42641ff1c4c7ee3ebb8545f7ca3
+  data.tar.gz: 73fa7e9b2e49b46bc6a45db1a932521fd137a42d6fe4c0c5e3fc88634ee265fcae99b247cddce55ec1a21bcf410e69ef35e9d8e1f1a2cbe5a87d70099efd370a

data/README.md

@@ -41,6 +41,8 @@ Or install it yourself as:
   #tsfile_path /var/tmp/fluent-plugin-elb-access-log.ts
   #interval 300
   #start_datetime 2015/05/24 17:00
+  #buffer_sec 600
+  #history_length 100
   #debug false
 </source>
 ```

data/lib/fluent/plugin/in_elb_access_log.rb

@@ -42,6 +42,8 @@ class Fluent::ElbAccessLogInput < Fluent::Input
   config_param :tsfile_path,      :string,  :default => '/var/tmp/fluent-plugin-elb-access-log.ts'
   config_param :interval,         :time,    :default => 300
   config_param :start_datetime,   :string,  :default => nil
+  config_param :buffer_sec,       :time,    :default => 600
+  config_param :history_length,   :integer, :default => 100
   config_param :debug,            :bool,    :default => false
 
   def initialize
@@ -64,6 +66,8 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     else
       @start_datetime = Time.parse(File.read(@tsfile_path)).utc rescue Time.now.utc
     end
+
+    @history = []
   end
 
   def start
@@ -78,10 +82,14 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     timer = TimerWatcher.new(@interval, true, log) do
       new_timestamp = fetch(timestamp)
 
-      if timestamp != new_timestamp
+      if new_timestamp > timestamp
         save_timestamp(new_timestamp)
         timestamp = new_timestamp
       end
+
+      if @history.length > @history_length
+        @history.shift(history.length - @history_length)
+      end
     end
 
     @loop.attach(timer)
@@ -111,13 +119,16 @@ class Fluent::ElbAccessLogInput < Fluent::Input
           account_id, logfile_const, region, elb_name, logfile_datetime, ip, logfile_suffix = obj.key.split('_', 7)
           logfile_datetime = Time.parse(logfile_datetime)
 
-          if logfile_suffix !~ /\.log\z/ or logfile_datetime <= timestamp
+          if logfile_suffix !~ /\.log\z/ or logfile_datetime <= (timestamp - @buffer_sec)
             next
           end
 
-          access_log = client.get_object(bucket: @s3_bucket, key: obj.key).first.body.string
-          emit_access_log(access_log)
-          last_timestamp = logfile_datetime
+          unless @history.include?(obj.key)
+            access_log = client.get_object(bucket: @s3_bucket, key: obj.key).first.body.string
+            emit_access_log(access_log)
+            last_timestamp = logfile_datetime
+            @history.push(obj.key)
+          end
         end
       end
     end

data/lib/fluent_plugin_elb_access_log/version.rb

@@ -1,3 +1,3 @@
 module FluentPluginElbAccessLog
-  VERSION = '0.1.4'
+  VERSION = '0.1.5'
 end

data/spec/in_elb_access_log_config_spec.rb

@@ -36,6 +36,8 @@ describe 'Fluent::ElbAccessLogInput#configure' do
       expect(driver.instance.tsfile_path).to eq '/var/tmp/fluent-plugin-elb-access-log.ts'
       expect(driver.instance.interval).to eq 300
       expect(driver.instance.start_datetime).to eq today
+      expect(driver.instance.buffer_sec).to eq 600
+      expect(driver.instance.history_length).to eq 100
       expect(driver.instance.debug).to be_falsey
     end
   end
@@ -51,6 +53,8 @@ describe 'Fluent::ElbAccessLogInput#configure' do
     let(:tsfile_path) { '/tmp/foo' }
     let(:interval) { 500 }
     let(:start_datetime) { today - 3600 }
+    let(:buffer_sec) { 1200 }
+    let(:history_length) { 200 }
 
     let(:fluentd_conf) do
       {
@@ -67,6 +71,8 @@ describe 'Fluent::ElbAccessLogInput#configure' do
         tsfile_path: tsfile_path,
         interval: interval,
         start_datetime: start_datetime,
+        buffer_sec: buffer_sec,
+        history_length: history_length,
         debug: true,
       }
     end
@@ -84,6 +90,8 @@ describe 'Fluent::ElbAccessLogInput#configure' do
       expect(driver.instance.tsfile_path).to eq tsfile_path
       expect(driver.instance.interval).to eq interval
       expect(driver.instance.start_datetime).to eq start_datetime
+      expect(driver.instance.buffer_sec).to eq buffer_sec
+      expect(driver.instance.history_length).to eq history_length
       expect(driver.instance.debug).to be_truthy
     end
   end

data/spec/in_elb_access_log_spec.rb

@@ -225,15 +225,15 @@ describe Fluent::ElbAccessLogInput do
       EOS
     end
 
-  let(:fluentd_conf) do
-    {
-      account_id: account_id,
-      s3_bucket: s3_bucket,
-      region: region,
-      start_datetime: (today - 1).to_s,
-      tag: 'any.tag'
-    }
-  end
+    let(:fluentd_conf) do
+      {
+        account_id: account_id,
+        s3_bucket: s3_bucket,
+        region: region,
+        start_datetime: (today - 1).to_s,
+        tag: 'any.tag'
+      }
+    end
 
     before do
       expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
@@ -287,4 +287,119 @@ describe Fluent::ElbAccessLogInput do
 
     it { is_expected.to eq expected_emits }
   end
+
+  context 'when access old log exists' do
+    let(:today_access_log) do
+      <<-EOS
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+      EOS
+    end
+
+    let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 600).iso8601}_52.68.51.1_8hSqR3o4.log" }
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        [double('today_s3_object', body: StringIO.new(today_access_log))]
+      end
+
+      expect(driver.instance).to_not receive(:save_timestamp)
+
+      driver.run
+    end
+
+    let(:expected_emits) do
+      [["elb.access_log",
+        Time.parse('2015-05-24 19:55:36 UTC').to_i,
+        {"timestamp"=>"2015-05-24T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client"=>"14.14.124.20",
+         "client_port"=>57673,
+         "backend"=>"10.0.199.184",
+         "backend_port"=>80,
+         "request_processing_time"=>5.3e-05,
+         "backend_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "backend_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1",
+         "request.uri.scheme"=>"http",
+         "request.uri.userinfo"=>nil,
+         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+         "request.uri.port"=>80,
+         "request.uri.registry"=>nil,
+         "request.uri.path"=>"/",
+         "request.uri.opaque"=>nil,
+         "request.uri.query"=>nil,
+         "request.uri.fragment"=>nil}]]
+    end
+
+    it { is_expected.to eq expected_emits }
+  end
+
+  context 'when access old log exists (timeout)' do
+    let(:today_access_log) do
+      <<-EOS
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+      EOS
+    end
+
+    let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 601).iso8601}_52.68.51.1_8hSqR3o4.log" }
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to_not receive(:get_object)
+      expect(driver.instance).to_not receive(:save_timestamp)
+
+      driver.run
+    end
+
+    it { is_expected.to be_empty }
+  end
+
+  context 'when emitted log exists' do
+    let(:today_access_log) do
+      <<-EOS
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+      EOS
+    end
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to_not receive(:get_object)
+
+      history = driver.instance.instance_variable_get(:@history)
+      history << today_object_key
+      expect(driver.instance).to_not receive(:save_timestamp)
+
+      driver.run
+    end
+
+    it { is_expected.to be_empty }
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-access-log
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Genki Sugawara
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-25 00:00:00.000000000 Z
+date: 2015-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd