fluent-plugin-elb-access-log 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23af18097699523eb79eacdfff39c155e14a074e
-  data.tar.gz: 9132f72caaa0e09e2bbbad4fefef5bb6654642ee
+  metadata.gz: 96510ed8b34e1025ead147d989eefc4459404ae8
+  data.tar.gz: f44e0a4c33aa20267fa604fe181f1861c24b1ed5
 SHA512:
-  metadata.gz: f3273e599277bb2ee5367e8934a4c8cb32d6337145c5eee60114558d2d111f4fde60149b3ef2ed811144ab35cdf5fb6e20a2cdbe2f8a8e3b6b8ccd5d4ad49c6d
-  data.tar.gz: 4c44fdc553b5afddb444cdead9a2c6bf00304059a4eea0f9abe2d6550588679536122afe2ffe36e5e52c9e0640841a3d287815c43d2096d01fec33d9f82fc842
+  metadata.gz: a0375b94759d747d2f3bbdf0cec1b048cb1ec81149f1db9f111edc66726563b4cb75316ecb45a3c87b248ae8215076f0b32b00457779aacb3c04e6b635d2d495
+  data.tar.gz: 573db4ec038fb669d091e4125a7f09d0c485d619f28db67aea3e120f2439d9d9386d1f065790ebd085bd0f1e90c286fde90b224c95f56182474abc5ed8453a4f

data/README.md

@@ -43,6 +43,7 @@ Or install it yourself as:
   #start_datetime 2015/05/24 17:00
   #buffer_sec 600
   #history_length 100
+  #sampling_interval 1
   #debug false
 </source>
 ```

data/lib/fluent/plugin/in_elb_access_log.rb

@@ -29,22 +29,23 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     define_method('router') { Fluent::Engine }
   end
 
-  config_param :aws_key_id,       :string,  :default => nil
-  config_param :aws_sec_key,      :string,  :default => nil
-  config_param :profile,          :string,  :default => nil
-  config_param :credentials_path, :string,  :default => nil
-  config_param :http_proxy,       :string,  :default => nil
-  config_param :account_id,       :string
-  config_param :region,           :string
-  config_param :s3_bucket,        :string
-  config_param :s3_prefix,        :string,  :default => nil
-  config_param :tag,              :string,  :default => 'elb.access_log'
-  config_param :tsfile_path,      :string,  :default => '/var/tmp/fluent-plugin-elb-access-log.ts'
-  config_param :interval,         :time,    :default => 300
-  config_param :start_datetime,   :string,  :default => nil
-  config_param :buffer_sec,       :time,    :default => 600
-  config_param :history_length,   :integer, :default => 100
-  config_param :debug,            :bool,    :default => false
+  config_param :aws_key_id,        :string,  :default => nil
+  config_param :aws_sec_key,       :string,  :default => nil
+  config_param :profile,           :string,  :default => nil
+  config_param :credentials_path,  :string,  :default => nil
+  config_param :http_proxy,        :string,  :default => nil
+  config_param :account_id,        :string
+  config_param :region,            :string
+  config_param :s3_bucket,         :string
+  config_param :s3_prefix,         :string,  :default => nil
+  config_param :tag,               :string,  :default => 'elb.access_log'
+  config_param :tsfile_path,       :string,  :default => '/var/tmp/fluent-plugin-elb-access-log.ts'
+  config_param :interval,          :time,    :default => 300
+  config_param :start_datetime,    :string,  :default => nil
+  config_param :buffer_sec,        :time,    :default => 600
+  config_param :history_length,    :integer, :default => 100
+  config_param :sampling_interval, :integer, :default => 1
+  config_param :debug,             :bool,    :default => false
 
   def initialize
     super
@@ -146,6 +147,10 @@ class Fluent::ElbAccessLogInput < Fluent::Input
   end
 
   def emit_access_log(access_log)
+    if @sampling_interval > 1
+      access_log = sampling(access_log)
+    end
+
     access_log = CSV.parse(access_log, :col_sep => ' ')
 
     access_log.each do |row|
@@ -165,6 +170,10 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     end
   end
 
+  def sampling(access_log)
+    access_log.split("\n").each_with_index.select {|row, i| (i % @sampling_interval).zero? }.map {|row, i| row }.join("\n")
+  end
+
   def split_address_port!(record, prefix)
     address, port = record["#{prefix}_port"].split(':', 2)
     record[prefix] = address

data/lib/fluent_plugin_elb_access_log/version.rb

@@ -1,3 +1,3 @@
 module FluentPluginElbAccessLog
-  VERSION = '0.1.7'
+  VERSION = '0.1.8'
 end

data/spec/in_elb_access_log_config_spec.rb

@@ -38,6 +38,7 @@ describe 'Fluent::ElbAccessLogInput#configure' do
       expect(driver.instance.start_datetime).to eq today
       expect(driver.instance.buffer_sec).to eq 600
       expect(driver.instance.history_length).to eq 100
+      expect(driver.instance.sampling_interval).to eq 1
       expect(driver.instance.debug).to be_falsey
     end
   end
@@ -55,6 +56,7 @@ describe 'Fluent::ElbAccessLogInput#configure' do
     let(:start_datetime) { today - 3600 }
     let(:buffer_sec) { 1200 }
     let(:history_length) { 200 }
+    let(:sampling_interval) { 100 }
 
     let(:fluentd_conf) do
       {
@@ -73,6 +75,7 @@ describe 'Fluent::ElbAccessLogInput#configure' do
         start_datetime: start_datetime,
         buffer_sec: buffer_sec,
         history_length: history_length,
+        sampling_interval: sampling_interval,
         debug: true,
       }
     end
@@ -92,6 +95,7 @@ describe 'Fluent::ElbAccessLogInput#configure' do
       expect(driver.instance.start_datetime).to eq start_datetime
       expect(driver.instance.buffer_sec).to eq buffer_sec
       expect(driver.instance.history_length).to eq history_length
+      expect(driver.instance.sampling_interval).to eq sampling_interval
       expect(driver.instance.debug).to be_truthy
     end
   end

data/spec/in_elb_access_log_spec.rb

@@ -217,6 +217,24 @@ describe Fluent::ElbAccessLogInput do
 
     it { is_expected.to eq expected_emits }
 
+    context 'when sampling' do
+      let(:fluentd_conf) do
+        {
+          account_id: account_id,
+          s3_bucket: s3_bucket,
+          region: region,
+          start_datetime: (today - 1).to_s,
+          sampling_interval: 2,
+        }
+      end
+
+      it do
+        expected_emits.delete_at(3)
+        expected_emits.delete_at(1)
+        is_expected.to eq expected_emits
+      end
+    end
+
     context 'when include bad URI' do
       let(:today_access_log) do
         <<-EOS

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-access-log
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - Genki Sugawara