fluent-plugin-elb-access-log 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c7fc100990f59fc2792cbf0506b75ef400a1dd8f
-  data.tar.gz: 9e6bda33e638700957ee92c685c699859eb4dfab
+  metadata.gz: 88a3b513a550db32f667fedc8ba75eea2e2f6199
+  data.tar.gz: 0e5dd5364065a98f38f15c54fe400e0826941ae4
 SHA512:
-  metadata.gz: f280a3e9212f18ce0446daacd6fffb6d9d3e59413458e3f6052acd240523a5c2db8d42ea2d73ed53f90fbc51abe554edc13c73eca2414c65bd8661839c6579a5
-  data.tar.gz: 26eb2f804a5245738cb6e7881ba3d454457ec3f575d8acc9d1b200da402f38a82eba0d263f0e42f4706dfeac18b3fcc62e50d40425c67b342c4cde93b5fa66e3
+  metadata.gz: 7c3c094dff47e82717c9cf8b80d2528f35257a61dc7bcb6c64c6943433e684082638c1e13256b43ea448b1e2e40487fbfcaa73353f8fb74245b739fc8e897edb
+  data.tar.gz: 8493ec02a57fb20f8fbaf144fa8930292b9658a90438b31a56d429b04a7690e72b4273d7a37de68422f721089181e02ecc9810a43bb271b7fea1bc5f61e8e806

data/README.md

@@ -48,6 +48,10 @@ Or install it yourself as:
   #sampling_interval 1
   #debug false
   #elb_type clb # or alb
+  #filter elb_status_code:^2,timestamp:^2018
+  #filter_operator and # or "or"
+  #type_cast true
+  #parse_request true
 </source>
 ```
 

data/lib/fluent/plugin/in_elb_access_log.rb

@@ -19,8 +19,8 @@ class Fluent::Plugin::ElbAccessLogInput < Fluent::Input
     'clb' => {
       'timestamp'                => nil,
       'elb'                      => nil,
-      'client_port'              => nil,
-      'backend_port'             => nil,
+      'client_port'              => :to_i,
+      'backend_port'             => :to_i,
       'request_processing_time'  => :to_f,
       'backend_processing_time'  => :to_f,
       'response_processing_time' => :to_f,
@@ -38,8 +38,8 @@ class Fluent::Plugin::ElbAccessLogInput < Fluent::Input
       'type'                     => nil,
       'timestamp'                => nil,
       'elb'                      => nil,
-      'client_port'              => nil,
-      'target_port'              => nil,
+      'client_port'              => :to_i,
+      'target_port'              => :to_i,
       'request_processing_time'  => :to_f,
       'target_processing_time'   => :to_f,
       'response_processing_time' => :to_f,
@@ -58,7 +58,7 @@ class Fluent::Plugin::ElbAccessLogInput < Fluent::Input
     },
   }
 
-  ELB_TYPES = %(clb alb)
+  ELB_TYPES = ACCESS_LOG_FIELDS.keys
 
   config_param :elb_type,          :string,  default: 'clb'
   config_param :aws_key_id,        :string,  default: nil, secret: true
@@ -79,12 +79,20 @@ class Fluent::Plugin::ElbAccessLogInput < Fluent::Input
   config_param :history_length,    :integer, default: 100
   config_param :sampling_interval, :integer, default: 1
   config_param :debug,             :bool,    default: false
+  config_param :filter,            :hash,    default: nil
+  config_param :filter_operator,   :string,  default: 'and'
+  config_param :type_cast,         :bool,    default: true
+  config_param :parse_request,     :bool,    default: true
 
   def configure(conf)
     super
 
     unless ELB_TYPES.include?(@elb_type)
-      raise raise Fluent::ConfigError, "Invalid ELB type: #{@elb_type}"
+      raise Fluent::ConfigError, "Invalid ELB type: #{@elb_type}"
+    end
+
+    unless %w(and or).include?(@filter_operator)
+      raise Fluent::ConfigError, "Invalid filter operator: #{@filter_operator}"
     end
 
     FileUtils.touch(@tsfile_path)
@@ -102,6 +110,10 @@ class Fluent::Plugin::ElbAccessLogInput < Fluent::Input
     end
 
     @history = load_history
+
+    if @filter
+      @filter = Hash[@filter.map {|k, v| [k.to_s, Regexp.new(v.to_s)] }]
+    end
   end
 
   def start
@@ -239,10 +251,6 @@ class Fluent::Plugin::ElbAccessLogInput < Fluent::Input
     parsed_access_log.each do |row|
       record = Hash[access_log_fields.keys.zip(row)]
 
-      access_log_fields.each do |name, conv|
-        record[name] = record[name].send(conv) if conv
-      end
-
       split_address_port!(record, 'client')
 
       case @elb_type
@@ -252,7 +260,23 @@ class Fluent::Plugin::ElbAccessLogInput < Fluent::Input
         split_address_port!(record, 'target')
       end
 
-      parse_request!(record)
+      if @filter
+        if @filter_operator == 'or'
+          next if @filter.all? {|k, r| record[k] !~ r }
+        else
+          next if @filter.any? {|k, r| record[k] !~ r }
+        end
+      end
+
+      if @type_cast
+        access_log_fields.each do |name, conv|
+          record[name] = record[name].send(conv) if conv
+        end
+      end
+
+      if @parse_request
+        parse_request!(record)
+      end
 
       records << record
     end
@@ -327,7 +351,7 @@ class Fluent::Plugin::ElbAccessLogInput < Fluent::Input
     return unless address_port
     address, port = address_port.split(':', 2)
     record[prefix] = address
-    record["#{prefix}_port"] = port.to_i
+    record["#{prefix}_port"] = port
   end
 
   def parse_request!(record)
@@ -344,7 +368,13 @@ class Fluent::Plugin::ElbAccessLogInput < Fluent::Input
 
       if uri
         [:scheme ,:user, :host, :port, :path, :query, :fragment].each do |key|
-          record["request.uri.#{key}"] = uri.send(key)
+          value = uri.send(key)
+
+          if not @type_cast and key == :port
+            value = value.to_s
+          end
+
+          record["request.uri.#{key}"] = value
         end
       end
     rescue => e

data/lib/fluent_plugin_elb_access_log/version.rb

@@ -1,3 +1,3 @@
 module FluentPluginElbAccessLog
-  VERSION = '0.4.2'
+  VERSION = '0.4.3'
 end

data/spec/in_elb_access_log_alb_spec.rb

@@ -275,6 +275,123 @@ https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
         is_expected.to match_table expected_emits
       end
     end
+
+    context 'with filter' do
+      let(:fluentd_conf) do
+        {
+          interval: 0,
+          account_id: account_id,
+          s3_bucket: s3_bucket,
+          region: region,
+          start_datetime: (today - 1).to_s,
+          elb_type: 'alb',
+          filter: '{"timestamp": "2015-05-25"}',
+        }
+      end
+
+      it do
+        expected_emits.slice!(0, 2)
+        is_expected.to match_table expected_emits
+      end
+    end
+
+    context 'with filter (or)' do
+      let(:fluentd_conf) do
+        {
+          interval: 0,
+          account_id: account_id,
+          s3_bucket: s3_bucket,
+          region: region,
+          start_datetime: (today - 1).to_s,
+          elb_type: 'alb',
+          filter: '{"timestamp": "2015-05-25"}',
+          filter_operator: 'or'
+        }
+      end
+
+      it do
+        expected_emits.slice!(0, 2)
+        is_expected.to match_table expected_emits
+      end
+    end
+
+    context 'with filter (or/multi)' do
+      let(:fluentd_conf) do
+        {
+          interval: 0,
+          account_id: account_id,
+          s3_bucket: s3_bucket,
+          region: region,
+          start_datetime: (today - 1).to_s,
+          elb_type: 'alb',
+          filter: '{"timestamp": "2015-05-25", "elb_status_code": "^2"}',
+          filter_operator: 'or'
+        }
+      end
+
+      it do
+        is_expected.to match_table expected_emits
+      end
+    end
+
+    context 'without type cast' do
+      let(:fluentd_conf) do
+        {
+          interval: 0,
+          account_id: account_id,
+          s3_bucket: s3_bucket,
+          region: region,
+          start_datetime: (today - 1).to_s,
+          elb_type: 'alb',
+          type_cast: 'false',
+        }
+      end
+
+      it do
+        expected_emits_without_type_cast = expected_emits.map do |tag, ts, h|
+          h = Hash[h.map {|k, v|
+            v = case v
+                when nil
+                  v
+                when Float
+                  "%.6f" % v
+                else
+                  v.to_s
+                end
+
+            [k, v]
+          }]
+
+          [tag, ts, h]
+        end
+
+        is_expected.to match_table expected_emits_without_type_cast
+      end
+    end
+
+
+    context 'without request parsing' do
+      let(:fluentd_conf) do
+        {
+          interval: 0,
+          account_id: account_id,
+          s3_bucket: s3_bucket,
+          region: region,
+          start_datetime: (today - 1).to_s,
+          elb_type: 'alb',
+          parse_request: 'false',
+        }
+      end
+
+      it do
+        expected_emits_without_request_parsing = expected_emits.map do |tag, ts, h|
+          h = Hash[h.select {|k, v| k !~ /\Arequest\./ }]
+          [tag, ts, h]
+        end
+
+        is_expected.to match_table expected_emits_without_request_parsing
+      end
+    end
   end
 
   context 'when include bad URI' do

data/spec/in_elb_access_log_clb_spec.rb

@@ -247,6 +247,117 @@ describe Fluent::Plugin::ElbAccessLogInput do
         is_expected.to match_table expected_emits
       end
     end
+
+    context 'with filter' do
+      let(:fluentd_conf) do
+        {
+          interval: 0,
+          account_id: account_id,
+          s3_bucket: s3_bucket,
+          region: region,
+          start_datetime: (today - 1).to_s,
+          filter: '{"timestamp": "2015-05-25"}',
+        }
+      end
+
+      it do
+        expected_emits.slice!(0, 2)
+        is_expected.to match_table expected_emits
+      end
+    end
+
+    context 'with filter (or)' do
+      let(:fluentd_conf) do
+        {
+          interval: 0,
+          account_id: account_id,
+          s3_bucket: s3_bucket,
+          region: region,
+          start_datetime: (today - 1).to_s,
+          filter: '{"timestamp": "2015-05-25"}',
+          filter_operator: 'or',
+        }
+      end
+
+      it do
+        expected_emits.slice!(0, 2)
+        is_expected.to match_table expected_emits
+      end
+    end
+
+    context 'with filter (or/multi)' do
+      let(:fluentd_conf) do
+        {
+          interval: 0,
+          account_id: account_id,
+          s3_bucket: s3_bucket,
+          region: region,
+          start_datetime: (today - 1).to_s,
+          filter: '{"timestamp": "2015-05-25", "elb_status_code": "^2"}',
+          filter_operator: 'or',
+        }
+      end
+
+      it do
+        is_expected.to match_table expected_emits
+      end
+    end
+
+    context 'without type cast' do
+      let(:fluentd_conf) do
+        {
+          interval: 0,
+          account_id: account_id,
+          s3_bucket: s3_bucket,
+          region: region,
+          start_datetime: (today - 1).to_s,
+          type_cast: 'false',
+        }
+      end
+
+      it do
+        expected_emits_without_type_cast = expected_emits.map do |tag, ts, h|
+          h = Hash[h.map {|k, v|
+            v = case v
+                when nil
+                  v
+                when Float
+                  "%.6f" % v
+                else
+                  v.to_s
+                end
+
+            [k, v]
+          }]
+
+          [tag, ts, h]
+        end
+
+        is_expected.to match_table expected_emits_without_type_cast
+      end
+    end
+
+    context 'without request parsing' do
+      let(:fluentd_conf) do
+        {
+          interval: 0,
+          account_id: account_id,
+          s3_bucket: s3_bucket,
+          region: region,
+          start_datetime: (today - 1).to_s,
+          parse_request: 'false',
+        }
+      end
+
+      it do
+        expected_emits_without_request_parsing = expected_emits.map do |tag, ts, h|
+          h = Hash[h.select {|k, v| k !~ /\Arequest\./ }]
+          [tag, ts, h]
+        end
+
+        is_expected.to match_table expected_emits_without_request_parsing
+      end
+    end
   end
 
   context 'when include bad URI' do

data/spec/in_elb_access_log_config_spec.rb

@@ -44,6 +44,10 @@ describe 'Fluent::Plugin::ElbAccessLogInput#configure' do
       expect(driver.instance.sampling_interval).to eq 1
       expect(driver.instance.debug).to be_falsey
       expect(driver.instance.elb_type).to eq 'clb'
+      expect(driver.instance.filter).to be_nil
+      expect(driver.instance.filter_operator).to eq 'and'
+      expect(driver.instance.type_cast).to be_truthy
+      expect(driver.instance.parse_request).to be_truthy
     end
   end
 
@@ -63,6 +67,8 @@ describe 'Fluent::Plugin::ElbAccessLogInput#configure' do
     let(:history_length) { 200 }
     let(:sampling_interval) { 100 }
     let(:elb_type) { 'alb' }
+    let(:filter) { 'elb_status_code:^2' }
+    let(:filter_operator) { 'or' }
 
     let(:fluentd_conf) do
       {
@@ -85,6 +91,10 @@ describe 'Fluent::Plugin::ElbAccessLogInput#configure' do
         sampling_interval: sampling_interval,
         debug: true,
         elb_type: elb_type,
+        filter: filter,
+        filter_operator: filter_operator,
+        type_cast: 'false',
+        parse_request: 'false',
       }
     end
 
@@ -107,6 +117,10 @@ describe 'Fluent::Plugin::ElbAccessLogInput#configure' do
       expect(driver.instance.sampling_interval).to eq sampling_interval
       expect(driver.instance.debug).to be_truthy
       expect(driver.instance.elb_type).to eq elb_type
+      expect(driver.instance.filter).to eq('elb_status_code' => /^2/)
+      expect(driver.instance.filter_operator).to eq filter_operator
+      expect(driver.instance.type_cast).to be_falsey
+      expect(driver.instance.parse_request).to be_falsey
     end
   end
 
@@ -190,4 +204,24 @@ describe 'Fluent::Plugin::ElbAccessLogInput#configure' do
       }.to raise_error 'Invalid ELB type: invalid'
     end
   end
+
+  context 'when an invalid filter operator' do
+    let(:start_datetime) { '2015-01-01 01:02:03 UTC' }
+
+    let(:fluentd_conf) do
+      {
+        account_id: account_id,
+        s3_bucket: s3_bucket,
+        region: region,
+        start_datetime: start_datetime,
+        filter_operator: 'invalid',
+      }
+    end
+
+    it do
+      expect {
+        subject
+      }.to raise_error 'Invalid filter operator: invalid'
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-access-log
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - Genki Sugawara
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-29 00:00:00.000000000 Z
+date: 2018-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd