fluent-plugin-elb-log 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6d34dc14c5627860aa7787a9c0e0dbac19edaf9d
-  data.tar.gz: 57c8a63726d0ad90f47653b6f3d20286d1527129
+  metadata.gz: f565fbbe387bad73d53cee18557d89ae0acd9f4f
+  data.tar.gz: 6a37e1c172a8d1c5b20394dbaa38923b9b82b0e4
 SHA512:
-  metadata.gz: 6854587f02206b4cdcd14863516d4df01b79eda6af2765d4250c3139e21ea1bb20896b7dba86cd29287aafa5b00e7983d02456ee50bfd29da6519d6c57622012
-  data.tar.gz: cfb29cc94c144bf8a877a3cd287510be473c31f23fa2f6879b0421ccba0e8a992559057e27f5f2c0edc12bd6877544872c878b570511bc1eb05b58d0e173b612
+  metadata.gz: 58c8fd4c5ba84514c783afb2867438c4646a171f41d3d8fcb052db6624f807578b7bec83061f357b04791662b5a42d96cf973e0426aa5d091a5690b62625a5ba
+  data.tar.gz: 85870e3681ddc11eb40ad22feed503aacfddd970dfc108917af87f86fd06fadba084f1f09ff4bd0d3d38a2c88d436e463bfef1403fde1c690735ddb4b8accb04

data/.travis.yml

@@ -1,4 +1,8 @@
-lang: ruby
+env:
+  global:
+    - CC_TEST_REPORTER_ID=bca5d9771c392e374d213de82dde44181a12b4c8b5d5bd37b249c81ca5bf32f1
+    - GIT_COMMITTED_AT=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then git log -1 --pretty=format:%ct; else git log -1 --skip 1 --pretty=format:%ct; fi)
+language: ruby
 
 rvm:
   - 2.1
@@ -8,3 +12,9 @@ rvm:
 branches:
   only:
     - master
+
+before_script:
+  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+  - chmod +x ./cc-test-reporter
+after_script:
+  - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT

data/README.md

@@ -1,5 +1,10 @@
 # Amazon ELB log input plugin for fluentd
 
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-elb-log.svg)](https://badge.fury.io/rb/fluent-plugin-elb-log)
+[![Build Status](https://travis-ci.org/shinsaka/fluent-plugin-elb-log.svg?branch=master)](https://travis-ci.org/shinsaka/fluent-plugin-elb-log)
+[![Code Climate](https://codeclimate.com/github/shinsaka/fluent-plugin-elb-log/badges/gpa.svg)](https://codeclimate.com/github/shinsaka/fluent-plugin-elb-log)
+[![Test Coverage](https://codeclimate.com/github/shinsaka/fluent-plugin-elb-log/badges/coverage.svg)](https://codeclimate.com/github/shinsaka/fluent-plugin-elb-log/coverage)
+
 ## Overview
 - Amazon Web Services ELB log input plubin for fluentd
 

data/fluent-plugin-elb-log.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-elb-log"
-  spec.version       = "0.4.0"
+  spec.version       = "0.4.1"
   spec.authors       = ["shinsaka"]
   spec.email         = ["shinx1265@gmail.com"]
   spec.summary       = "Amazon ELB log input plugin"
@@ -21,7 +21,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "aws-sdk", "~> 2"
 
   spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "test-unit", ">= 3.1.0"
-  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "rake", "~> 12"
+  spec.add_development_dependency "test-unit", "~> 3.2"
+  spec.add_development_dependency "webmock", "~>3"
+  spec.add_development_dependency "simplecov", "~>0"
 end

data/lib/fluent/plugin/in_elb_log.rb

@@ -33,6 +33,7 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     end
     raise Fluent::ConfigError.new("s3_bucketname is required") unless @s3_bucketname
     raise Fluent::ConfigError.new("timestamp_file is required") unless @timestamp_file
+    raise Fluent::ConfigError.new("s3 bucket not found #{@s3_bucketname}") unless s3bucket_is_ok?
   end
 
   def start
@@ -42,8 +43,6 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     File.open(@timestamp_file, File::RDWR|File::CREAT).close
     File.open(@buf_file, File::RDWR|File::CREAT).close
 
-    raise StandardError.new("s3 bucket not found #{@s3_bucketname}") unless s3bucket_is_ok()
-
     timer_execute(:in_elb_log, @refresh_interval, &method(:input))
   end
 
@@ -104,15 +103,15 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     end
   end
 
-  def s3bucket_is_ok
-    begin
-      log.debug "search bucket #{@s3_bucketname}"
+  def s3bucket_is_ok?
+    log.debug "searching for bucket #{@s3_bucketname}"
 
-      s3_client.list_buckets.buckets.any? do |bucket|
-        bucket.name == @s3_bucketname
-      end
+    begin
+      # try get one
+      !(get_object_list(1).nil?)
     rescue => e
-      log.warn "S3 Client error occurred: #{e.message}"
+      log.warn "error occurred: #{e.message}"
+      false
     end
   end
 
@@ -180,14 +179,8 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
 
   def get_object_keys_from_s3
     begin
-      objects = s3_client.list_objects(
-        bucket: @s3_bucketname,
-        max_keys: 100,
-        prefix: @s3_prefix,
-      )
-
       object_keys = []
-      objects.each do |object|
+      get_object_list(100).each do |object|
         object.contents.each do |content|
           object_keys << content.key
         end
@@ -208,6 +201,14 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     end
   end
 
+  def get_object_list(max_num)
+    s3_client.list_objects(
+      bucket: @s3_bucketname,
+      max_keys: max_num,
+      prefix: @s3_prefix
+    )
+  end
+
   def get_file_from_s3(object_name)
     begin
       log.debug "getting object from s3 name is #{object_name}"
@@ -244,37 +245,38 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
             next
           end
 
-          record = {
-            "time" => line_match[:time].gsub(/Z/, "+0000"),
-            "elb" => line_match[:elb],
-            "client" => line_match[:client],
-            "client_port" => line_match[:client_port],
-            "backend" => line_match[:backend],
-            "backend_port" => line_match[:backend_port],
-            "request_processing_time" => line_match[:request_processing_time].to_f,
-            "backend_processing_time" => line_match[:backend_processing_time].to_f,
-            "response_processing_time" => line_match[:response_processing_time].to_f,
-            "elb_status_code" => line_match[:elb_status_code],
-            "backend_status_code" => line_match[:backend_status_code],
-            "received_bytes" => line_match[:received_bytes].to_i,
-            "sent_bytes" => line_match[:sent_bytes].to_i,
-            "request_method" => line_match[:request_method],
-            "request_uri" => line_match[:request_uri],
-            "request_protocol" => line_match[:request_protocol],
-            "user_agent" => line_match[:user_agent],
-            "ssl_cipher" => line_match[:ssl_cipher],
-            "ssl_protocol" => line_match[:ssl_protocol],
-            "type" => line_match[:type],
-            "target_group_arn" => line_match[:target_group_arn],
-            "trace_id" => line_match[:trace_id],
-            "option3" => line_match[:option3],
-          }
-
-          router.emit(@tag, Fluent::Engine.now, record_common.merge(record))
+          router.emit(@tag, Fluent::Engine.now, record_common.merge(format_record(line_match)))
         end
       end
     rescue => e
       log.warn "error occurred: #{e.message}"
     end
   end
+
+  def format_record(item)
+    { "time" => item[:time].gsub(/Z/, '+0000'),
+      "elb" => item[:elb],
+      "client" => item[:client],
+      "client_port" => item[:client_port],
+      "backend" => item[:backend],
+      "backend_port" => item[:backend_port],
+      "request_processing_time" => item[:request_processing_time].to_f,
+      "backend_processing_time" => item[:backend_processing_time].to_f,
+      "response_processing_time" => item[:response_processing_time].to_f,
+      "elb_status_code" => item[:elb_status_code],
+      "backend_status_code" => item[:backend_status_code],
+      "received_bytes" => item[:received_bytes].to_i,
+      "sent_bytes" => item[:sent_bytes].to_i,
+      "request_method" => item[:request_method],
+      "request_uri" => item[:request_uri],
+      "request_protocol" => item[:request_protocol],
+      "user_agent" => item[:user_agent],
+      "ssl_cipher" => item[:ssl_cipher],
+      "ssl_protocol" => item[:ssl_protocol],
+      "type" => item[:type],
+      "target_group_arn" => item[:target_group_arn],
+      "trace_id" => item[:trace_id],
+      "option3" => item[:option3]
+    }
+  end
 end

data/test/helper.rb

@@ -1,6 +1,9 @@
 require 'bundler/setup'
 require 'test/unit'
 require 'webmock/test_unit'
+require 'simplecov'
+
+SimpleCov.start
 
 $LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
 $LOAD_PATH.unshift(__dir__)

data/test/plugin/in_elb_log.rb

@@ -10,7 +10,7 @@ class Elb_LogInputTest < Test::Unit::TestCase
     access_key_id: 'dummy_access_key_id',
     secret_access_key: 'dummy_secret_access_key',
     s3_endpoint: 's3-ap-northeast-1.amazonaws.com',
-    s3_bucketname: 'bummy_bucket',
+    s3_bucketname: 'dummy_bucket',
     s3_prefix: 'test',
     region: 'ap-northeast-1',
     timestamp_file: 'elb_last_at.dat',
@@ -26,7 +26,7 @@ class Elb_LogInputTest < Test::Unit::TestCase
   end
 
   def iam_info_url
-    "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
+    'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
   end
 
   def use_iam_role
@@ -50,7 +50,18 @@ class Elb_LogInputTest < Test::Unit::TestCase
       .to_return(status: [404, 'Not Found'])
   end
 
-  def test_confiture_default
+  def s3bucket_ok
+    stub_request(:get, 'https://s3-ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test')
+      .to_return(status: 200, body: "", headers: {})
+  end
+
+  def s3bucket_not_found
+    stub_request(:get, 'https://s3-ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test')
+      .to_return(status: 404, body: "", headers: {})
+  end
+
+  def test_configure_default
+    s3bucket_ok
     use_iam_role
     assert_nothing_raised { create_driver }
 
@@ -70,6 +81,7 @@ class Elb_LogInputTest < Test::Unit::TestCase
   end
 
   def test_configure_in_EC2_with_IAM_role
+    s3bucket_ok
     use_iam_role
     conf = DEFAULT_CONFIG.clone
     conf.delete(:access_key_id)
@@ -78,8 +90,9 @@ class Elb_LogInputTest < Test::Unit::TestCase
   end
 
   def test_configure_in_EC2_without_IAM_role
-    not_use_iam_role
     exception = assert_raise(Fluent::ConfigError) {
+      s3bucket_ok
+      not_use_iam_role
       conf = DEFAULT_CONFIG.clone
       conf.delete(:access_key_id)
       create_driver(conf)
@@ -95,6 +108,7 @@ class Elb_LogInputTest < Test::Unit::TestCase
   end
 
   def test_configure_outside_EC2
+    s3bucket_ok
     iam_info_timeout
 
     assert_nothing_raised { create_driver }
@@ -113,6 +127,15 @@ class Elb_LogInputTest < Test::Unit::TestCase
     assert_equal('secret_access_key is required', exception.message)
   end
 
+  def test_not_found_s3bucket
+    e = assert_raise(Fluent::ConfigError) {
+      use_iam_role
+      s3bucket_not_found
+      create_driver(DEFAULT_CONFIG.clone)
+    }
+    assert_equal('s3 bucket not found dummy_bucket', e.message)
+  end
+
   def test_logfilename_classic_lb_parse
     logfile_classic = 'classic/AWSLogs/123456789012/elasticloadbalancing/ap-northeast-1/2017/05/03/123456789012_elasticloadbalancing_ap-northeast-1_elbname_20170503T1250Z_10.0.0.1_43nzjpdj.log'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-log
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - shinsaka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-07 00:00:00.000000000 Z
+date: 2017-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -62,42 +62,56 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: '3.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 description: Amazon ELB log input plugin for fluentd