fluent-plugin-elb-log 0.2.8 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d4964547c8f6c81ddc8be0aee95bc942420cd824
-  data.tar.gz: 30fb414dc08343f64ee4c671aab0e5246fb8ba55
+  metadata.gz: 9e2cdd52d7bb6925077930b4af585a251b7adefc
+  data.tar.gz: 64429389036f8c97f4a502dad80cfebce71aaa83
 SHA512:
-  metadata.gz: 26bb99092bb3b2d5bd403f1e840cfe0f33ac75355e26dd02794300383f7ed0b29e86bd3bd66953d33fca842ddbf661b00290815af6f9b4f1c6b79b609c6bca53
-  data.tar.gz: 3d4c476adb92920fa4c51a1f7d109fbe5fe8138b9a05b6b5d8c939f5ea7d9ac6e2899656b769e861008636a5f886396f5462271af878bbab56db46b2c96438b6
+  metadata.gz: 05735444de0f2eceaaa76a417cd146971bc90ac614626dbaab1d4881b85651c06ab24aabad039fa9c90047f96e53e9dc406ba12c47743522d8f6947e34dce660
+  data.tar.gz: 8592521b71b5bb4df3311ea87f833627e129e7582ccb5cb8360a5bab67140e4efc284a35d273b767ec088db432aebea1505ba2b76efa2e1871027e989181e2a9

data/.travis.yml

@@ -1,9 +1,10 @@
 lang: ruby
 
 rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.1.*
+  - 2.1
+  - 2.2
+  - 2.3.3
+  - 2.4.1
 branches:
   only:
     - master

data/README.md

@@ -3,6 +3,15 @@
 ## Overview
 - Amazon Web Services ELB log input plubin for fluentd
 
+## Requirements
+
+## Requirements
+
+| fluent-plugin-elb-log | fluentd    | ruby   |
+|-----------------------|------------|--------|
+| >= 0.3.0              | >= v0.14.0 | >= 2.1 |
+| < 0.3.0               | >= v0.12.0 | >= 1.9 |
+
 ## Installation
 
     $ fluentd-gem fluent-plugin-elb-log

data/Rakefile

@@ -5,6 +5,7 @@ Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
   test.pattern = 'test/**/*.rb'
   test.verbose = true
+  test.warning = false
 end
 
 desc 'Run tests for all'

data/fluent-plugin-elb-log.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-elb-log"
-  spec.version       = "0.2.8"
+  spec.version       = "0.3.0"
   spec.authors       = ["shinsaka"]
   spec.email         = ["shinx1265@gmail.com"]
   spec.summary       = "Amazon ELB log input plugin"
@@ -17,8 +17,11 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "fluentd", "~> 0"
+  spec.add_dependency "fluentd", ">= 0.14.0", "< 2"
   spec.add_dependency "aws-sdk", "~> 2"
 
   spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "test-unit", ">= 3.1.0"
+  spec.add_development_dependency "webmock"
 end

data/lib/fluent/plugin/in_elb_log.rb

@@ -1,38 +1,30 @@
+require 'time'
+require 'aws-sdk'
 require 'fluent/input'
 
-class Fluent::Elb_LogInput < Fluent::Input
+class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
   Fluent::Plugin.register_input('elb_log', self)
 
+  helpers :timer
+
   LOGFILE_REGEXP = /^((?<prefix>.+?)\/|)AWSLogs\/(?<account_id>[0-9]{12})\/elasticloadbalancing\/(?<region>.+?)\/(?<logfile_date>[0-9]{4}\/[0-9]{2}\/[0-9]{2})\/[0-9]{12}_elasticloadbalancing_.+?_(?<logfile_elb_name>[^_]+)_(?<elb_timestamp>[0-9]{8}T[0-9]{4}Z)_(?<elb_ip_address>.+?)_(?<logfile_hash>.+)\.log$/
   ACCESSLOG_REGEXP = /^(?<time>\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z) (?<elb>.+?) (?<client>[^ ]+)\:(?<client_port>.+?) (?<backend>.+?)(\:(?<backend_port>.+?))? (?<request_processing_time>.+?) (?<backend_processing_time>.+?) (?<response_processing_time>.+?) (?<elb_status_code>.+?) (?<backend_status_code>.+?) (?<received_bytes>.+?) (?<sent_bytes>.+?) \"(?<request_method>.+?) (?<request_uri>.+?) (?<request_protocol>.+?)\"( \"(?<user_agent>.*?)\" (?<ssl_cipher>.+?) (?<ssl_protocol>.+)(| (?<option3>.*)))?/
 
-  # To support log_level option implemented by Fluentd v0.10.43
-  unless method_defined?(:log)
-    define_method("log") { $log }
-  end
-
-  # Define `router` method to support v0.10.57 or earlier
-  unless method_defined?(:router)
-    define_method("router") { Fluent::Engine }
-  end
-
-  config_param :access_key_id, :string, :default => nil, :secret => true
-  config_param :secret_access_key, :string, :default => nil, :secret => true
-  config_param :region, :string, :default => nil
-  config_param :s3_bucketname, :string, :default => nil
-  config_param :s3_prefix, :string, :default => nil
-  config_param :tag, :string, :default => 'elb.access'
-  config_param :timestamp_file, :string, :default => nil
-  config_param :refresh_interval, :integer, :default => 300
-  config_param :buf_file, :string, :default => './fluentd_elb_log_buf_file'
-  config_param :http_proxy, :string, :default => nil
-  config_param :start_time, :string, :default => nil
+  config_param :access_key_id, :string, default: nil, secret: true
+  config_param :secret_access_key, :string, default: nil, secret: true
+  config_param :region, :string
+  config_param :s3_bucketname, :string, default: nil
+  config_param :s3_prefix, :string, default: nil
+  config_param :tag, :string, default: 'elb.access'
+  config_param :timestamp_file, :string, default: nil
+  config_param :refresh_interval, :integer, default: 300
+  config_param :buf_file, :string, default: './fluentd_elb_log_buf_file'
+  config_param :http_proxy, :string, default: nil
+  config_param :start_time, :string, default: nil
 
   def configure(conf)
     super
-    require 'aws-sdk'
 
-    raise Fluent::ConfigError.new("region is required") unless @region
     if !has_iam_role?
       raise Fluent::ConfigError.new("access_key_id is required") if @access_key_id.nil?
       raise Fluent::ConfigError.new("secret_access_key is required") if @secret_access_key.nil?
@@ -50,16 +42,7 @@ class Fluent::Elb_LogInput < Fluent::Input
 
     raise StandardError.new("s3 bucket not found #{@s3_bucketname}") unless s3bucket_is_ok()
 
-    @loop = Coolio::Loop.new
-    timer_trigger = TimerWatcher.new(@refresh_interval, true, &method(:input))
-    timer_trigger.attach(@loop)
-    @thread = Thread.new(&method(:run))
-  end
-
-  def shutdown
-    super
-    @loop.stop
-    @thread.join
+    timer_execute(:in_elb_log, @refresh_interval, &method(:input))
   end
 
   private
@@ -74,7 +57,6 @@ class Fluent::Elb_LogInput < Fluent::Input
   end
 
   def get_timestamp_file
-    require 'time'
     begin
       # get timestamp last proc
       start_time = @start_time ? Time.parse(@start_time).utc : Time.at(0)
@@ -104,7 +86,7 @@ class Fluent::Elb_LogInput < Fluent::Input
   def s3_client
     begin
       options = {
-        :region => @region,
+        region: @region,
       }
       if @access_key_id && @secret_access_key
         options[:access_key_id] = @access_key_id
@@ -273,20 +255,4 @@ class Fluent::Elb_LogInput < Fluent::Input
       log.warn "error occurred: #{e.message}"
     end
   end
-
-  def run
-    @loop.run
-  end
-
-  class TimerWatcher < Coolio::TimerWatcher
-    def initialize(interval, repeat, &callback)
-      @callback = callback
-      on_timer # first call
-      super(interval, repeat)
-    end
-
-    def on_timer
-      @callback.call
-    end
-  end
 end

data/test/helper.rb

@@ -1,19 +1,15 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
+require 'bundler/setup'
 require 'test/unit'
 require 'webmock/test_unit'
 
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
+$LOAD_PATH.unshift(__dir__)
 require 'fluent/test'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/input'
 require 'fluent/plugin/in_elb_log'
 
 class Test::Unit::TestCase
+  include Fluent::Test::Helpers
+  extend Fluent::Test::Helpers
 end

data/test/plugin/in_elb_log.rb

@@ -7,13 +7,14 @@ class Elb_LogInputTest < Test::Unit::TestCase
   end
 
   DEFAULT_CONFIG = {
-    :access_key_id     => 'dummy_access_key_id',
-    :secret_access_key => 'dummy_secret_access_key',
-    :s3_endpoint       => 's3-ap-northeast-1.amazonaws.com',
-    :s3_bucketname     => 'bummy_bucket',
-    :s3_prefix         => 'test',
-    :timestamp_file    => 'elb_last_at.dat',
-    :refresh_interval  => 300
+    access_key_id: 'dummy_access_key_id',
+    secret_access_key: 'dummy_secret_access_key',
+    s3_endpoint: 's3-ap-northeast-1.amazonaws.com',
+    s3_bucketname: 'bummy_bucket',
+    s3_prefix: 'test',
+    region: 'ap-northeast-1',
+    timestamp_file: 'elb_last_at.dat',
+    refresh_interval: 300
   }
 
   def parse_config(conf = {})
@@ -21,36 +22,49 @@ class Elb_LogInputTest < Test::Unit::TestCase
   end
 
   def create_driver(conf = DEFAULT_CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::Elb_LogInput).configure(parse_config conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::Elb_LogInput).configure(parse_config conf)
   end
 
-  def iam_info
-    stub_request(:get, "http://169.254.169.254/latest/meta-data/iam/info")
+  def iam_info_url
+    "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
   end
 
   def use_iam_role
-    iam_info.to_return(:status => [200, 'OK'])
+    stub_request(:get, iam_info_url)
+      .to_return(status: [200, 'OK'], body: "hostname")
+    stub_request(:get, "#{iam_info_url}hostname")
+      .to_return(status: [200, 'OK'],
+                 body: {
+                   "AccessKeyId" => "dummy",
+                   "SecretAccessKey" => "secret",
+                   "Token" => "token"
+                 }.to_json)
+  end
+
+  def iam_info_timeout
+    stub_request(:get, iam_info_url).to_timeout
   end
 
   def not_use_iam_role
-    iam_info.to_return(:status => [404, 'Not Found'])
+    stub_request(:get, iam_info_url)
+      .to_return(status: [404, 'Not Found'])
   end
 
   def test_confiture_default
     use_iam_role
-    assert_nothing_raised { driver = create_driver }
+    assert_nothing_raised { create_driver }
 
     exception = assert_raise(Fluent::ConfigError) {
       conf = DEFAULT_CONFIG.clone
       conf.delete(:s3_bucketname)
-      driver = create_driver(conf)
+      create_driver(conf)
     }
     assert_equal('s3_bucketname is required', exception.message)
 
     exception = assert_raise(Fluent::ConfigError) {
       conf = DEFAULT_CONFIG.clone
       conf.delete(:timestamp_file)
-      driver = create_driver(conf)
+      create_driver(conf)
     }
     assert_equal('timestamp_file is required', exception.message)
   end
@@ -60,7 +74,7 @@ class Elb_LogInputTest < Test::Unit::TestCase
     conf = DEFAULT_CONFIG.clone
     conf.delete(:access_key_id)
     conf.delete(:secret_access_key)
-    assert_nothing_raised { driver = create_driver(conf) }
+    assert_nothing_raised { create_driver(conf) }
   end
 
   def test_configure_in_EC2_without_IAM_role
@@ -68,33 +82,33 @@ class Elb_LogInputTest < Test::Unit::TestCase
     exception = assert_raise(Fluent::ConfigError) {
       conf = DEFAULT_CONFIG.clone
       conf.delete(:access_key_id)
-      driver = create_driver(conf)
+      create_driver(conf)
     }
     assert_equal('access_key_id is required', exception.message)
 
     exception = assert_raise(Fluent::ConfigError) {
       conf = DEFAULT_CONFIG.clone
       conf.delete(:secret_access_key)
-      driver = create_driver(conf)
+      create_driver(conf)
     }
     assert_equal('secret_access_key is required', exception.message)
   end
 
   def test_configure_outside_EC2
-    iam_info.to_timeout
+    iam_info_timeout
 
-    assert_nothing_raised { driver = create_driver }
+    assert_nothing_raised { create_driver }
     exception = assert_raise(Fluent::ConfigError) {
       conf = DEFAULT_CONFIG.clone
       conf.delete(:access_key_id)
-      driver = create_driver(conf)
+      create_driver(conf)
     }
     assert_equal('access_key_id is required', exception.message)
 
     exception = assert_raise(Fluent::ConfigError) {
       conf = DEFAULT_CONFIG.clone
       conf.delete(:secret_access_key)
-      driver = create_driver(conf)
+      create_driver(conf)
     }
     assert_equal('secret_access_key is required', exception.message)
   end

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-log
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.0
 platform: ruby
 authors:
 - shinsaka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-18 00:00:00.000000000 Z
+date: 2017-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +58,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Amazon ELB log input plugin for fluentd
 email:
 - shinx1265@gmail.com