fluent-plugin-rds-slowlog 0.0.5 → 0.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8f911129eaa88bdb7e6637df787ccfb7b5c42a9d
+  data.tar.gz: 3889a1524f662e638f104adce831750700481b64
+SHA512:
+  metadata.gz: ffc66003262cb3c9c7492835064f7e4fa8e7e0e25575eb3d596a96aa54349cf61f75441aee4bc3179a59c419fab4da509b8ebcaf191d86e435edc8bca233972c
+  data.tar.gz: d5c3bd047eb583eae7073a038b123d99943dc4057b760cf432558a38a4fc881acdaa4ae3484b5d795ef91705c078eacb65500761bb563f145d987ff2660d4aa0

data/.travis.yml

@@ -4,5 +4,7 @@ rvm:
   - 1.9.2
   - 1.9.3
   - 2.0.0
+  - 2.1
+  - 2.2
 
 script: bundle exec rake test

data/README.md

@@ -10,10 +10,10 @@
 - setting `long_query_time`
 
 ## Overview
-***AWS RDS slow_log*** input plugin.  
+***AWS RDS slow_log*** input plugin.
 
-1. **"SELECT * FROM slow_log"**
-2. **"CALL mysql.rds_rotate_slow_log"**
+1. **"CALL mysql.rds_rotate_slow_log"**
+2. **"SELECT * FROM slow_log_backup"**
 
 every 10 seconds from AWS RDS.
 
@@ -38,6 +38,7 @@ every 10 seconds from AWS RDS.
   host [RDS Hostname]
   username [RDS Username]
   password [RDS Password]
+  interval 10
 </source>
 
 <match rds-slowlog>

data/fluent-plugin-rds-slowlog.gemspec

@@ -4,9 +4,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-rds-slowlog"
-  gem.version       = "0.0.5"
-  gem.authors       = ["kenjiskywalker"]
-  gem.email         = ["git@kenjiskywalker.org"]
+  gem.version       = "0.0.6"
+  gem.authors       = ["kenjiskywalker", "winebarrel"]
+  gem.email         = ["git@kenjiskywalker.org", "sgwr_dts@yahoo.co.jp"]
   gem.description   = "Amazon RDS slow_log input plugin for Fluent event collector"
   gem.homepage      = "https://github.com/kenjiskywalker/fluent-plugin-rds-slowlog"
   gem.summary       = gem.description
@@ -14,7 +14,15 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_dependency "fluentd", "~> 0.10.30"
+
+  if RUBY_VERSION >= '1.9.3'
+    gem.add_dependency "fluentd"
+  else
+    gem.add_dependency "fluentd", "<= 0.10.55"
+  end
+
   gem.add_dependency "mysql2",  "~> 0.3.11"
   gem.add_development_dependency "rake", ">= 10.0.4"
+  gem.add_development_dependency "test-unit", "~> 3.1.3"
+  gem.add_development_dependency "timecop"
 end

data/lib/fluent/plugin/in_rds_slowlog.rb

@@ -6,11 +6,17 @@ class Fluent::Rds_SlowlogInput < Fluent::Input
     define_method("log") { $log }
   end
 
+  # Define `router` method of v0.12 to support v0.10 or earlier
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
   config_param :tag,      :string
   config_param :host,     :string,  :default => nil
   config_param :port,     :integer, :default => 3306
   config_param :username, :string,  :default => nil
-  config_param :password, :string,  :default => nil
+  config_param :password, :string,  :default => nil, :secret => true
+  config_param :interval, :integer, :default => 10
 
   def initialize
     super
@@ -34,6 +40,9 @@ class Fluent::Rds_SlowlogInput < Fluent::Input
 
   def start
     super
+    @loop = Coolio::Loop.new
+    timer = TimerWatcher.new(@interval, true, log, &method(:output))
+    @loop.attach(timer)
     @watcher = Thread.new(&method(:watch))
   end
 
@@ -45,21 +54,36 @@ class Fluent::Rds_SlowlogInput < Fluent::Input
 
   private
   def watch
-    while true
-      sleep 10
-      output
-    end
+    @loop.run
+  rescue => e
+    log.error(e.message)
+    log.error_backtrace(e.backtrace)
   end
 
   def output
+    @client.query('CALL mysql.rds_rotate_slow_log')
+
     slow_log_data = []
-    slow_log_data = @client.query('SELECT * FROM slow_log', :cast => false)
+    slow_log_data = @client.query('SELECT * FROM slow_log_backup', :cast => false)
 
     slow_log_data.each do |row|
       row.each_key {|key| row[key].force_encoding(Encoding::ASCII_8BIT) if row[key].is_a?(String)}
-      Fluent::Engine.emit(tag, Fluent::Engine.now, row)
+      router.emit(tag, Fluent::Engine.now, row)
     end
+  end
 
-    @client.query('CALL mysql.rds_rotate_slow_log')
+  class TimerWatcher < Coolio::TimerWatcher
+    def initialize(interval, repeat, log, &callback)
+      @callback = callback
+      @log = log
+      super(interval, repeat)
+    end
+
+    def on_timer
+      @callback.call
+    rescue => e
+      @log.error(e.message)
+      @log.error_backtrace(e.backtrace)
+    end
   end
 end

data/test/helper.rb

@@ -9,6 +9,8 @@ rescue Bundler::BundlerError => e
   exit e.status_code
 end
 require "test/unit"
+require "mysql2"
+require "timecop"
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
 $LOAD_PATH.unshift(File.dirname(__FILE__))

data/test/plugin/test_in_rds_slowlog.rb

@@ -1,8 +1,84 @@
 require 'helper'
 
 class Rds_SlowlogInputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      setup_database
+      Timecop.freeze(Time.parse('2015/05/24 18:30 UTC'))
+    end
+
+    def shutdown
+      cleanup_database
+    end
+
+    def setup_database
+      client = mysql2_client
+      client.query("GRANT ALL ON *.* TO test_rds_user@localhost IDENTIFIED BY 'test_rds_password'")
+
+      client.query <<-EOS
+        CREATE PROCEDURE `mysql`.`rds_rotate_slow_log`()
+        BEGIN
+          DECLARE sql_logging BOOLEAN;
+          select @@sql_log_bin into sql_logging;
+          set @@sql_log_bin=off;
+          CREATE TABLE IF NOT EXISTS mysql.slow_log2 LIKE mysql.slow_log;
+
+          #{insert_slow_log_sql}
+
+          DROP TABLE IF EXISTS mysql.slow_log_backup;
+          RENAME TABLE mysql.slow_log TO mysql.slow_log_backup, mysql.slow_log2 TO mysql.slow_log;
+          set @@sql_log_bin=sql_logging;
+        END
+      EOS
+    end
+
+    def cleanup_database
+      client = mysql2_client
+      client.query("DROP USER test_rds_user@localhost")
+      client.query("DROP PROCEDURE `mysql`.`rds_rotate_slow_log`")
+    end
+
+    def insert_slow_log_sql
+      if has_thread_id?
+        <<-EOS
+          INSERT INTO `mysql`.`slow_log2` (
+            `start_time`, `user_host`, `query_time`, `lock_time`, `rows_sent`, `rows_examined`, `db`, `last_insert_id`, `insert_id`, `server_id`, `sql_text`, `thread_id`)
+          VALUES
+            ('2015-09-29 15:43:44', 'root@localhost', '00:00:00', '00:00:00', 0, 0, 'employees', 0, 0, 1, 'SELECT 1', 0)
+           ,('2015-09-29 15:43:45', 'root@localhost', '00:00:00', '00:00:00', 0, 0, 'employees', 0, 0, 1, 'SELECT 2', 0)
+          ;
+        EOS
+      else
+        <<-EOS
+          INSERT INTO `mysql`.`slow_log2` (
+            `start_time`, `user_host`, `query_time`, `lock_time`, `rows_sent`, `rows_examined`, `db`, `last_insert_id`, `insert_id`, `server_id`, `sql_text`)
+          VALUES
+            ('2015-09-29 15:43:44', 'root@localhost', '00:00:00', '00:00:00', 0, 0, 'employees', 0, 0, 1, 'SELECT 1')
+           ,('2015-09-29 15:43:45', 'root@localhost', '00:00:00', '00:00:00', 0, 0, 'employees', 0, 0, 1, 'SELECT 2')
+          ;
+        EOS
+      end
+    end
+
+    def has_thread_id?
+      client = mysql2_client
+      fields = client.query("SHOW FULL FIELDS FROM `mysql`.`slow_log`").map {|r| r['Field'] }
+      fields.include?('thread_id')
+    end
+
+    def mysql2_client
+      Mysql2::Client.new(:username => 'root')
+    end
+  end
+
+  def rotate_slow_log
+    client = self.class.mysql2_client
+    client.query("CALL `mysql`.`rds_rotate_slow_log`")
+  end
+
   def setup
     Fluent::Test.setup
+    rotate_slow_log
   end
 
   CONFIG = %[
@@ -10,6 +86,7 @@ class Rds_SlowlogInputTest < Test::Unit::TestCase
     host localhost
     username test_rds_user
     password test_rds_password
+    interval 0
   ]
 
   def create_driver(conf = CONFIG)
@@ -22,6 +99,21 @@ class Rds_SlowlogInputTest < Test::Unit::TestCase
     assert_equal 'localhost', d.instance.host
     assert_equal 'test_rds_user', d.instance.username
     assert_equal 'test_rds_password', d.instance.password
+    assert_equal 0, d.instance.interval
   end
 
+  def test_output
+    d = create_driver
+    d.run
+    records = d.emits
+
+    unless self.class.has_thread_id?
+      records.each {|r| r[2]["thread_id"] = "0" }
+    end
+
+    assert_equal [
+      ["rds-slowlog", 1432492200, {"start_time"=>"2015-09-29 15:43:44", "user_host"=>"root@localhost", "query_time"=>"00:00:00", "lock_time"=>"00:00:00", "rows_sent"=>"0", "rows_examined"=>"0", "db"=>"employees", "last_insert_id"=>"0", "insert_id"=>"0", "server_id"=>"1", "sql_text"=>"SELECT 1", "thread_id"=>"0"}],
+      ["rds-slowlog", 1432492200, {"start_time"=>"2015-09-29 15:43:45", "user_host"=>"root@localhost", "query_time"=>"00:00:00", "lock_time"=>"00:00:00", "rows_sent"=>"0", "rows_examined"=>"0", "db"=>"employees", "last_insert_id"=>"0", "insert_id"=>"0", "server_id"=>"1", "sql_text"=>"SELECT 2", "thread_id"=>"0"}],
+    ], records
+  end
 end

metadata

@@ -1,73 +1,96 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rds-slowlog
 version: !ruby/object:Gem::Version
-  version: 0.0.5
-  prerelease: 
+  version: 0.0.6
 platform: ruby
 authors:
 - kenjiskywalker
+- winebarrel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-05 00:00:00.000000000 Z
+date: 2015-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.30
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.30
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.11
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.11
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 10.0.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 10.0.4
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.3
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Amazon RDS slow_log input plugin for Fluent event collector
 email:
 - git@kenjiskywalker.org
+- sgwr_dts@yahoo.co.jp
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -78,27 +101,26 @@ files:
 - test/plugin/test_in_rds_slowlog.rb
 homepage: https://github.com/kenjiskywalker/fluent-plugin-rds-slowlog
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.4.5
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Amazon RDS slow_log input plugin for Fluent event collector
 test_files:
 - test/helper.rb