fluent-plugin-mysql-status 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 068bb41e6821684fbbfe794d73c091f89fa60525
+  data.tar.gz: 907ca182d8db0739c9cfad713157cf733baca1d3
+SHA512:
+  metadata.gz: 27f8cb79fc23cc083a5d03f8b678c07c1ef518fb6cd3b36a4bfc3166b089a63456b3611732ef57f5f677b87db2e425bfde50396eb950d180b08683f1f652c420
+  data.tar.gz: 14722355dcdcf6aa7f98284008a854f34ac0c1762cce80e1172e4c515877809f72551efa39ff44562468723828bc9cf2893d3258de2ff847f1643a330c44c296

data/.gitignore

@@ -0,0 +1,14 @@
+.bundle
+.yardoc
+Gemfile.lock
+_yardoc
+coverage
+doc
+pkg
+spec/reports
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - 2.2.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-mysql-status.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (C) 2016- gumi Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,173 @@
+# fluent-plugin-mysql-status
+
+[fluentd](http://fluentd.org) input plugin that monitor status of MySQL Server.
+
+[![Build Status](https://travis-ci.org/gumi/fluent-plugin-mysql-status.svg?branch=master)](https://travis-ci.org/gumi/fluent-plugin-mysql-status)
+[![Code Climate](https://codeclimate.com/github/gumi/fluent-plugin-plugin-mysql/badges/gpa.svg)](https://codeclimate.com/github/gumi/fluent-plugin-mysql-status)
+
+## Installation
+
+Install with gem or fluent-gem command as:
+
+```
+# for fluentd
+$ gem install fluent-plugin-mysql-status
+
+# for td-agent
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-mysql-status
+```
+
+## Configuration
+
+```
+<source>
+  type mysql_status
+  tag mysql_status                  # Required
+
+  host 127.0.0.1                    # Optional (default: localhost)
+  port 3306                         # Optional (default: 3306)
+  username ham                      # Optional (default: root)
+  password egg                      # Optional (default: nopassword)
+  database spam                     # Optional (default: unselected database)
+  encoding utf8                     # Optional (default: utf8)
+
+  comment Fluent::MySQLStatusInput  # Optional (default: Fluent::MySQLStatusInput)
+
+  <query>
+    type processlist  # Required if path and string is undefined
+    tag processlist   # Required
+    interval 1        # Optional (default: 10)
+  </query>
+
+  <query>
+    path /path/to/statements_with_errors.sql  # Required if type and string is undefined
+    tag errors
+  </query>
+
+  <query>
+    string SHOW STATUS                  # Required if type and path is undefined
+    tag status
+    omit_variable_name_from_record true # Optional (default: false)
+    clump_records true                  # Optional (default: false)
+  </query>
+
+  <query>
+    string SELECT DIGEST_TEXT FROM performance_schema.events_statements_summary_by_digest
+    tag summary
+    clump_records true          # Optional (default: false)
+    clumped_records_key records # Optional (default: records)
+  </query>
+</source>
+```
+
+Since this plugin runs multiple queries with a single connection, it doesn't put too much load on the MySQL server.
+However, since queries run sequentially, slow queries hinder other queries from running.
+
+### query.type
+
+The following types are available.
+
+- processlist - SHOW FULL PROCESSLIST
+- status - SHOW OPEN TABLES
+- open_tables - SHOW /*!50002 GLOBAL */ STATUS
+
+### query.string
+
+If you don't like the types mentioned above, you can write a query freely.
+
+### query.path
+
+If your query is long, write the query to a file and set the file path to this setting.
+
+### tag + query.tag
+
+If you set the following configuration:
+
+```
+<source>
+  tag ham
+
+  <query>
+    tag egg
+  </query>
+</source>
+```
+
+This plugin make the following tag:
+
+```
+ham.egg
+```
+
+### query.interval
+
+Set the interval at which the query runs in seconds.
+
+### query.omit_variable_name_from_record
+
+If the result of the query is in the following format:
+
+```
+2016-12-23 23:09:56 +0900 mysql_status.spam: {'Variable_name': 'ham', 'Value': 'egg'}
+2016-12-23 23:09:56 +0900 mysql_status.spam: {'Variable_name': 'foo', 'Value': 'bar'}
+```
+
+Setting this to true results in the following result:
+
+```
+2016-12-23 23:09:56 +0900 mysql_status.spam.ham: 'egg'
+2016-12-23 23:09:56 +0900 mysql_status.spam.foo: 'bar'
+```
+
+### query.clump_records
+
+If the result of the query is in the following format:
+
+```
+2016-12-23 23:09:56 +0900 mysql_status.spam: {'ham': 1, 'egg': 2}
+2016-12-23 23:09:56 +0900 mysql_status.spam: {'ham': 3, 'egg': 4}
+```
+
+Setting this to true results in the following result:
+
+```
+2016-12-23 23:09:56 +0900 mysql_status.spam: {'records': [{'ham': 1, 'egg': 2}, {'ham': 3, 'egg': 4}]}
+```
+
+### query.clumped_records_key
+
+Set the key name to clump the records.
+
+### comment
+
+This setting is added as a comment to all queries.
+
+If you set the following configuration:
+
+```
+<source>
+  comment This query was ran by fluentd
+
+  <query>
+    string SELECT spam FROM ham.egg
+  </query>
+
+  <query>
+    string SELECT foo FROM bar.baz
+  </query>
+</source>
+```
+
+This plugin executes the following queries:
+
+```
+SELECT spam FROM ham.egg /* This query was ran by fluentd */
+SELECT foo FROM bar.baz /* This query was ran by fluentd */
+```
+
+## Copyright
+
+- Copyright
+  - Copyright (C) 2016- gumi Inc.
+- License
+  - Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-mysql-status.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluent/plugin/mysql_status/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-mysql-status"
+  spec.version       = Fluent::MySQLStatus::VERSION
+  spec.authors       = ["IKUTA Masahito"]
+  spec.email         = ["masahito.ikuta@gu3.co.jp"]
+  spec.summary       = %q{Fluentd input plugin that monitor status of MySQL Server.}
+  spec.description   = spec.summary
+  spec.homepage      = "http://github.com/gumi/fluent-plugin-mysql-status"
+  spec.license       = "APLv2"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit", ["~> 3.0", "~> 3.1"]
+
+  spec.add_dependency "fluentd", ">= 0.10.55"
+  spec.add_dependency "mysql2", "~> 0.3.11"
+end

data/lib/fluent/plugin/in_mysql_status.rb

@@ -0,0 +1,243 @@
+require 'mysql2'
+
+module Fluent
+  class MySQLStatusInput < Input
+    Plugin.register_input('mysql_status', self)
+
+    config_param :tag, :string
+
+    config_param :host, :string, :default => 'localhost'
+    config_param :port, :integer, :default => 3306
+    config_param :username, :string, :default => 'root'
+    config_param :password, :string, :default => nil
+    config_param :database, :string, :default => nil
+    config_param :encoding, :string, :default => 'utf8'
+
+    config_param :comment, :string, :default => 'Fluent::MySQLStatusInput'
+
+    attr_reader :queries
+
+    def initialize
+      super
+      @queries = []
+    end
+
+    def configure(conf)
+      super
+
+      conf.elements.select {|element|
+        element.name == 'query'
+      }.each do |element|
+        tag = element['tag'] or raise ConfigError, "Missing 'tag' parameter on <query> directive"
+        string, default = configure_query_string(element)
+        interval = element['interval'] || 10
+
+        omit_variable_name_from_record, clump_records = [
+          'omit_variable_name_from_record', 'clump_records',
+        ].map do |key|
+          configure_query_format_flag(element, key, default)
+        end
+
+        clumped_records_key = element['clumped_records_key'] || 'records'
+
+        @queries << {
+          :tag => tag,
+          :string => string,
+          :interval => Integer(interval),
+          :omit_variable_name_from_record => omit_variable_name_from_record,
+          :clump_records => clump_records,
+          :clumped_records_key => clumped_records_key,
+        }
+      end
+    end
+
+    def start
+      @watcher = Thread.new(&method(:watch))
+    end
+
+    def shutdown
+      @watcher.kill
+    end
+
+    def watch
+      client = ensure_connect()
+      counter = generate_counter()
+      loop do
+        begin
+          emit_queries(client, counter.next())
+        rescue => e
+          $log.error e.message
+          client = ensure_connect()
+        end
+        sleep 1
+      end
+    end
+
+    private
+
+    def configure_query_string(conf)
+      type = conf['type']
+      path = conf['path']
+      string = conf['string']
+
+      if type and path or type and string or path and string
+        raise ConfigError, "'type', 'path' and 'string' parameter can't be defined together."
+      end
+
+      if type
+        return case type
+        when 'processlist'
+          ['SHOW FULL PROCESSLIST', false]
+        when 'open_tables'
+          ['SHOW OPEN TABLES', false]
+        when 'status'
+          ['SHOW /*!50002 GLOBAL */ STATUS', true]
+        else
+          raise ConfigError, "Missing 'type' parameter on <query> directive"
+        end
+      elsif path
+        return [File.read(path), false]
+      elsif string
+        return [string, false]
+      else
+        raise ConfigError, "Missing 'type' parameter on <query> directive"
+      end
+    end
+
+    def configure_query_format_flag(conf, key, default)
+      case conf[key]
+      when nil
+        default
+      when 'true'
+        true
+      else
+        false
+      end
+    end
+
+    def ensure_connect
+      intervals = generate_intervals()
+      begin
+        return Mysql2::Client.new({
+          :host => @host,
+          :port => @port,
+          :username => @username,
+          :password => @password,
+          :database => @database,
+          :encoding => @encoding,
+          # :reconnect => true,
+        })
+      rescue => e
+        $log.error e.message
+        sleep intervals.next()
+        retry
+      end
+    end
+
+    def generate_intervals
+      return Enumerator.new do |yielder|
+        temp, interval = 0, 1
+        loop do
+          if 55 <= interval
+            yielder << 60
+          else
+            temp, interval = interval, temp + interval
+            yielder << interval
+          end
+        end
+      end
+    end
+
+    def generate_counter
+      return Enumerator.new do |yielder|
+        counter = 0
+        loop do
+          counter += 1
+          yielder << counter
+          if 86400 <= counter
+            counter = 0
+          end
+        end
+      end
+    end
+
+    def emit_queries(client, counter)
+      @queries.each do |query|
+        next unless can_emit?(counter, query[:interval])
+        result = run_query(client, query[:string]) or next
+        if query[:omit_variable_name_from_record]
+          omit_and_emit(query, result)
+        else
+          emit(query, result)
+        end
+      end
+    end
+
+    def can_emit?(counter, interval)
+      return counter % interval == 0
+    end
+
+    def run_query(client, query)
+      begin
+        return client.query("#{query} /* #{@comment} */", :cast => false, :cache_rows => false)
+      rescue Mysql2::Error => e
+        raise if [nil, 1040, 1053, 2002, 2003, 2006, 2013].include?(e.error_number)
+        $log.error %Q(query: "#{query}" reason: "#{e.message}")
+        return nil
+      end
+    end
+
+    def emit(query, result)
+      tag = make_tag(query[:tag])
+      time = Engine.now
+
+      records = []
+      result.each do |row|
+        record = {}
+        row.each do |key, value|
+          record[key] = format(value)
+        end
+
+        if query[:clump_records]
+          records << record
+        else
+          router.emit(tag, time, record)
+        end
+      end
+
+      if query[:clump_records]
+        router.emit(tag, time, {query[:clumped_records_key] => records})
+      end
+    end
+
+    def omit_and_emit(query, result)
+      tag = make_tag(query[:tag])
+      time = Engine.now
+
+      record = {}
+      result.each do |row|
+        key = row.fetch('Variable_name')
+        value = format(row.fetch('Value'))
+
+        if query[:clump_records]
+          record[key] = value
+        else
+          emit_tag = [tag, key].join('.')
+          router.emit(emit_tag, time, value)
+        end
+      end
+
+      if query[:clump_records]
+        router.emit(tag, time, record)
+      end
+    end
+
+    def make_tag(tag_suffix)
+      return [@tag, tag_suffix].join('.')
+    end
+
+    def format(value)
+      return Integer(value) rescue Float(value) rescue value
+    end
+  end
+end

data/lib/fluent/plugin/mysql_status/version.rb

@@ -0,0 +1,5 @@
+module Fluent
+  module MySQLStatus
+    VERSION = '0.0.1'
+  end
+end

data/test/helper.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fileutils'
+require 'fluent/log'
+require 'fluent/test'
+
+$log = Fluent::Log.new(STDOUT, Fluent::Log::LEVEL_DEBUG)
+
+unless defined?(Test::Unit::AssertionFailedError)
+  class Test::Unit::AssertionFailedError < StandardError
+  end
+end
+
+require 'fluent/plugin/in_mysql_status'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_mysql_status.rb

@@ -0,0 +1,114 @@
+require 'helper'
+
+class MySQLStatusInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  TAG = 'tag'
+  HOST = 'localhost'
+  PORT = 3306
+  USERNAME = 'user'
+  PASSWORD = 'pw'
+  DATABASE = 'test_db'
+  ENCODING = 'utf8'
+  COMMENT = 'Fluent::MySQLStatusInput'
+
+  PATH = File.expand_path('../../test.sql', __FILE__)
+
+  QUERIES = [
+    {
+      :tag => 'tag1',
+      :string => 'select * from test.test',
+      :interval => 10,
+      :omit_variable_name_from_record => false,
+      :clump_records => false,
+      :clumped_records_key => 'records',
+    },
+    {
+      :tag => 'tag2',
+      :string => 'SHOW FULL PROCESSLIST',
+      :interval => 5,
+      :omit_variable_name_from_record => false,
+      :clump_records => false,
+      :clumped_records_key => 'records',
+    },
+    {
+      :tag => 'tag3',
+      :string => 'SHOW OPEN TABLES',
+      :interval => 10,
+      :omit_variable_name_from_record => false,
+      :clump_records => true,
+      :clumped_records_key => 'results',
+    },
+    {
+      :tag => 'tag4',
+      :string => 'SHOW /*!50002 GLOBAL */ STATUS',
+      :interval => 10,
+      :omit_variable_name_from_record => true,
+      :clump_records => true,
+      :clumped_records_key => 'records',
+    },
+    {
+      :tag => 'tag5',
+      :string => File.read(PATH),
+      :interval => 10,
+      :omit_variable_name_from_record => false,
+      :clump_records => false,
+      :clumped_records_key => 'records',
+    },
+  ]
+
+  CONFIG = %[
+    tag #{TAG}
+    host #{HOST}
+    username #{USERNAME}
+    password #{PASSWORD}
+    database #{DATABASE}
+
+    <query>
+      tag tag1
+      string select * from test.test
+    </query>
+
+    <query>
+      tag tag2
+      type processlist
+      interval 5
+    </query>
+
+    <query>
+      tag tag3
+      type open_tables
+      clump_records true
+      clumped_records_key results
+    </query>
+
+    <query>
+      tag tag4
+      type status
+    </query>
+
+    <query>
+      tag tag5
+      path #{PATH}
+    </query>
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::MySQLStatusInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal TAG, d.instance.tag
+    assert_equal HOST, d.instance.host
+    assert_equal PORT, d.instance.port
+    assert_equal USERNAME, d.instance.username
+    assert_equal PASSWORD, d.instance.password
+    assert_equal DATABASE, d.instance.database
+    assert_equal ENCODING, d.instance.encoding
+    assert_equal COMMENT, d.instance.comment
+    assert_equal QUERIES, d.instance.queries
+  end
+end

data/test/test.sql

@@ -0,0 +1,3 @@
+SELECT DIGEST_TEXT
+  FROM performance_schema.events_statements_summary_by_digest
+ WHERE SUM_ERRORS > 0

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-mysql-status
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- IKUTA Masahito
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-12-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.55
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.55
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.11
+description: Fluentd input plugin that monitor status of MySQL Server.
+email:
+- masahito.ikuta@gu3.co.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-mysql-status.gemspec
+- lib/fluent/plugin/in_mysql_status.rb
+- lib/fluent/plugin/mysql_status/version.rb
+- test/helper.rb
+- test/plugin/test_in_mysql_status.rb
+- test/test.sql
+homepage: http://github.com/gumi/fluent-plugin-mysql-status
+licenses:
+- APLv2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Fluentd input plugin that monitor status of MySQL Server.
+test_files:
+- test/helper.rb
+- test/plugin/test_in_mysql_status.rb
+- test/test.sql