fluent-plugin-rds-error-log 0.2.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d315c53a55a2110c23be6faf35c81f687ad1d262
+  data.tar.gz: f1ed7bbb5a565d624754d690ae170d3dbc363377
+SHA512:
+  metadata.gz: a37ff98498106ae1c4409bba3d0762460ec4c203fcbf2d20600bd5f82bd4a8f652fead484dbf2490f09926d3466f5708580197ccc31be6bbe7b622ed74c3c7c2
+  data.tar.gz: d5866eac496bf562e0635a2c85ab0b977d1757bdc108a4946fe5e00847cef9376dbd4162372980c910e66ed7bb618a9be63853ab5f292650182b57dd60f1abed

data/.gitignore

@@ -0,0 +1,3 @@
+*.gem
+fluent.conf
+

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+ - 2.2
+ - jruby
+ - rbx-2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+gem 'rake'
+gem 'activerecord'
+gem 'pg'

data/Gemfile.lock

@@ -0,0 +1,35 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (4.2.8)
+      activesupport (= 4.2.8)
+      builder (~> 3.1)
+    activerecord (4.2.8)
+      activemodel (= 4.2.8)
+      activesupport (= 4.2.8)
+      arel (~> 6.0)
+    activesupport (4.2.8)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    arel (6.0.4)
+    builder (3.2.3)
+    i18n (0.8.4)
+    minitest (5.10.2)
+    pg (0.20.0)
+    rake (12.0.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.3)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  pg
+  rake
+
+BUNDLED WITH
+   1.15.1

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 shinsaka
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# Amazon RDS for MySQL log input plugin for fluentd
+(Pulled from forked version of fluent-plugin-mysql-log repo)
+
+## Overview
+- Amazon Web Services RDS log input plugin for fluentd
+
+## Configuration
+
+```config
+<source>
+  type rds_error_log
+  # required
+  region                 <region name>
+  db_instance_identifier <instance identifier>
+  # optional if you can IAM credentials
+  access_key_id          <access_key>
+  secret_access_key      <secret_access_key>
+  # optional
+  refresh_interval       <interval number by second(default: 30)>
+  tag                    <tag name(default: rds-mysql.log>
+  pos_file               <log getting position file(default: rds-mysql.log)>
+</source>
+```
+
+### Example setting
+```config
+<source>
+  type rds_error_log
+  region ap-northeast-1
+  db_instance_identifier test-mysql
+  access_key_id     XXXXXXXXXXXXXXXXXXXX
+  secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+  refresh_interval  30
+  tag mysql.log
+  pos_file /tmp/mysql-log-pos.dat
+</source>
+
+<match mysql.log>
+  type stdout
+</match>
+```

data/fluent-plugin-rds-error-log.gemspec

@@ -0,0 +1,24 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-rds-error-log"
+  spec.version       = "0.2.2"
+  spec.authors       = ["Jen LaGrutta"]
+  spec.email         = ["jen@tune.com"]
+  spec.summary       = "Amazon RDS for Error/Audit log input plugin"
+  spec.description   = "fluentd plugin for Amazon RDS for Error/Audit log input"
+  spec.homepage      = "https://github.com/hasjenl/fluent-plugin-rds-error-log"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "fluentd", "~> 0"
+  spec.add_dependency "aws-sdk", "~> 2"
+  spec.add_dependency "myslog", "~> 0.0"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+end

data/fluent.conf.sample

@@ -0,0 +1,14 @@
+<source>
+  type rds_error_log
+  access_key_id     XXXXXXXXXXXXXXXXXXXX
+  secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+  region ap-northeast-1
+  db_instance_identifier test-mysql
+  refresh_interval 30
+  tag mysql.log
+  pos_file /error/mysql-error.log
+</source>
+
+<match mysql.log>
+  type stdout
+</match>

data/lib/fluent/plugin/in_rds_error_log.rb

@@ -0,0 +1,298 @@
+require 'fluent/input'
+
+class Fluent::RdsErrorLogInput < Fluent::Input
+  Fluent::Plugin.register_input("rds_error_log", self)
+
+  # To support log_level option implemented by Fluentd v0.10.43
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  # Define `router` method of v0.12 to support v0.10 or earlier
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
+  LOG_REGEXP = /^(?<time>\d{4}-\d{2}-\d{2} \d{2}\:\d{2}\:\d{2})( (?<pid>\d+))?( \[(?<message_level>[^\]]*?)\])? (?<message>.*)$/
+
+  config_param :access_key_id, :string, :default => nil
+  config_param :secret_access_key, :string, :default => nil
+  config_param :region, :string, :default => nil
+  config_param :db_instance_identifier, :string, :default => nil
+  config_param :pos_file, :string, :default => "fluent-plugin-rds-error-log-pos.dat"
+  config_param :refresh_interval, :integer, :default => 30
+  config_param :tag, :string, :default => "rds-mysql.log"
+  config_param :output_per_file, :bool, default: false
+  config_param :raw_output, :bool, default: false
+
+  def configure(conf)
+    super
+    require 'aws-sdk'
+
+    raise Fluent::ConfigError.new("region is required") unless @region
+    if !has_iam_role?
+      raise Fluent::ConfigError.new("access_key_id is required") if @access_key_id.nil?
+      raise Fluent::ConfigError.new("secret_access_key is required") if @secret_access_key.nil?
+    end
+    raise Fluent::ConfigError.new("db_instance_identifier is required") unless @db_instance_identifier
+    raise Fluent::ConfigError.new("pos_file is required") unless @pos_file
+    raise Fluent::ConfigError.new("refresh_interval is required") unless @refresh_interval
+    raise Fluent::ConfigError.new("tag is required") unless @tag
+  end
+
+  def start
+    super
+
+    # pos file touch
+    File.open(@pos_file, File::RDWR|File::CREAT).close
+
+    begin
+      options = {
+        :region => @region,
+      }
+      if @access_key_id && @secret_access_key
+        options[:access_key_id] = @access_key_id
+        options[:secret_access_key] = @secret_access_key
+      end
+      @rds = Aws::RDS::Client.new(options)
+    rescue => e
+      $log.warn "RDS Client error occurred: #{e.message}"
+    end
+
+    @loop = Coolio::Loop.new
+    timer_trigger = TimerWatcher.new(@refresh_interval, true, &method(:input))
+    timer_trigger.attach(@loop)
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    super
+    @watcher.terminate
+    @thread.join
+  end
+
+  private
+
+  def run
+    @loop.run
+  end
+
+  def input
+    get_and_parse_posfile
+    log_files = get_logfile_list
+    get_logfile(log_files)
+    put_posfile
+  end
+
+  def has_iam_role?
+    begin
+      ec2 = Aws::EC2::Client.new(region: @region)
+      !ec2.config.credentials.nil?
+    rescue => e
+      $log.warn "EC2 Client error occurred: #{e.message}"
+    end
+  end
+
+  def get_and_parse_posfile
+    begin
+      # get & parse pos file
+      $log.debug "pos file get start"
+
+      pos_last_written_timestamp = 0
+      pos_info = {}
+      pos_log_hash = {}
+      File.open(@pos_file, File::RDONLY) do |file|
+        file.each_line do |line|
+
+          pos_match = /^timestamp: (\d+)$/.match(line)
+          if pos_match
+            pos_last_written_timestamp = pos_match[1].to_i
+            $log.debug "pos_last_written_timestamp: #{pos_last_written_timestamp}"
+          end
+
+          pos_match = /^(.+)\t(.+)\t(.+)$/.match(line)
+          if pos_match
+            pos_info[pos_match[1]] = pos_match[2]
+            pos_log_hash[pos_match[1]] = pos_match[3]
+            $log.debug "log_file: #{pos_match[1]}, marker: #{pos_match[2]}, hash: #{pos_match[3]}"
+          end
+        end
+        @pos_last_written_timestamp = pos_last_written_timestamp
+        @pos_info = pos_info
+        @pos_log_hash = pos_log_hash
+      end
+    rescue => e
+      $log.warn "pos file get and parse error occurred: #{e.message}"
+    end
+  end
+
+  def put_posfile
+    # pos file write
+    @pos_log_hash = @current_pos_log_hash
+    begin
+      $log.debug "pos file write"
+      File.open(@pos_file, File::WRONLY|File::TRUNC) do |file|
+        file.puts "timestamp: #{@pos_last_written_timestamp.to_s}"
+
+        @pos_info.each do |log_file_name, marker|
+          file.puts "#{log_file_name}\t#{marker}\t#{@pos_log_hash[log_file_name]}"
+        end
+      end
+    rescue => e
+      $log.warn "pos file write error occurred: #{e.message}"
+    end
+  end
+
+  def get_logfile_list
+    begin
+      $log.debug "get logfile-list from rds: db_instance_identifier=#{@db_instance_identifier}, pos_last_written_timestamp=#{@pos_last_written_timestamp}"
+      log_files = @rds.describe_db_log_files(
+        db_instance_identifier: @db_instance_identifier,
+        file_last_written: @pos_last_written_timestamp,
+        max_records: 10,
+      )
+
+      @current_pos_log_hash = {}
+      log_files.each do |log_file|
+        log_file.describe_db_log_files.each do |item|
+          log_file_name = item[:log_file_name]
+          @current_pos_log_hash[log_file_name] = item.hash
+        end
+      end
+
+      log_files
+    rescue => e
+      $log.warn "RDS Client describe_db_log_files error occurred: #{e.message}"
+    end
+  end
+
+  def rotated?(log_file)
+      utc_hour = (Time.now - 3600).utc.hour
+      hash_target = ""
+      case log_file
+      when "error/mysql-error.log" then
+        hash_target = "error/mysql-error-running.log"
+      else
+        hash_target = "#{log_file}.#{utc_hour}"
+      end
+      return @current_pos_log_hash[hash_target] && @pos_log_hash[hash_target] != @current_pos_log_hash[hash_target]
+  end
+
+  def get_logfile(log_files)
+    begin
+      log_files.each do |log_file|
+        log_file.describe_db_log_files.each do |item|
+          # save maximum written timestamp value
+          @pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
+
+          # log file download
+          log_file_name = item[:log_file_name]
+          next if log_file_name[%r{error/mysql-error-running.log}] || log_file_name[%r{slowquery/mysql-slowquery.log\.}] || log_file_name[%r{general/mysql-general.log\.}]
+          marker = @pos_info.has_key?(log_file_name) ? @pos_info[log_file_name] : "0"
+          marker = "0" if rotated?(log_file_name)
+
+          $log.debug "download log from rds: log_file_name=#{log_file_name}, marker=#{marker}"
+          logs = @rds.download_db_log_file_portion(
+            db_instance_identifier: @db_instance_identifier,
+            log_file_name: log_file_name,
+            marker: marker,
+          )
+          raw_records = get_logdata(logs)
+
+          #emit
+          parse_and_emit(raw_records, log_file_name) unless raw_records.nil?
+        end
+      end
+    rescue => e
+      $log.warn e.message
+    end
+  end
+
+  def get_logdata(logs)
+    log_file_name = logs.context.params[:log_file_name]
+    raw_records = []
+    begin
+      logs.each do |log|
+        # save got line's marker
+        @pos_info[log_file_name] = log.marker
+
+        raw_records += log.log_file_data.split("\n")
+      end
+    rescue => e
+      $log.warn e.message
+    end
+    return raw_records
+  end
+
+  def parse_and_emit(raw_records, log_file_name)
+    begin
+      $log.debug "raw_records.count: #{raw_records.count}"
+
+      record = {
+        "db_instance_identifier" => @db_instance_identifier,
+        "region" => @region,
+        "log_file_name" => log_file_name,
+      }
+      output_tag = @tag
+      output_tag += ".#{log_file_name.gsub(/\/|\./, '_')}" if @output_per_file
+
+      if log_file_name != "slowquery/mysql-slowquery.log"
+        raw_records.each do |raw_record|
+          $log.debug "raw_record=#{raw_record}"
+          line_match = LOG_REGEXP.match(raw_record)
+          next unless line_match
+
+          record["raw"] = raw_record if @raw_output
+          record["time"] = line_match[:time]
+          record["message"] = line_match[:message]
+          record["pid"] = line_match[:pid] if line_match[:pid]
+          record["message_level"] = line_match[:message_level] if line_match[:message_level]
+
+          router.emit(output_tag, Time.parse(line_match[:time] + ' +0000').to_i, record)
+        end
+      else
+        myslog = MySlog.new
+        myslog.divide(raw_records).each do |raw_record|
+          $log.debug "raw_record=#{raw_record}"
+          begin
+            raw = raw_record.join("\n") if @raw_output
+            record = record.merge(stringify_keys(myslog.parse_record(raw_record)))
+            record["raw"] = raw if @raw_output
+            if time = record.delete('date')
+              time = time.to_i
+            else
+              time = Time.now.to_i
+            end
+
+            router.emit(output_tag, time, record)
+          rescue => e
+            $log.warn e.message
+          end
+        end
+      end
+    rescue => e
+      $log.warn e.message
+    end
+  end
+
+  def stringify_keys(record)
+    result = {}
+    record.each_key do |key|
+      result[key.to_s] = record[key]
+    end
+    result
+  end
+
+  class TimerWatcher < Coolio::TimerWatcher
+    def initialize(interval, repeat, &callback)
+      require 'myslog'
+      @callback = callback
+      on_timer # first call
+      super(interval, repeat)
+    end
+
+    def on_timer
+      @callback.call
+    end
+  end
+end

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-rds-error-log
+version: !ruby/object:Gem::Version
+  version: 0.2.2
+platform: ruby
+authors:
+- Jen LaGrutta
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-06-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: myslog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+description: fluentd plugin for Amazon RDS for Error/Audit log input
+email:
+- jen@tune.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- fluent-plugin-rds-error-log.gemspec
+- fluent.conf.sample
+- lib/fluent/plugin/in_rds_error_log.rb
+homepage: https://github.com/hasjenl/fluent-plugin-rds-error-log
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Amazon RDS for Error/Audit log input plugin
+test_files: []