fluent-plugin-rds-pgsql-log 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 490d5c629a6533d73d1c2a85ad7ccf8962ac6d30
+  data.tar.gz: e892a87af285abe6a941ce5387f52507bb9569eb
+SHA512:
+  metadata.gz: 04b299e27b14de7411e573cdfc3d4d16579a3ee069e1afaab07e04ff2e7e2819fdaf35a0d1edf4d8048d277211bc50f5125e519e39582a559c39b630d38c6078
+  data.tar.gz: 1bbd9858e9b20f54fcc8f93da6cb61be9ef987f7dfc8cf4c74b017567e540bb0934ddd3f476d1117788df05046741fda1db205ddd56d25cb871d56c382f1f9f0

data/.gitignore

@@ -0,0 +1,3 @@
+*.gem
+fluent.conf
+

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 shinsaka
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,72 @@
+# Amazon RDS for PostgreSQL log input plugin for fluentd
+
+## Overview
+- Amazon Web Services RDS log input plugin for fluentd
+
+## Installation
+
+    $ fluentd-gem fluent-plugin-rds-pgsql-log
+
+## AWS ELB Settings
+- settings see: [PostgreSQL Database Log Files](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_LogAccess.Concepts.PostgreSQL.html)
+
+## When SSL certification error
+log:
+```
+SSL_connect returned=1 errno=0 state=SSLv3 read server certificate B: certificate verify failed
+```
+Do env setting follows:
+```
+SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
+```
+
+## Configuration
+
+```config
+<source>
+  type rds_pgsql_log
+  # required
+  region                 <region name>
+  db_instance_identifier <instance identifier>
+  # optional if you can IAM credentials
+  access_key_id          <access_key>
+  secret_access_key      <secret_access_key>
+  # optional
+  refresh_interval       <interval number by second(default: 30)>
+  tag                    <tag name(default: rds-pgsql.log>
+  pos_file               <log getting position file(default: rds-pgsql.log)>
+</source>
+```
+
+### Example setting
+```config
+<source>
+  type rds_pgsql_log
+  region ap-northeast-1
+  db_instance_identifier test-postgres
+  access_key_id     XXXXXXXXXXXXXXXXXXXX
+  secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+  refresh_interval  30
+  tag pgsql.log
+  pos_file /tmp/pgsql-log-pos.dat
+</source>
+
+<match pgsql.log>
+  type stdout
+</match>
+```
+
+### json output example
+```
+
+{"time":"2015-05-30 02:19:22 UTC",
+ "host":"192.168.30.175(53092)",
+ "user":"testuser",
+ "database":"db1",
+ "pid":"9769",
+ "message_level":"LOG",
+ "message":"  statement: select 1;",
+ "log_file_name":"error/postgresql.log.2015-05-30-02"
+}
+```
+

data/fluent-plugin-rds-pgsql-log.gemspec

@@ -0,0 +1,23 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-rds-pgsql-log"
+  spec.version       = "0.1.0"
+  spec.authors       = ["shinsaka"]
+  spec.email         = ["shinx1265@gmail.com"]
+  spec.summary       = "Amazon RDS for PostgreSQL log input plugin"
+  spec.description   = "fluentd plugin for Amazon RDS for PostgreSQL log input"
+  spec.homepage      = "https://github.com/shinsaka/fluent-plugin-rds-pgsql-log"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "fluentd", "~> 0"
+  spec.add_dependency "aws-sdk", "~> 2"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+end

data/fluent.conf.sample

@@ -0,0 +1,14 @@
+<source>
+  type rds_pgsql_log
+  access_key_id     XXXXXXXXXXXXXXXXXXXX
+  secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+  region ap-northeast-1
+  db_instance_identifier test-postgres
+  refresh_interval 30
+  tag pgsql.log
+  pos_file /tmp/pgsql-log-pos.dat
+</source>
+
+<match pgsql.log>
+  type stdout
+</match>

data/lib/fluent/plugin/in_rds_pgsql_log.rb

@@ -0,0 +1,232 @@
+class Fluent::RdsPgsqlLogInput < Fluent::Input
+  Fluent::Plugin.register_input('rds_pgsql_log', self)
+
+  LOG_REGEXP = /^(?<time>\d{4}-\d{2}-\d{2} \d{2}\:\d{2}\:\d{2} .+?):(?<host>.*?):(?<user>.*?)@(?<database>.*?):\[(?<pid>.*?)\]:(?<message_level>.*?):(?<message>.*)$/
+
+  config_param :access_key_id, :string, :default => nil
+  config_param :secret_access_key, :string, :default => nil
+  config_param :region, :string, :default => nil
+  config_param :db_instance_identifier, :string, :default => nil
+  config_param :pos_file, :string, :default => "fluent-plugin-rds-pgsql-log-pos.dat"
+  config_param :refresh_interval, :integer, :default => 30
+  config_param :tag, :string, :default => "rds-pgsql.log"
+
+  def configure(conf)
+    super
+    require 'aws-sdk'
+
+    if !has_iam_role?
+      raise Fluent::ConfigError.new("access_key_id is required") if @access_key_id.nil?
+      raise Fluent::ConfigError.new("secret_access_key is required") if @secret_access_key.nil?
+    end
+    raise Fluent::ConfigError.new("region is required") unless @region
+    raise Fluent::ConfigError.new("db_instance_identifier is required") unless @db_instance_identifier
+    raise Fluent::ConfigError.new("pos_file is required") unless @pos_file
+    raise Fluent::ConfigError.new("refresh_interval is required") unless @refresh_interval
+    raise Fluent::ConfigError.new("tag is required") unless @tag
+  end
+
+  def start
+    super
+
+    # pos file touch
+    File.open(@pos_file, File::RDWR|File::CREAT).close
+
+    begin
+      options = {
+        :region => @region,
+      }
+      if @access_key_id && @secret_access_key
+        options[:access_key_id] = @access_key_id
+        options[:secret_access_key] = @secret_access_key
+      end
+      @rds = Aws::RDS::Client.new(options)
+    rescue => e
+      $log.warn "RDS Client error occurred: #{e.message}"
+    end
+
+    @loop = Coolio::Loop.new
+    timer_trigger = TimerWatcher.new(@refresh_interval, true, &method(:input))
+    timer_trigger.attach(@loop)
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    super
+    @loop.stop
+    @thread.join
+    @timestamp_file.close
+  end
+
+  private
+
+  def run
+    @loop.run
+  end
+
+  def input
+    get_and_parse_posfile
+    log_files = get_logfile_list
+    get_logfile(log_files)
+    put_posfile
+  end
+
+  def has_iam_role?
+    begin
+      ec2 = Aws::EC2::Client.new(region: @region)
+      !ec2.config.credentials.nil?
+    rescue => e
+      $log.warn "EC2 Client error occurred: #{e.message}"
+    end
+  end
+
+  def get_and_parse_posfile
+    begin
+      # get & parse pos file
+      $log.debug "pos file get start"
+
+      pos_last_written_timestamp = 0
+      pos_info = {}
+      File.open(@pos_file, File::RDONLY) do |file|
+        file.each_line do |line|
+
+          pos_match = /^(\d+)$/.match(line)
+          if pos_match
+            pos_last_written_timestamp = pos_match[1].to_i
+            $log.debug "pos_last_written_timestamp: #{pos_last_written_timestamp}"
+          end
+
+          pos_match = /^(.+)\t(.+)$/.match(line)
+          if pos_match
+            pos_info[pos_match[1]] = pos_match[2]
+            $log.debug "log_file: #{pos_match[1]}, marker: #{pos_match[2]}"
+          end
+        end
+        @pos_last_written_timestamp = pos_last_written_timestamp
+        @pos_info = pos_info
+      end
+    rescue => e
+      $log.warn "pos file get and parse error occurred: #{e.message}"
+    end
+  end
+
+  def put_posfile
+    # pos file write
+    begin
+      $log.debug "pos file write"
+      File.open(@pos_file, File::WRONLY|File::TRUNC) do |file|
+        file.puts @pos_last_written_timestamp.to_s
+
+        @pos_info.each do |log_file_name, marker|
+          file.puts "#{log_file_name}\t#{marker}"
+        end
+      end
+    rescue => e
+      $log.warn "pos file write error occurred: #{e.message}"
+    end
+  end
+
+  def get_logfile_list
+    begin
+      $log.debug "get logfile-list from rds: db_instance_identifier=#{@db_instance_identifier}, pos_last_written_timestamp=#{@pos_last_written_timestamp}"
+      @rds.describe_db_log_files(
+        db_instance_identifier: @db_instance_identifier,
+        file_last_written: @pos_last_written_timestamp,
+        max_records: 10,
+      )
+    rescue => e
+      $log.warn "RDS Client describe_db_log_files error occurred: #{e.message}"
+    end
+  end
+
+  def get_logfile(log_files)
+    begin
+      log_files.each do |log_file|
+        log_file.describe_db_log_files.each do |item|
+          # save maximum written timestamp value
+          @pos_last_written_timestamp = item[:last_written] if @pos_last_written_timestamp < item[:last_written]
+
+          # log file download
+          log_file_name = item[:log_file_name]
+          marker = @pos_info.has_key?(log_file_name) ? @pos_info[log_file_name] : "0"
+
+          $log.debug "download log from rds: log_file_name=#{log_file_name}, marker=#{marker}"
+          logs = @rds.download_db_log_file_portion(
+            db_instance_identifier: @db_instance_identifier,
+            log_file_name: log_file_name,
+            marker: marker,
+          )
+          raw_records = get_logdata(logs)
+
+          #emit
+          parse_and_emit(raw_records, log_file_name) unless raw_records.nil?
+        end
+      end
+    rescue => e
+      $log.warn e.message
+    end
+  end
+
+  def get_logdata(logs)
+    log_file_name = logs.context.params[:log_file_name]
+    raw_records = []
+    begin
+      logs.each do |log|
+        # save got line's marker
+        @pos_info[log_file_name] = log.marker
+
+        raw_records += log.log_file_data.split("\n")
+      end
+    rescue => e
+      $log.warn e.message
+    end
+    return raw_records
+  end
+
+  def parse_and_emit(raw_records, log_file_name)
+    begin
+      $log.debug "raw_records.count: #{raw_records.count}"
+      record = nil
+      raw_records.each do |raw_record|
+        $log.debug "raw_record=#{raw_record}"
+        line_match = LOG_REGEXP.match(raw_record)
+
+        unless line_match
+          # combine chain of log
+          record["message"] << "\n" + raw_record unless record.nil?
+        else
+          # emit before record
+          Fluent::Engine.emit(@tag, Fluent::Engine.now, record) unless record.nil?
+
+          # set a record
+          record = {
+            "time" => line_match[:time],
+            "host" => line_match[:host],
+            "user" => line_match[:user],
+            "database" => line_match[:database],
+            "pid" => line_match[:pid],
+            "message_level" => line_match[:message_level],
+            "message" => line_match[:message],
+            "log_file_name" => log_file_name,
+          }
+        end
+      end
+      # emit last record
+      Fluent::Engine.emit(@tag, Fluent::Engine.now, record) unless record.nil?
+    rescue => e
+      $log.warn e.message
+    end
+  end
+
+  class TimerWatcher < Coolio::TimerWatcher
+    def initialize(interval, repeat, &callback)
+      @callback = callback
+      on_timer # first call
+      super(interval, repeat)
+    end
+
+    def on_timer
+      @callback.call
+    end
+  end
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-rds-pgsql-log
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- shinsaka
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-05-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+description: fluentd plugin for Amazon RDS for PostgreSQL log input
+email:
+- shinx1265@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- fluent-plugin-rds-pgsql-log.gemspec
+- fluent.conf.sample
+- lib/fluent/plugin/in_rds_pgsql_log.rb
+homepage: https://github.com/shinsaka/fluent-plugin-rds-pgsql-log
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
+specification_version: 4
+summary: Amazon RDS for PostgreSQL log input plugin
+test_files: []