fluent-plugin-rds-pgsql-log 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 05d2d7d1b6a3b106433747c8e6229f63bf602529
-  data.tar.gz: aad9c859045a94d59d22b93aad98140a8ea8ca5f
+  metadata.gz: fab744e32ec93e5bef3dea688e69b125f07469e7
+  data.tar.gz: 5aa94105f7c5f92d986aab90d929505fb8ee1d1a
 SHA512:
-  metadata.gz: 4508cfa98b617ff1709986a0986f0235f2b3cf40db3d283410b21317ad9d7994f6adb91c7bc155c51042d6026def2b5df9ff67a0df0f342b77e38d231ee1c2ff
-  data.tar.gz: b8a3159094a83aff08f37150331d229b41b2df534a5e95bc430894459752888d914424be9a80a18e0cf62f0e72758e8fe8ca9f9a04610230634e1c38b7759179
+  metadata.gz: 28b7c0f858271e41ae47d07a94e33bd0be57f5dfa04d9d96db2185f612680280d803f7593ac534aa29ce8cf7574d8cee5a4ae157a2321ad73fe5557e9cadf8ad
+  data.tar.gz: 262bdd80dcf2f10924c23d69d91bbe271b3692983bf943d2045accb6e65098893a64ae5564b4701816862c8fc8cdad04d15c9032ed9356119424f2201ca9a864

data/README.md

@@ -76,3 +76,5 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
 ### 0.2.0
 - require fluentd version 0.14.0
 
+### 0.3.0
+- use AWS SDK v3.x

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require "rake/testtask"
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-rds-pgsql-log.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-rds-pgsql-log"
-  spec.version       = "0.3.0"
+  spec.version       = "0.3.1"
   spec.authors       = ["shinsaka"]
   spec.email         = ["shinx1265@gmail.com"]
   spec.summary       = "Amazon RDS for PostgreSQL log input plugin"
@@ -20,4 +20,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "aws-sdk", "~> 3"
 
   spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 12"
+  spec.add_development_dependency "test-unit", "~> 3.2"
+  spec.add_development_dependency "webmock", "~>3"
+  spec.add_development_dependency "simplecov", "~>0"
 end

data/lib/fluent/plugin/in_rds_pgsql_log.rb

@@ -44,7 +44,7 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
       end
       @rds = Aws::RDS::Client.new(options)
     rescue => e
-      $log.warn "RDS Client error occurred: #{e.message}"
+      log.warn "RDS Client error occurred: #{e.message}"
     end
 
     @loop = Coolio::Loop.new
@@ -77,14 +77,14 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
       ec2 = Aws::EC2::Client.new(region: @region)
       !ec2.config.credentials.nil?
     rescue => e
-      $log.warn "EC2 Client error occurred: #{e.message}"
+      log.warn "EC2 Client error occurred: #{e.message}"
     end
   end
 
   def get_and_parse_posfile
     begin
       # get & parse pos file
-      $log.debug "pos file get start"
+      log.debug "pos file get start"
 
       pos_last_written_timestamp = 0
       pos_info = {}
@@ -94,27 +94,27 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
           pos_match = /^(\d+)$/.match(line)
           if pos_match
             pos_last_written_timestamp = pos_match[1].to_i
-            $log.debug "pos_last_written_timestamp: #{pos_last_written_timestamp}"
+            log.debug "pos_last_written_timestamp: #{pos_last_written_timestamp}"
           end
 
           pos_match = /^(.+)\t(.+)$/.match(line)
           if pos_match
             pos_info[pos_match[1]] = pos_match[2]
-            $log.debug "log_file: #{pos_match[1]}, marker: #{pos_match[2]}"
+            log.debug "log_file: #{pos_match[1]}, marker: #{pos_match[2]}"
           end
         end
         @pos_last_written_timestamp = pos_last_written_timestamp
         @pos_info = pos_info
       end
     rescue => e
-      $log.warn "pos file get and parse error occurred: #{e.message}"
+      log.warn "pos file get and parse error occurred: #{e.message}"
     end
   end
 
   def put_posfile
     # pos file write
     begin
-      $log.debug "pos file write"
+      log.debug "pos file write"
       File.open(@pos_file, File::WRONLY|File::TRUNC) do |file|
         file.puts @pos_last_written_timestamp.to_s
 
@@ -123,20 +123,20 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
         end
       end
     rescue => e
-      $log.warn "pos file write error occurred: #{e.message}"
+      log.warn "pos file write error occurred: #{e.message}"
     end
   end
 
   def get_logfile_list
     begin
-      $log.debug "get logfile-list from rds: db_instance_identifier=#{@db_instance_identifier}, pos_last_written_timestamp=#{@pos_last_written_timestamp}"
+      log.debug "get logfile-list from rds: db_instance_identifier=#{@db_instance_identifier}, pos_last_written_timestamp=#{@pos_last_written_timestamp}"
       @rds.describe_db_log_files(
         db_instance_identifier: @db_instance_identifier,
         file_last_written: @pos_last_written_timestamp,
         max_records: 10,
       )
     rescue => e
-      $log.warn "RDS Client describe_db_log_files error occurred: #{e.message}"
+      log.warn "RDS Client describe_db_log_files error occurred: #{e.message}"
     end
   end
 
@@ -151,7 +151,7 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
           log_file_name = item[:log_file_name]
           marker = @pos_info.has_key?(log_file_name) ? @pos_info[log_file_name] : "0"
 
-          $log.debug "download log from rds: log_file_name=#{log_file_name}, marker=#{marker}"
+          log.debug "download log from rds: log_file_name=#{log_file_name}, marker=#{marker}"
           logs = @rds.download_db_log_file_portion(
             db_instance_identifier: @db_instance_identifier,
             log_file_name: log_file_name,
@@ -164,7 +164,7 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
         end
       end
     rescue => e
-      $log.warn e.message
+      log.warn e.message
     end
   end
 
@@ -179,17 +179,17 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
         raw_records += log.log_file_data.split("\n")
       end
     rescue => e
-      $log.warn e.message
+      log.warn e.message
     end
     return raw_records
   end
 
   def parse_and_emit(raw_records, log_file_name)
     begin
-      $log.debug "raw_records.count: #{raw_records.count}"
+      log.debug "raw_records.count: #{raw_records.count}"
       record = nil
       raw_records.each do |raw_record|
-        $log.debug "raw_record=#{raw_record}"
+        log.debug "raw_record=#{raw_record}"
         line_match = LOG_REGEXP.match(raw_record)
 
         unless line_match
@@ -215,7 +215,7 @@ class Fluent::Plugin::RdsPgsqlLogInput < Fluent::Plugin::Input
       # emit last record
       router.emit(@tag, Fluent::Engine.now, record) unless record.nil?
     rescue => e
-      $log.warn e.message
+      log.warn e.message
     end
   end
 

data/test/helper.rb

@@ -0,0 +1,20 @@
+require 'bundler/setup'
+require 'test/unit'
+require 'webmock/test_unit'
+require 'simplecov'
+
+SimpleCov.start do
+  add_filter '/test/'
+end
+
+$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
+$LOAD_PATH.unshift(__dir__)
+require 'fluent/test'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_rds_pgsql_log'
+
+class Test::Unit::TestCase
+  include Fluent::Test::Helpers
+  extend Fluent::Test::Helpers
+end

data/test/plugin/test_in_rds_pgsql.rb

@@ -0,0 +1,51 @@
+require_relative '../helper'
+
+class RdsPgsqlLogInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  DEFAULT_CONFIG = {
+    access_key_id: 'dummy_access_key_id',
+    secret_access_key: 'dummy_secret_access_key',
+    region: 'ap-northeast-1',
+    db_instance_identifier: 'test-postgres-id',
+    refresh_interval: 30,
+    pos_file: 'pgsql-log-pos.dat',
+  }
+
+  def parse_config(conf = {})
+    ''.tap{|s| conf.each { |k, v| s << "#{k} #{v}\n" } }
+  end
+
+  def create_driver(conf = DEFAULT_CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::RdsPgsqlLogInput).configure(parse_config conf)
+  end
+
+  def iam_info_url
+    'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
+  end
+
+  def use_iam_role
+    stub_request(:get, iam_info_url)
+      .to_return(status: [200, 'OK'], body: "hostname")
+    stub_request(:get, "#{iam_info_url}hostname")
+      .to_return(status: [200, 'OK'],
+                 body: {
+                   "AccessKeyId" => "dummy",
+                   "SecretAccessKey" => "secret",
+                   "Token" => "token"
+                 }.to_json)
+  end
+
+  def test_configure
+    use_iam_role
+    d = create_driver
+    assert_equal 'dummy_access_key_id', d.instance.access_key_id
+    assert_equal 'dummy_secret_access_key', d.instance.secret_access_key
+    assert_equal 'ap-northeast-1', d.instance.region
+    assert_equal 'test-postgres-id', d.instance.db_instance_identifier
+    assert_equal 'pgsql-log-pos.dat', d.instance.pos_file
+    assert_equal 30, d.instance.refresh_interval
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rds-pgsql-log
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - shinsaka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-27 00:00:00.000000000 Z
+date: 2018-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -58,6 +58,62 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
 description: fluentd plugin for Amazon RDS for PostgreSQL log input
 email:
 - shinx1265@gmail.com
@@ -69,9 +125,12 @@ files:
 - Gemfile
 - LICENSE.txt
 - README.md
+- Rakefile
 - fluent-plugin-rds-pgsql-log.gemspec
 - fluent.conf.sample
 - lib/fluent/plugin/in_rds_pgsql_log.rb
+- test/helper.rb
+- test/plugin/test_in_rds_pgsql.rb
 homepage: https://github.com/shinsaka/fluent-plugin-rds-pgsql-log
 licenses:
 - MIT
@@ -96,4 +155,6 @@ rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Amazon RDS for PostgreSQL log input plugin
-test_files: []
+test_files:
+- test/helper.rb
+- test/plugin/test_in_rds_pgsql.rb