fluent-plugin-aurora-slowquerylog 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5dd1670153a78f65af70f9303aa318c9a8fdbc78
-  data.tar.gz: fe100a3287322ea38a4e059cf3022a2a0795e564
+  metadata.gz: 902711f56bfb142c7fe24ce7a5affb1315d54009
+  data.tar.gz: 15a4881d9f91ac27806da775e7e156085a510be4
 SHA512:
-  metadata.gz: ab47e4561a21ae79196fc5528b14f3dfcb6f91fac533c593700dddcc162c1e96d97cec7a1dd003b527591cfb9d7028203e0e7d27cf6cfeb45f1a8ede9229b999
-  data.tar.gz: 6754db30aae7c4ab901888ff4f5d49b1948f09d690b0f084cb977261d2e24c5be6afb67921f02e654e9a742a6b9ab8a0809b13eb8ab4b01866792b98a455157d
+  metadata.gz: 760664ecf893dea66deb99952b687846a3502898c13a7318005426b17c9b16e05f4dd36606fba47774084c31e2ac89011f37d9ca706d92135132b56401ac2edc
+  data.tar.gz: 91b63cebb52470b87aae79cbb3985815a3a752a666e45191c889945abd8c8945fd88d433e0669e5e9c8f0e6e294657effe53856c6ec17115ac604c378a7e408f

data/README.md

@@ -1,28 +1,66 @@
-# Fluent::Plugin::Aurora::Slowquerylog
+# [Fluentd](http://fluentd.org) input plugin for AWS RDS Aurora
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/fluent/plugin/aurora/slowquerylog`. To experiment with that code, run `bin/console` for an interactive prompt.
+## Overview
+This fluent input plugin collects RDS Aurora slowquery log with ParameterGroup option `log_output=FILE`.
+This plugin fetches only the difference between the latest and previous fetched slow query log.
 
-TODO: Delete this and the text above, and describe your gem
+## Background
+There are a lot of RDS mysql slowlog input plugins that collect mysql slowquery logs with ParameterGroup option `log_output=TABLE`.
+However if you use RDS Aurora with the option, there are two following problems:
 
-## Installation
+### 1. mysql.slow_log table is operated with Engine=CSV
 
-Add this line to your application's Gemfile:
+mysql.slow_log table is operated with Engine=CSV and we cannot add index to the table.
+So if tons of slow queries are registered to mysql.slow_log, your `SELECT * FROM mysql.slow_log WHERE condition` queries also become slow.
 
-```ruby
-gem 'fluent-plugin-aurora-slowquerylog'
+### 2. No way to rotate Aurora slowlog table
+
+mysql.slow_log table on RDS mysql can be purged by using `CALL mysql.rds_rotate_slow_log`. On the other hand, the stored procedure executed on Aurora makes following error.
+
+```
+> CALL mysql.rds_rotate_slow_log;
+ERROR 1289 (HY000): The 'CSV' feature is disabled; you need MySQL built with 'CSV' to have it working
 ```
 
-And then execute:
+According to the above reasons, I have implemented input plugin with `log_output=FILE` for Aurora.
 
-    $ bundle
+## Installation
 
-Or install it yourself as:
+```ruby
+$ fluent-gem install fluent-plugin-aurora-slowquerylog --no-document
+```
 
-    $ gem install fluent-plugin-aurora-slowquerylog
+## Input: How It Works
+TBW
 
 ## Usage
+### Sample configuration
+
+```
+<source>
+  @type aurora_slowquerylog
+  tag  aurora.slowlog
+  db_instance_identifier  aurora_node_id
+  region  us-east-1
+  log_file_name  slowquery/mysql-slowquery.log
+  aurora_state_file  /var/run/fluentd/aurora_state
+  log_fetch_interval  30 #optionnal
+  aws_access_key_id  your_aws_access_key_id #optionnal
+  aws_secret_access_key  your_aws_secret_access_key #optionnal
+  filename_contains mysql-slowquery.log # default 'mysql-slowquery.log'
+</source>
+```
+
+* **tag** tag name of events
+* **db_instance_identifier** AWS Aurora node id
+* **region** AWS region name
+* **log_file_name** RDS slowlog name. Currently we cannot change file name from 'slowquery/mysql-slowquery.log'
+* **aurora_state_file** state file that keeps maker information and current & last slowquery log name
+* **log_fetch_interval** interval time(second) for log fetch (optional)
+* **aws_access_key_id** AWS access key id. For AWS user IAM instance profile is recommended without using this option (optional)
+* **aws_secret_access_key** AWS secret access key. For AWS user IAM instance profile is recommended without using this option (optional)
+* **filename_contains** filter condition for fetching slow query log (optional)
 
-TODO: Write usage instructions here
 
 ## Development
 

data/fluent-plugin-aurora-slowquerylog.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-aurora-slowquerylog"
-  spec.version       = "0.0.3"
+  spec.version       = "0.0.4"
   spec.authors       = ["Takayuki WATANABE"]
   spec.email         = ["takanabe.w@gmail.com"]
 

data/lib/fluent/plugin/in_aurora_slowquerylog.rb

@@ -86,17 +86,15 @@ module Fluent
         state = load_state
         if slowlog_rotated?(state)
           if state["additional_data_pending"]
-            log_file_name = "#{@log_file_name}.#{Time.now.utc.strftime("%Y-%m-%d.%H")}"
-            fetch_and_emit_log(log_file_name, state["marker"],true)
+            fetch_and_emit_log(@previous_slowlog, state["marker"],true)
           else
-            log_file_name = "#{@log_file_name}.#{Time.now.utc.strftime("%Y-%m-%d.%H")}"
-            fetch_and_emit_log(@log_file_name, state["marker"],false)
+            fetch_and_emit_log(@previous_slowlog, state["marker"],false)
           end
         else
-          fetch_and_emit_log(@log_file_name, state["marker"],true)
+          fetch_and_emit_log(@current_slowlog, state["marker"],true)
         end
       else
-        fetch_and_emit_log(@log_file_name, false, true)
+        fetch_and_emit_log(@current_slowlog, false, true)
       end
     end
 
@@ -113,7 +111,7 @@ module Fluent
         else
           fetched_file = @rds_client.download_db_log_file_portion(
             db_instance_identifier: @db_instance_identifier,
-            log_file_name: "slowquery/mysql-slowquery.log")
+            log_file_name: log_file_name)
           save_state(fetched_file)
           records = parse_fetched_file(fetched_file)
           emit_slowlogs(records)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-aurora-slowquerylog
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Takayuki WATANABE
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-02 00:00:00.000000000 Z
+date: 2016-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd