fluent-plugin-proxysql-query-log 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d4c903553f7ca67a822ef78e620e680c2cfbec12f9978f5a47699e7b15a7dcef
-  data.tar.gz: 3215c368881952fa7cfb90cd86ef98638f24f91244d671363a5dcc398aa377ff
+  metadata.gz: 93568be0e3dbf36c2d55d5413d72a6583854043f8c3f0c9eab69a75891d04fdc
+  data.tar.gz: 33971a068177dc8532636d36ada3b4cc906ed512115970c32c715cf14d71911f
 SHA512:
-  metadata.gz: 9fd09286e67b625845cbbd86e22476a8b05b06d22d4b7a617ede115d493838b6c2d1a062dd098debcdef3036c76155ff23b212f1d2d660ebfbbe5efe8b5665a9
-  data.tar.gz: 552cc4c68d8bbe43baf3c9c0cdf2084ea3b469745e38f8a66e1558560523d092d41dbeadbbfc3d6686ae8f8e8fa7c6df43924ca594da2292599b10a84f5b2d36
+  metadata.gz: e856303f5d1c291e1c44ea303bb81f6cc74b8bc056419fbe088386df0ec18a0df92c6bff7a93f4f21a62c83adf701e89213b1a80d9d9e27c90a7aa280010fb0b
+  data.tar.gz: 41f8339b5a393c328c504db9c8c6548f23760d3be9f6d73ab9135e50cc39d7827528406495615659b880296f4383387bc9be002d8d0f9c30d044a1e64dbb33f4

data/.gitignore

@@ -1,2 +1,3 @@
 vendor
 .bundle/config
+test/tmp/proxysql_query_log/query_log*

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-proxysql-query-log (0.1.0)
+    fluent-plugin-proxysql-query-log (0.2.0)
       cool.io
       fluentd (= 0.14.16)
       proxysql_query_log-parser

data/fluent-plugin-proxysql-query-log.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-proxysql-query-log"
-  spec.version = "0.1.0"
+  spec.version = "0.2.0"
   spec.authors = ["r_takaishi"]
   spec.email   = ["ryo.takaishi.0@gmail.com"]
 

data/lib/fluent/plugin/in_proxysql_query_log.rb

@@ -25,7 +25,7 @@ module Fluent
     class ProxysqlQueryLogInput < Fluent::Plugin::Input
       Fluent::Plugin.register_input("proxysql_query_log", self)
 
-      helpers :event_loop, :storage
+      helpers :event_loop, :storage, :timer
 
       DEFAULT_STORAGE_TYPE = 'local'
 
@@ -33,6 +33,7 @@ module Fluent
       config_param :read_from_head, :bool, default: false
       desc 'The paths to exclude the files from watcher list.'
       config_param :exclude_path, :array, default: []
+      config_param :refresh_interval, :time, default: 60
       config_param :tag, :string
 
       config_section :storage do
@@ -56,7 +57,13 @@ module Fluent
       def start
         super
 
+        refresh_watchers
+        timer_execute(:in_proxysql_query_log_refresh_watchers, @refresh_interval, &method(:refresh_watchers))
+      end
+
+      def refresh_watchers
         target_paths = expand_paths
+        stop_watchers(target_paths)
         start_watchers(target_paths)
       end
 
@@ -64,10 +71,14 @@ module Fluent
 
         paths.each do |path|
           log.debug("start watch: #{path}")
-          w = Watcher.new(path, 0, @pos_storage, router, @tag, log)
-          event_loop_attach(w)
+          @watchers[path] = Watcher.new(path, 0, @pos_storage, router, @tag, log)
+        end
+      end
 
-          @watchers[path] = w
+      def stop_watchers(paths)
+        paths.each do |path|
+          w = @watchers.delete(path) if w
+          w.detach if w
         end
       end
 

data/lib/fluent/plugin/in_proxysql_query_log/watcher.rb

@@ -10,6 +10,7 @@ module Fluent
           @router = router
           @tag = tag
           @log = log
+          @attached = false
           read
         end
 
@@ -60,6 +61,19 @@ module Fluent
         def convert_time(t)
           Time.at(t/1000/1000).utc.strftime('%Y-%m-%d %H:%M:%S')
         end
+
+        def attach(loop)
+          @attached = true
+          super
+        end
+
+        def detach
+          @attached = false
+          super
+        end
+        def attached?
+          @attached
+        end
       end
     end
   end

data/test/plugin/test_in_proxysql_query_log.rb

@@ -4,27 +4,62 @@ require "fluent/plugin/in_proxysql_query_log.rb"
 class ProxysqlQueryLogInputTest < Test::Unit::TestCase
   setup do
     Fluent::Test.setup
+    Pathname.glob("#{TMP_DIR}/*").each{|p| File.delete(p)}
   end
 
   TMP_DIR = File.dirname(__FILE__) + '/../tmp/proxysql_query_log'
   CONFIG = config_element('ROOT', '', {
-      'path' => "#{TMP_DIR}/query_log",
+      'path' => "#{TMP_DIR}/query_log.00000001",
       'tag' => 't1',
+      'refresh_interval' => 2
   })
 
+  MULTI_FILE_CONFIG = config_element('ROOT', '', {
+      'path' => "#{TMP_DIR}/query_log*",
+      'tag' => 't1',
+      'refresh_interval' => 2
+  })
+
+  QUERY_1 = {
+      thread_id: 9,
+      username: 'root',
+      schema_name: 'alpaca',
+      client: '127.0.0.1:34612',
+      hid: 0,
+      server: '127.0.0.1:3306',
+      start_time: 1525944256367381,
+      end_time: 1525944256367837,
+      digest: '0xD69C6B36F32D2EAE',
+      query: 'SELECT * FROM test'
+  }
+
+  QUERY_2 = {
+      thread_id: 9,
+      username: 'root',
+      schema_name: 'alpaca',
+      client: '127.0.0.1:34612',
+      hid: 0,
+      server: '127.0.0.1:3306',
+      start_time: 1525944256367381,
+      end_time: 1525944256367837,
+      digest: '0xD69C6B36F32D2EAE',
+      query: 'show databases'
+  }
+
   test 'singlefile' do
-    File.open("#{TMP_DIR}/query_log", 'wb') {|f|
-      write_record(f)
+    File.open("#{TMP_DIR}/query_log.00000001", 'wb') {|f|
+      write_record(f, QUERY_1)
     }
     config = CONFIG
     d = create_driver(config)
     d.run(expect_emits: 1) do
-      File.open("#{TMP_DIR}/query_log", "ab") {|f|
-        write_record(f)
+      File.open("#{TMP_DIR}/query_log.00000001", "ab") {|f|
+        write_record(f, QUERY_1)
       }
     end
 
     events = d.events
+    assert_equal(1, d.instance.instance_variable_get('@watchers').size)
     assert_equal(true, events.length > 0)
     assert_equal(1, 1)
     assert_equal(9, events[0][2]['thread_id'])
@@ -39,20 +74,116 @@ class ProxysqlQueryLogInputTest < Test::Unit::TestCase
     assert_equal('SELECT * FROM test', events[0][2]['query'])
   end
 
+  test 'multifile' do
+    File.open("#{TMP_DIR}/query_log.00000001", 'wb') {|f|
+      write_record(f, QUERY_1)
+    }
+    File.open("#{TMP_DIR}/query_log.00000002", 'wb') {|f|
+      write_record(f, QUERY_2)
+    }
+
+    config = MULTI_FILE_CONFIG
+    d = create_driver(config)
+    d.run(expect_emits: 2) do
+      File.open("#{TMP_DIR}/query_log.00000001", "ab") {|f|
+        write_record(f, QUERY_1)
+      }
+      File.open("#{TMP_DIR}/query_log.00000002", "ab") {|f|
+        write_record(f, QUERY_2)
+      }
+    end
+
+    events = d.events
+    assert_equal(2, d.instance.instance_variable_get('@watchers').size)
+    assert_equal(true, events.length > 0)
+    assert_equal(1, 1)
+    assert_equal(9, events[0][2]['thread_id'])
+    assert_equal('root', events[0][2]['username'])
+    assert_equal('alpaca', events[0][2]['schema_name'])
+    assert_equal('127.0.0.1:34612', events[0][2]['client'])
+    assert_equal(0, events[0][2]['HID'])
+    assert_equal('127.0.0.1:3306', events[0][2]['server'])
+    assert_equal('2018-05-10 09:24:16', events[0][2]['start_time'])
+    assert_equal('2018-05-10 09:24:16', events[0][2]['end_time'])
+    assert_equal('0xD69C6B36F32D2EAE', events[0][2]['digest'])
+    assert_equal('SELECT * FROM test', events[0][2]['query'])
+
+    assert_equal(2, d.instance.instance_variable_get('@watchers').size)
+    assert_equal(true, events.length > 0)
+    assert_equal(1, 1)
+    assert_equal(9, events[1][2]['thread_id'])
+    assert_equal('root', events[1][2]['username'])
+    assert_equal('alpaca', events[1][2]['schema_name'])
+    assert_equal('127.0.0.1:34612', events[1][2]['client'])
+    assert_equal(0, events[1][2]['HID'])
+    assert_equal('127.0.0.1:3306', events[1][2]['server'])
+    assert_equal('2018-05-10 09:24:16', events[1][2]['start_time'])
+    assert_equal('2018-05-10 09:24:16', events[1][2]['end_time'])
+    assert_equal('0xD69C6B36F32D2EAE', events[1][2]['digest'])
+    assert_equal('show databases', events[1][2]['query'])
+  end
+
+  test 'rotate_file' do
+    File.open("#{TMP_DIR}/query_log.00000001", 'wb') {|f|
+      write_record(f, QUERY_1)
+    }
+
+    config = MULTI_FILE_CONFIG
+    d = create_driver(config)
+    d.run(expect_emits: 2) do
+      File.open("#{TMP_DIR}/query_log.00000002", "ab") {|f|
+        write_record(f, QUERY_2)
+      }
+      sleep 5
+
+      File.open("#{TMP_DIR}/query_log.00000002", "ab") {|f|
+        write_record(f, QUERY_2)
+      }
+    end
+
+    events = d.events
+
+    assert_equal(2, d.instance.instance_variable_get('@watchers').size)
+    assert_equal(true, events.length > 0)
+
+    assert_equal(9, events[0][2]['thread_id'])
+    assert_equal('root', events[0][2]['username'])
+    assert_equal('alpaca', events[0][2]['schema_name'])
+    assert_equal('127.0.0.1:34612', events[0][2]['client'])
+    assert_equal(0, events[0][2]['HID'])
+    assert_equal('127.0.0.1:3306', events[0][2]['server'])
+    assert_equal('2018-05-10 09:24:16', events[0][2]['start_time'])
+    assert_equal('2018-05-10 09:24:16', events[0][2]['end_time'])
+    assert_equal('0xD69C6B36F32D2EAE', events[0][2]['digest'])
+    assert_equal('SELECT * FROM test', events[0][2]['query'])
+
+    assert_equal(9, events[1][2]['thread_id'])
+    assert_equal('root', events[1][2]['username'])
+    assert_equal('alpaca', events[1][2]['schema_name'])
+    assert_equal('127.0.0.1:34612', events[1][2]['client'])
+    assert_equal(0, events[1][2]['HID'])
+    assert_equal('127.0.0.1:3306', events[1][2]['server'])
+    assert_equal('2018-05-10 09:24:16', events[1][2]['start_time'])
+    assert_equal('2018-05-10 09:24:16', events[1][2]['end_time'])
+    assert_equal('0xD69C6B36F32D2EAE', events[1][2]['digest'])
+    assert_equal('show databases', events[1][2]['query'])
+
+  end
+
   private
 
-  def write_record(f)
+  def write_record(f, param)
     q = ProxysqlQueryLog::Query.new
-    q.thread_id = 9
-    q.username = 'root'
-    q.schema_name = 'alpaca'
-    q.client = '127.0.0.1:34612'
-    q.hid = 0
-    q.server = '127.0.0.1:3306'
-    q.start_time = 1525944256367381
-    q.end_time = 1525944256367837
-    q.digest = '0xD69C6B36F32D2EAE'
-    q.query = 'SELECT * FROM test'
+    q.thread_id = param[:thread_id]
+    q.username = param[:username]
+    q.schema_name = param[:schema_name]
+    q.client = param[:client]
+    q.hid = param[:hid]
+    q.server = param[:server]
+    q.start_time = param[:start_time]
+    q.end_time = param[:end_time]
+    q.digest = param[:digest]
+    q.query = param[:query]
 
     f.write([total_length(q), 0, 0, 0, 0, 0, 0, 0].pack('C*'))
     f.write(to_binary(q))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-proxysql-query-log
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - r_takaishi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-29 00:00:00.000000000 Z
+date: 2018-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler