flydata 0.7.4 → 0.7.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35042672a68ea10f4ec090aa77f218c885aed555
-  data.tar.gz: 4223e306573350bb767eda1259e71ccc9d7d182a
+  metadata.gz: 44371bbacace581ba7bfc886a977202c84028ff9
+  data.tar.gz: bbac27c82da13ad6fce04dbbdc420feebc73a17c
 SHA512:
-  metadata.gz: 143b46f4b1fb7c7e7472932b668caeb66c2201f91bf2bc334553583cf03a6f7800b4b1d895e50174e6043c2328abcf8836b2dc7c2e1848457bf1e0993d3149d9
-  data.tar.gz: 22ec19871da8e8d1545cf019ec09f4113fddf5ed59da2fe5f285cbce2e616db8f5ecb9320169efefd4444d79594b1907b55a507d8674a9999e2255f7c2ee69ca
+  metadata.gz: 9b52b6990ae5706dfdb79a1a81a10666d09de149562dac3d2f9defb23383625c8c8b429d006c8d9a84978f1362cd74c81f05d0cd5c35b042164af32c06462059
+  data.tar.gz: bfbe4e1c24723bd23fa8650427fa3daed18d7c619ace48e3bcb1e050822a495c80e007d46989758fe6327f20e6a9b4a64f4712e4ab2b15bc44430db8b5fb6043

data/VERSION

@@ -1 +1 @@
-0.7.4
+0.7.5

data/lib/flydata/api/data_entry.rb

@@ -10,14 +10,152 @@ module Flydata
         super
       end
 
+# Response example (/buffer_stat.json for a Sync data entry
+# {
+#   "complete": true,
+#   "state": "complete",
+#   "stat": {
+#     "buffer": {
+#       "proxy": {
+#         "buffer_size": 0,
+#         "queue_length": 0
+#       },
+#       "redshift": {
+#         "buffer_size": 0,
+#         "queue_length": 0
+#       },
+#       "sync_redshift": {
+#         "buffer_size": 0,
+#         "queue_length": 0
+#       }
+#     },
+#     "queue": {
+#       "copy_queue": {
+#         "num_items": 0
+#       },
+#       "query_queue": {
+#         "num_items": 0
+#       }
+#     }
+#   },
+#   "message": "",
+#   "success": true
+# }
+
+      NUM_TABLES_PER_REQUEST = 150
+
       def buffer_stat(data_entry_id, options = {})
-        tables = options[:tables] ? options[:tables].join(',') : ''
-        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}", nil, {tables: tables})
+        table_array = options[:tables] || ['']
+        table_array = [''] if table_array.empty?
+        results = []
+        error = false
+        table_array.each_slice(NUM_TABLES_PER_REQUEST) do |ta|
+          tables = ta.join(',')
+          result = @client.post("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{options[:mode]}", nil, {tables: tables})
+          unless result["state"]
+            error = result["message"]
+            break
+          end
+          results << result
+        end
+        if error
+          # mimicking the error value of /buffer_stat API
+          {"complete"=> false,
+           "message" => error,
+           "success" => true,
+          }
+        else
+          stat = results.inject({"buffer"=>{},"queue"=>{}}) do |h, result|
+            %w[redshift sync_redshift].each do |fluent_process|
+              %w[buffer_size queue_length].each do |stat|
+                if result["stat"]["buffer"][fluent_process] && result["stat"]["buffer"][fluent_process][stat]
+                  h["buffer"][fluent_process] ||= {}
+                  h["buffer"][fluent_process][stat] ||= 0
+                  h["buffer"][fluent_process][stat] += result["stat"]["buffer"][fluent_process][stat]
+                end
+              end
+            end
+            %w[copy_queue query_queue].each do |queue|
+              if result["stat"]["queue"][queue] && result["stat"]["queue"][queue]["num_items"]
+                h["queue"][queue] ||= {}
+                h["queue"][queue]["num_items"] ||= 0
+                h["queue"][queue]["num_items"] += result["stat"]["queue"][queue]["num_items"]
+              end
+            end
+            h
+          end
+          if proxy = results.first["stat"]["buffer"]["proxy"]
+            # proxy values are not per-table
+            stat["buffer"]["proxy"] ||= {}
+            stat["buffer"]["proxy"]["buffer_size"] = proxy["buffer_size"]
+            stat["buffer"]["proxy"]["queue_length"] = proxy["queue_length"]
+          end
+          # TODO Refactor.  The logic below is copied from web data_entries_controller
+          buffer_empty = stat["buffer"].all?{|k,v| v["buffer_size"] == 0}
+          queue_empty = stat["queue"].all?{|k,v| v["num_items"] == 0}
+
+          complete = (buffer_empty && queue_empty)
+          state =  if complete
+                     'complete'
+                   elsif !buffer_empty
+                     'processing'
+                   else
+                     'uploading'
+                   end
+          message = results.count == 1 ? results.first["message"]
+                                       : build_message(stat, complete)
+
+          {
+            "complete"=>complete,
+            "state" => state,
+            "stat" => stat,
+            "message" => message,
+            "success" => true,
+          }
+        end
       end
 
+# Response example (/table_status.json)
+# {
+#   "table_status": [
+#     {
+#       "updated_at": 1462586673,
+#       "status": "ready",
+#       "table_name": "items",
+#       "seq": 0,
+#       "created_at": 1462586673,
+#       "src_pos": "-",
+#       "records_copied": "0",
+#       "num_items": 0,
+#       "next_item": false
+#     },
+#     {
+#       "updated_at": 1462586706,
+#       "status": "ready",
+#       "table_name": "users",
+#       "seq": 0,
+#       "created_at": 1462586674,
+#       "src_pos": "-",
+#       "records_copied": "0",
+#       "records_sent": "8",
+#       "num_items": 0,
+#         "next_item": false
+#     }
+#   ]
+# }
+
       def table_status(data_entry_id, options = {})
-        tables = options[:tables] ? options[:tables].join(',') : ''
-        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/table_status", nil, {tables: tables})
+        table_array = options[:tables] || ['']
+        results = []
+        table_array.each_slice(NUM_TABLES_PER_REQUEST) do |ta|
+          tables = ta.join(',')
+          result = @client.post("/#{@model_name.pluralize}/#{data_entry_id}/table_status", nil, {tables: tables})
+          results << result
+        end
+        results.inject({"table_status"=>[], "success"=>true}) do |h, result|
+          h["table_status"] += result["table_status"]
+          h
+        end
       end
 
       def cleanup_sync(data_entry_id, tables, options = {})
@@ -39,6 +177,27 @@ module Flydata
       def complete_init_sync(data_entry_id, stats_hash)
         @client.post("/#{@model_name.pluralize}/#{data_entry_id}/complete_init_sync", {:headers => {:content_type => :json}}, stats_hash.to_json)
       end
+
+      private
+
+      # TODO Refactor.  This code is copied from web data_entries_controller
+      def build_message(stat, complete)
+        return "" if complete
+
+        message = nil
+        buffer_size = stat["buffer"].inject(0) {|m, (k, v)| m + v["buffer_size"]}
+        chunk_num  = stat["queue"].inject(0) {|m, (k, v)| m + v["num_items"]}
+
+        if buffer_size > 0
+          message = "  -> %<buffer_size>s bytes remaining..." % {buffer_size: buffer_size.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse}
+        elsif chunk_num > 0
+          message = "  -> %<chunk_num>s data chunk(s) remaining..." % {chunk_num: chunk_num.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse}
+        else
+          message = "  -> Calculating remaining bytes..."
+        end
+
+        message
+      end
     end
   end
 end

data/lib/flydata/command/sync.rb

@@ -266,6 +266,7 @@ EOS
       def self.slop_repair
         Slop.new do
           on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
+          on 'skip-start', 'Skip auto start after repair is completed.'
           on 'force-run', 'Run forcefully, ignoring exclusive run info'
         end
       end
@@ -274,6 +275,9 @@ EOS
       #  - Entry method
       def repair
         _repair
+        unless opts.skip_start?
+          Flydata::Command::Sender.new.start
+        end
       end
       run_exclusive :repair
 

data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb

@@ -42,9 +42,10 @@ module Fluent
       positions_path = @sync_fm.table_positions_dir_path
       Dir.mkdir positions_path unless File.exists? positions_path
 
-      @tables = @tables.split(/(?:\s*,\s*|\s+)/)
+      @tables = @tables.to_s.split(/(?:\s*,\s*|\s+)/)
       @omit_events = Hash.new
-      @tables_append_only.split(/(?:\s*,\s*|\s+)/).each do |table|
+      @tables_append_only = @tables_append_only.to_s.split(/(?:\s*,\s*|\s+)/)
+      @tables_append_only.each do |table|
         @tables << table unless @tables.include?(table)
         @omit_events[table] = [:delete, :truncate_table]
       end

data/lib/flydata/fluent-plugins/flydata_plugin_ext/preference.rb

@@ -30,7 +30,8 @@ module Fluent
     end
 
     def apply_custom_conf(conf, key, type, option)
-      if conf[type.to_s] and value = conf[type.to_s][key.to_s]
+      if conf[type.to_s] and conf[type.to_s].has_key?(key.to_s)
+        value = conf[type.to_s][key.to_s]
         var_name = option[:var_name] || key
         instance_variable_set(:"@#{var_name}", value)
       end

data/lib/flydata/fluent-plugins/flydata_plugin_ext/transaction_support.rb

@@ -27,7 +27,7 @@ module Fluent
 
     def start
       if File.exists?(@lock_file)
-        $log.error "Previous process was terminated abnormally.  To start, remove the lock file after checking data integrity."
+        $log.warn "Previous process was terminated abnormally.  To start, remove the lock file after checking data integrity."
         @abort = true
         Process.kill(:TERM, Process.ppid)
         return

data/lib/flydata/source/sync.rb

@@ -57,10 +57,10 @@ class Sync < Component
     if prefs['tables_append_only']
       prefs['tables_append_only'] =
         prefs['tables_append_only'].split(/(?:\s*,\s*|\s+)/).uniq
-      prefs['tables'] = (prefs['tables'].split(/(?:\s*,\s*|\s+)/) +
+      prefs['tables'] = (prefs['tables'].to_s.split(/(?:\s*,\s*|\s+)/) +
                          prefs['tables_append_only']).uniq
     else
-      prefs['tables'] = prefs['tables'].split(/(?:\s*,\s*|\s+)/).uniq
+      prefs['tables'] = prefs['tables'].to_s.split(/(?:\s*,\s*|\s+)/).uniq
     end
     prefs['invalid_tables'] =
       prefs['invalid_tables'].kind_of?(String) ?

data/lib/flydata/source_mysql/parser/dump_parser.rb

@@ -211,7 +211,7 @@ EOS
         def flush_tables_with_read_lock_query(client, database, tbls)
           tbls ||= []
           tables = ""
-          if mysql_server_version(client) >= "5.5"
+          if mysql_server_version(client).to_f >= 5.5
             # FLUSH TABLES table_names,... WITH READ LOCK syntax is supported from MySQL 5.5
             tables = tbls.collect{|t| "`#{database}`.`#{t}`"}.join(",")
           end
@@ -629,13 +629,14 @@ SELECT
 FROM
   information_schema.tables
 WHERE
-  table_schema NOT IN ('information_schema','performance_schema','mysql') AND table_name in (%s);
+  table_schema = '%s' AND table_name in (%s);
 EOT
 
         def initialize(de_conf)
           @de_conf = de_conf
+          @database = de_conf['database']
           @tables = de_conf['tables']
-          @query = SIZE_CHECK_QUERY % [@tables.collect{|t| "'#{t}'"}.join(',')]
+          @query = SIZE_CHECK_QUERY % [@database, @tables.collect{|t| "'#{t}'"}.join(',')]
         end
 
         def get_db_bytesize

data/lib/flydata/source_mysql/plugin_support/alter_table_query_handler.rb

@@ -36,7 +36,7 @@ Received unsupported alter table query. normalized query:'#{record['normalized_q
 EOS
           $log.error(msg)
         end
-        $log.info(ret) # log all alter table events even if the table is not subject to sync.  This is for troubleshooting purpose.
+        $log.info(ret.to_s + " at \"#{binlog_pos(record)}\"") # log all alter table events even if the table is not subject to sync.  This is for troubleshooting purpose.
         ret
       end
     end

data/lib/flydata/source_mysql/plugin_support/binlog_query_dispatcher.rb

@@ -1,6 +1,7 @@
 require 'flydata/source_mysql/plugin_support/alter_table_query_handler'
 require 'flydata/source_mysql/plugin_support/truncate_table_query_handler'
 require 'flydata/source_mysql/plugin_support/drop_database_query_handler'
+require 'flydata/source_mysql/plugin_support/unsupported_query_handler'
 
 module Flydata
 module SourceMysql
@@ -63,6 +64,7 @@ module PluginSupport
         AlterTableQueryHandler.new(context),
         TruncateTableQueryHandler.new(context),
         DropDatabaseQueryHandler.new(context),
+        UnsupportedQueryHandler.new(context), # This must be a last element
       ]
     end
   end

data/lib/flydata/source_mysql/plugin_support/unsupported_query_handler.rb

@@ -0,0 +1,34 @@
+require 'flydata/parser/parser_provider'
+require 'flydata/source_mysql/plugin_support/ddl_query_handler'
+
+module Flydata
+module SourceMysql
+
+module PluginSupport
+  class UnsupportedQueryHandler < TableDdlQueryHandler
+    PATTERN = /.*/  # Catch all ddl
+
+    def initialize(context)
+      super
+    end
+
+    def pattern
+      PATTERN
+    end
+
+    def process(record)
+      query = record['normalized_query']
+      if acceptable_db?(record) && should_log?(query)
+        $log.info({ type: :unsupported_query, query: query.to_s.strip}.to_s + " at \"#{binlog_pos(record)}\"")
+      end
+    end
+
+    def should_log?(query)
+      return false if query.to_s.empty?
+      @context.tables.any?{|tbl| query.include?(tbl)}
+    end
+  end
+end
+
+end
+end

data/spec/flydata/api/data_entry_spec.rb

@@ -2,17 +2,345 @@ require 'spec_helper'
 require 'flydata/api/data_entry'
 
 module Flydata
-  module Api
-    describe DataEntry do
-      describe '#create' do
-        it 'creates data entry' do
-          api_client = double('api_client')
-          allow(api_client).to receive(:post)
-
-          api = DataEntry.new(api_client)
-          api.create(data_port_id: 1, log_path: 'log-path')
+module Api
+
+describe DataEntry do
+  let(:subject_object) { described_class.new(api_client) }
+
+  let(:api_client) { double('api_client') }
+
+  let(:data_entry_id) { 1234 }
+  let(:options) { { tables: tables } }
+
+  describe '#create' do
+    subject { subject_object.create(params) }
+
+    let(:params) { {data_port_id: 1, log_path: 'log-path'} }
+    it 'creates data entry' do
+      expect(api_client).to receive(:post)
+
+      subject
+    end
+  end
+
+  def build_result(complete, state,
+                   proxy_size, proxy_length, redshift_size, redshift_length,
+                   sync_redshift_size, sync_redshift_length,
+                   copy_num, query_num, message)
+    buffer = nil
+    queue = nil
+    if sync_redshift_size
+      buffer = { "proxy" => {
+                   "buffer_size" => proxy_size,
+                   "queue_length" => proxy_length,
+                 },
+                 "redshift" => {
+                   "buffer_size" => redshift_size,
+                   "queue_length" => redshift_length,
+                 },
+                 "sync_redshift" => {
+                   "buffer_size" => sync_redshift_size,
+                   "queue_length" => sync_redshift_length,
+                 },
+               }
+      queue = { "copy_queue" => { "num_items" => copy_num },
+                "query_queue" => { "num_items" => query_num },
+              }
+    else
+      buffer = { "redshift" => {
+                   "buffer_size" => redshift_size,
+                   "queue_length" => redshift_length,
+                 },
+               }
+      queue = {
+                "copy_queue" => { "num_items" => copy_num },
+              }
+    end
+
+    {
+      "complete" => complete,
+      "state" => state,
+      "stat" => { "buffer" => buffer, "queue" => queue },
+      "message" => message,
+      "success" => true
+    }
+  end
+
+  describe '#buffer_stat' do
+    subject { subject_object.buffer_stat(data_entry_id, options) }
+
+    let(:result1) {
+      build_result(complete1, state1, proxy_size1, proxy_length1,
+                   redshift_size1, redshift_length1,
+                   sync_redshift_size1, sync_redshift_length1,
+                   copy_num1, query_num1, message1)
+    }
+    let(:complete1) { false }
+    let(:state1) { "processing" }
+    let(:proxy_size1) { 1 }
+    let(:proxy_length1) { 3 }
+    let(:redshift_size1) { 5 }
+    let(:redshift_length1) { 7 }
+    let(:sync_redshift_size1) { 11 }
+    let(:sync_redshift_length1) { 13 }
+    let(:copy_num1) { 17 }
+    let(:query_num1) { 19 }
+    let(:message1) { double('message1') }
+    let(:message2) { double('message2') }
+
+    shared_examples "calling api once and return expected result" do
+      it do
+        expect(api_client).to receive(:post).
+         with("/data_entries/#{data_entry_id}/buffer_stat/",
+              nil, tables: expected_tables).
+         and_return(api_result)
+
+        expect(subject).to eq expected_result
+      end
+    end
+
+    context "with no tables" do
+      let(:tables) { nil }
+      let(:expected_tables) { '' }
+      let(:api_result) { result1 }
+      let(:expected_result) { result1 }
+      let(:message) { message1 }
+
+      it_behaves_like "calling api once and return expected result"
+    end
+    context "with empty table list" do
+      let(:tables) { [] }
+      let(:expected_tables) { '' }
+      let(:api_result) { result1 }
+      let(:expected_result) { result1 }
+      let(:message) { message1 }
+
+      it_behaves_like "calling api once and return expected result"
+    end
+    context "with less than 100 tables" do
+      let(:tables) { %w[a b] }
+      let(:expected_tables) { "a,b" }
+      let(:api_result) { result1 }
+      let(:expected_result) { result1 }
+
+      it_behaves_like "calling api once and return expected result"
+
+      context 'when buffer is empty' do
+        let(:proxy_size1) { 0 }
+        let(:proxy_length1) { 0 }
+        let(:redshift_size1) { 0 }
+        let(:redshift_length1) { 0 }
+        let(:sync_redshift_size1) { 0 }
+        let(:sync_redshift_length1) { 0 }
+
+        let(:api_result) { result1 }
+
+        context 'when queue is empty' do
+          let(:copy_num1) { 0 }
+          let(:query_num1) { 0 }
+
+          let(:expected_result) {
+            build_result(true, "complete", proxy_size1, proxy_length1,
+                         redshift_size1, redshift_length1,
+                         sync_redshift_size1, sync_redshift_length1,
+                         copy_num1, query_num1, message1)
+          }
+
+         it_behaves_like "calling api once and return expected result"
+        end
+        context 'when queue is not empty' do
+          let(:expected_result) {
+            build_result(false, "uploading", proxy_size1, proxy_length1,
+                         redshift_size1, redshift_length1,
+                         sync_redshift_size1, sync_redshift_length1,
+                         copy_num1, query_num1, message1)
+          }
+
+         it_behaves_like "calling api once and return expected result"
+        end
+      end
+      context 'when buffer is not empty' do
+        context 'when queue is empty' do
+          let(:copy_num1) { 0 }
+          let(:query_num1) { 0 }
+
+          let(:expected_result) {
+            build_result(false, "processing", proxy_size1, proxy_length1,
+                         redshift_size1, redshift_length1,
+                         sync_redshift_size1, sync_redshift_length1,
+                         copy_num1, query_num1, message1)
+          }
+
+         it_behaves_like "calling api once and return expected result"
+        end
+      end
+    end
+    context "with more than 100 tables" do
+      let(:tables) { 180.times.collect{|i| "tbl#{i}"} }
+      let(:expected_tables1) { tables[0...150].join(",") }
+      let(:expected_tables2) { tables[150..-1].join(",") }
+
+      let(:expected_message) { double('expected_message') }
+
+      context "when no error" do
+        shared_examples "calling api twice and return combined result" do
+          it do
+            expect(api_client).to receive(:post).
+             with("/data_entries/#{data_entry_id}/buffer_stat/",
+                  nil, tables: expected_tables1).
+             and_return(api_result1)
+            expect(api_client).to receive(:post).
+             with("/data_entries/#{data_entry_id}/buffer_stat/",
+                  nil, tables: expected_tables2).
+             and_return(api_result2)
+            expect(subject_object).to receive(:build_message).
+             and_return(expected_message)
+
+            expect(subject).to eq expected_result
+          end
+        end
+
+        context "with Sync entry" do
+          let(:api_result1) {
+            build_result(complete1, state1, proxy_size1, proxy_length1,
+                         redshift_size1, redshift_length1,
+                         sync_redshift_size1, sync_redshift_length1,
+                         copy_num1, query_num1, message1)
+          }
+          let(:api_result2) {
+            build_result(complete1, state1, proxy_size1, proxy_length1,
+                         redshift_size1, redshift_length1,
+                         sync_redshift_size1, sync_redshift_length1,
+                         copy_num1, query_num1, message2)
+          }
+          let(:expected_result) {
+            build_result(complete1, state1,
+                         proxy_size1,
+                         proxy_length1,
+                         redshift_size1 + redshift_size1,
+                         redshift_length1 + redshift_length1,
+                         sync_redshift_size1 + sync_redshift_size1,
+                         sync_redshift_length1 + sync_redshift_length1,
+                         copy_num1 + copy_num1,
+                         query_num1 + query_num1, expected_message)
+          }
+          it_behaves_like "calling api twice and return combined result"
+        end
+        context "with AutoLoad entry" do
+          let(:api_result1) {
+            build_result(complete1, state1, nil, nil,
+                         redshift_size1, redshift_length1,
+                         nil, nil,
+                         copy_num1, query_num1, message1)
+          }
+          let(:api_result2) {
+            build_result(complete1, state1, nil, nil,
+                         redshift_size1, redshift_length1,
+                         nil, nil,
+                         copy_num1, query_num1, message2)
+          }
+          let(:expected_result) {
+            build_result(complete1, state1,
+                         nil,
+                         nil,
+                         redshift_size1 + redshift_size1,
+                         redshift_length1 + redshift_length1,
+                         nil,
+                         nil,
+                         copy_num1 + copy_num1,
+                         query_num1 + query_num1, expected_message)
+          }
+          it_behaves_like "calling api twice and return combined result"
+        end
+      end
+      context "with an error with the first api call" do
+        let(:api_result1) { {
+          "complete" => false,
+          "message" => expected_message,
+         "success" => true,
+        } }
+        let(:api_result2) { result1 }
+        let(:expected_result) { api_result1 }
+        it do
+          expect(api_client).to receive(:post).
+           with("/data_entries/#{data_entry_id}/buffer_stat/",
+                nil, tables: expected_tables1).
+           and_return(api_result1)
+          expect(subject_object).not_to receive(:build_message)
+
+          expect(subject).to eq expected_result
         end
       end
     end
   end
+
+  describe '#table_status' do
+    subject { subject_object.table_status(data_entry_id, options) }
+
+    let(:result1) {
+      { "table_status" => [ status11, status12 ], "success" => true }
+    }
+    let(:status11) { double('status11') }
+    let(:status12) { double('status12') }
+
+    shared_examples "calling api with tables and return expected result" do
+      it do
+        expect(api_client).to receive(:post).
+         with("/data_entries/#{data_entry_id}/table_status",
+              nil, tables: expected_tables).
+         and_return(api_result)
+
+        expect(subject).to eq expected_result
+      end
+    end
+
+    context "with no tables" do
+      let(:tables) { nil }
+      let(:expected_tables) { '' }
+      let(:api_result) { result1 }
+      let(:expected_result) { result1 }
+
+      it_behaves_like "calling api with tables and return expected result"
+    end
+
+    context "with less than 100 tables" do
+      let(:tables) { %w[a b] }
+      let(:expected_tables) { 'a,b' }
+      let(:api_result) { result1 }
+      let(:expected_result) { result1 }
+
+      it_behaves_like "calling api with tables and return expected result"
+    end
+
+    let(:result2) {
+      { "table_status" => [ status21, status22 ], "success" => true }
+    }
+    let(:status21) { double('status21') }
+    let(:status22) { double('status22') }
+
+    context "with more than 100 tables" do
+      let(:tables) { 180.times.collect {|i| "tbl#{i}"} }
+      let(:expected_tables1) { tables[0...150].join(",") }
+      let(:expected_tables2) { tables[150..-1].join(",") }
+
+      it do
+        expect(api_client).to receive(:post).
+         with("/data_entries/#{data_entry_id}/table_status",
+              nil, tables: expected_tables1).
+         and_return(result1)
+
+        expect(api_client).to receive(:post).
+         with("/data_entries/#{data_entry_id}/table_status",
+              nil, tables: expected_tables2).
+         and_return(result2)
+
+        expect(subject).to eq(
+           "table_status" => [ status11, status12, status21, status22 ],
+           "success" => true )
+      end
+    end
+  end
+end
+
+end
 end

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -264,6 +264,10 @@ EOT
         expect(plugin.instance_variable_get(:@tables)).to eq(%w(test_table))
       end
 
+      it 'sets data tables_append_only as array' do
+        expect(plugin.instance_variable_get(:@tables_append_only)).to eq(%w(test_table_3))
+      end
+
       it 'sets sync_fm' do
         expect(plugin.instance_variable_get(:@sync_fm)).to be_kind_of(Flydata::SyncFileManager)
       end

data/spec/flydata/source_mysql/parser/dump_parser_spec.rb

@@ -31,10 +31,12 @@ module Flydata
         end
 
         describe MysqlDumpGeneratorNoMasterData do
+          let(:subject_object) { described_class.new(default_conf) }
+
           let(:stderr) { double(:stderr) }
           let(:wait_thr) { double(:wait_thr) }
 
-          let(:default_dump_generator) { MysqlDumpGeneratorNoMasterData.new(default_conf) }
+          let(:default_dump_generator) { subject_object }
           let(:binlog_pos) { { binfile: 'mysql-bin.000451', pos: 89872 } }
           let(:mysql_client) do
             m = double(:mysql_client)
@@ -80,6 +82,54 @@ module Flydata
               File.delete(file_path) if File.exists?(file_path)
             end
           end
+
+          describe '#flush_tables_with_read_lock_query' do
+            subject { subject_object.send(:flush_tables_with_read_lock_query,
+                      client, database, tbls) }
+
+            let(:client) { double('client') }
+            let(:database) { "db1" }
+            let(:tbls) { ['table1'] }
+
+            before do
+              allow(subject_object).to receive(:mysql_server_version).
+               and_return(version)
+            end
+
+            let(:flush_tables_with_tables) { <<EOS.strip }
+FLUSH TABLES `db1`.`table1` WITH READ LOCK;
+EOS
+            let(:flush_tables_without_table) { <<EOS.strip }
+FLUSH TABLES  WITH READ LOCK;
+EOS
+
+            context 'with MySQL 5.1' do
+              let(:version) { "5.1.67-log" }
+              it 'returns flush command without table because the syntax is only supported by MySQL 5.5 or later' do
+                is_expected.to eq(flush_tables_without_table)
+              end
+            end
+            context 'with Debian MySQL 5.5' do
+              let(:version) { "5.5.24-2+deb.sury.org~precise+2-log" }
+              it { is_expected.to eq(flush_tables_with_tables) }
+            end
+            context 'with Amazon RDS MySQL 5.6' do
+              let(:version) { "5.6.19-log" }
+              it { is_expected.to eq(flush_tables_with_tables) }
+            end
+            context 'with MariaDB 10.0' do
+              let(:version) { "10.0.24-MariaDB" }
+              it 'returns flush command with tables because MariaDB 10.0 is compatible with MySQL 5.6' do
+                is_expected.to eq(flush_tables_with_tables)
+              end
+            end
+            context 'with Amazon Aurora' do
+              let(:version) { "5.6.10" }
+              it do
+                is_expected.to eq(flush_tables_with_tables)
+              end
+            end
+          end
         end
       end
 

data/spec/flydata/source_mysql/plugin_support/binlog_query_dispatcher_spec.rb

@@ -64,6 +64,11 @@ EOT
       allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::DropDatabaseQueryHandler::PATTERN)
       r
     end
+    let(:unsupported_query_handler) do
+      r = double('unsupported_query_handler')
+      allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::UnsupportedQueryHandler::PATTERN)
+      r
+    end
     let(:context) { double('context') }
     let(:subject_object) { described_class.new(context) }
     subject { subject_object }
@@ -79,6 +84,7 @@ EOT
       expect(AlterTableQueryHandler).to receive(:new).with(context).and_return(alter_query_handler)
       expect(TruncateTableQueryHandler).to receive(:new).with(context).and_return(truncate_query_handler)
       expect(DropDatabaseQueryHandler).to receive(:new).with(context).and_return(drop_database_query_handler)
+      allow(UnsupportedQueryHandler).to receive(:new).with(context).and_return(unsupported_query_handler)
     end
 
     describe '#dispatch' do
@@ -115,11 +121,26 @@ ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0 /*applic
 /*!*/;
 SET TIMESTAMP=1415925954/*!*/;
 EOS
-          let(:normalized_query) { <<EOS.gsub(/\n$/, '') }
+          let(:normalized_query_1) { 'use `mydatabase` ;' }
+          let(:normalized_query_2) { <<EOS.gsub(/\n$/, '') }
 ALTER TABLE `apps` ADD `created_by_existing_admin` tinyint(1) DEFAULT 0 ;
 EOS
+          let(:normalized_query_3) { 'SET TIMESTAMP=1415925954 ;' }
+
+          let(:record) do
+            r = double('record')
+            allow(r).to receive(:[]).with("query").and_return(query)
+            expect(r).to receive(:merge).with({"normalized_query" => normalized_query_1}).and_return(r)
+            expect(r).to receive(:merge).with({"normalized_query" => normalized_query_2}).and_return(r)
+            expect(r).to receive(:merge).with({"normalized_query" => normalized_query_3}).and_return(r)
+            r
+          end
 
-          it_behaves_like "a dispatcher that calls query handler with correct query"
+          it do
+            expect(unsupported_query_handler).to receive(:process).with(record).twice
+            expect(alter_query_handler).to receive(:process).with(record).once
+            expect(subject.dispatch(record))
+          end
         end
         context "query with a comment with no space" do
           let(:query) { <<EOS.gsub(/\n$/, '') }

data/spec/flydata/source_mysql/plugin_support/unsupported_query_handler_spec.rb

@@ -0,0 +1,42 @@
+require 'fluent_plugins_spec_helper'
+require 'flydata/source_mysql/plugin_support/unsupported_query_handler'
+require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
+
+module Flydata::SourceMysql::PluginSupport
+  describe UnsupportedQueryHandler do
+    include_context "query handler context"
+
+    describe '#process' do
+      before do
+        record['normalized_query'] = normalized_query
+      end
+
+      context 'when the query includes a sync table' do
+        let(:normalized_query) { 'drop table foo;' }
+        it do
+          expect($log).to receive(:info).with("{:type=>:unsupported_query, :query=>\"drop table foo;\"} at \"mysql-bin.000066\t180\"")
+          subject.process(record)
+        end
+      end
+
+      context 'when the query does not include a sync table' do
+        let(:normalized_query) { 'drop table bar;' }
+        it do
+          expect($log).to receive(:info).never
+          subject.process(record)
+        end
+      end
+
+      context 'when the query is issued in another database' do
+        let(:normalized_query) { 'drop table foo;' }
+        before do
+          record['db_name'] = 'another_db'
+        end
+        it do
+          expect($log).to receive(:info).never
+          subject.process(record)
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.7.5
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-28 00:00:00.000000000 Z
+date: 2016-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -731,6 +731,7 @@ files:
 - lib/flydata/source_mysql/plugin_support/drop_database_query_handler.rb
 - lib/flydata/source_mysql/plugin_support/source_position_file.rb
 - lib/flydata/source_mysql/plugin_support/truncate_table_query_handler.rb
+- lib/flydata/source_mysql/plugin_support/unsupported_query_handler.rb
 - lib/flydata/source_mysql/setup.rb
 - lib/flydata/source_mysql/source_pos.rb
 - lib/flydata/source_mysql/sync.rb
@@ -828,6 +829,7 @@ files:
 - spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb
 - spec/flydata/source_mysql/plugin_support/source_position_file_spec.rb
 - spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb
+- spec/flydata/source_mysql/plugin_support/unsupported_query_handler_spec.rb
 - spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb
 - spec/flydata/source_mysql/table_ddl_spec.rb
 - spec/flydata/source_mysql/table_meta_spec.rb
@@ -867,7 +869,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.0.14.1
 signing_key: 
 specification_version: 4
 summary: FlyData Agent