flydata 0.5.17 → 0.5.20

data/lib/flydata/event/api_event_sender.rb

@@ -0,0 +1,16 @@
+require 'flydata-core/event/flydata_event_sender'
+require 'flydata-core/event/flydata_event'
+require 'singleton'
+
+module FlydataCore
+  module Event
+    class ApiEventSender <FlydataEventSender
+      include Singleton
+      protected
+      def send(event, routing_key=nil)
+        #TODO async when async api code is pushed
+        Flydata::ApiClient.instance.post("/events/process",{:headers => {:content_type => :json}}, event.to_json)
+      end
+    end
+  end
+end

data/lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb

@@ -5,6 +5,14 @@ module Mysql
 class DdlQueryHandler < BinlogQueryHandler
   DDL_TABLE_QUERY = /^(?:(?:ALTER|CREATE|DROP|RENAME) +(?:\w+ +)*TABLE +([^ ]+)|TRUNCATE +(?:TABLE +)?([^ ;]+))/i
 
+  def emit_record(type, record)
+    # ddl event record doesn't have "table_name"
+    record['table_name'] = table_info(record)[:table_name]
+    super do |opt|
+      yield opt
+    end
+  end
+
   def acceptable_db?(record)
     supported_database == table_info(record)[:db_name]
   end

data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb

@@ -16,7 +16,7 @@ module Mysql
     def process(record)
       emit_record(:truncate_table, record) do |opt|
         {
-          table_name: table_info(record)[:table_name],
+          table_name: record['table_name'],
           query: record["query"]
         }
       end

data/lib/flydata/output/forwarder.rb

@@ -60,15 +60,13 @@ module Flydata
           @buffer_record_count += 1
           @buffer_size += event_data.bytesize
         end
-        if @buffer_size > @buffer_size_limit
-          send
-        else
-          false
-        end
+        buffer_full?
       end
 
       #TODO retry logic
       def send
+        byte_size = nil
+        record_count = nil
         if @buffer_size > 0
         else
           return false
@@ -113,6 +111,10 @@ module Flydata
         { byte_size: byte_size, record_count: record_count }
       end
 
+      def buffer_full?
+        @buffer_size > @buffer_size_limit
+      end
+
       #TODO: Check server status
       def pickup_server
         ret_server = @servers[@server_index]

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -38,8 +38,7 @@ module Flydata
           @value_converters = {}
         end
 
-        attr_accessor :table_name, :columns, :column_names, :primary_keys
-        attr_reader :value_converters
+        attr_accessor :table_name, :columns, :column_names, :primary_keys, :value_converters
 
         def add_column(column)
           cn = column[:column_name]
@@ -82,7 +81,7 @@ module Flydata
           dump_cmd = generate_dump_cmd(@conf, file_path)
 
           # RDS doesn't allow obtaining binlog position using mysqldump.  Get it separately and insert it into the dump file.
-          table_locker = create_table_locker
+          table_locker = create_table_locker(@conf["database"], @conf["tables"])
           table_locker.resume # Lock tables
 
           begin
@@ -181,12 +180,12 @@ AND TABLE_SCHEMA IN (%s)
 AND ENGINE NOT IN ('MEMORY', 'CSV', 'PERFORMANCE_SCHEMA');
 EOS
 
-        def create_table_locker
+        def create_table_locker(database, tables)
           Fiber.new do
             # short timeout to avoid blocking other queries
             client = FlydataMysqlClient.new({reconnect: true, read_timeout:9}.merge(@db_opts))
 
-            q = flush_tables_with_read_lock_query(client)
+            q = flush_tables_with_read_lock_query(client, database, tables)
             $log.debug "FLUSH TABLES query: #{q}"
             thread_id = nil
             begin
@@ -238,12 +237,12 @@ EOS
           end
         end
 
-        def flush_tables_with_read_lock_query(client)
+        def flush_tables_with_read_lock_query(client, database, tbls)
+          tbls ||= []
           tables = ""
           if mysql_server_version(client) >= "5.5"
             # FLUSH TABLES table_names,... WITH READ LOCK syntax is supported from MySQL 5.5
-            result = client.query(USER_TABLES_QUERY % ["mysql", client.query_options[:database].to_s].collect{|db| "'#{db}'"}.join(","))
-            tables = result.collect{|r| r['tables']}.join(", ")
+            tables = tbls.collect{|t| "`#{database}`.`#{t}`"}.join(",")
           end
           FLUSH_TABLES_QUERY_TEMPLATE % [tables]
         end
@@ -307,11 +306,12 @@ EOS
           @option = option
         end
 
-        def parse(dump_io, create_table_block, insert_record_block, check_point_block)
-          unless dump_io.kind_of?(IO)
+        def parse(dmpio, create_table_block, insert_record_block, check_point_block)
+          unless dmpio.kind_of?(IO)
             raise ArgumentError.new("Invalid argument. The first parameter must be io.")
           end
 
+          dump_io = nil
           invalid_file = false
           current_state = State::START
           substate = nil
@@ -449,6 +449,7 @@ EOS
             current_state = State::INSERT_RECORD
 
             if insert_record_block.call(current_table, values_set)
+              values_set = nil
               check_point_block.call(current_table, dump_io.pos, bytesize, @binlog_pos, current_state)
             end
           end
@@ -457,13 +458,15 @@ EOS
           begin
             # resume(only when using dump file)
             if @option[:last_pos] && (@option[:last_pos].to_i != -1)
-              dump_io.pos = @option[:last_pos].to_i
+              dmpio.pos = @option[:last_pos].to_i
               current_state = @option[:state]
               substate = @option[:substate]
               current_table = @option[:mysql_table]
-              bytesize = dump_io.pos
+              bytesize = dmpio.pos
             end
 
+            dump_io = AsyncIO.new(dmpio)
+
             until dump_io.eof? do
               case current_state
               when State::START
@@ -480,6 +483,8 @@ EOS
                 state_parsing_insert_record.call
               end
             end
+          ensure
+            dump_io.close
           end
           @binlog_pos
         end
@@ -679,6 +684,65 @@ EOT
           client.close rescue nil
         end
       end
+
+      # Read and buffer data in a separate thread
+      class AsyncIO
+        MAX_ITEMS = 200
+        def initialize(io, options = {})
+          max_items = options[:max_items] ? options[:max_items] : MAX_ITEMS
+          @io = io
+          @queue = SizedQueue.new(max_items)
+          _readline
+          @last = @queue.shift
+          @stop = false
+          @thread = Thread.new(&method(:run))
+        end
+
+        def readline
+          if @last[:eof]
+            raise EOFError.new("end of file reached")
+          else
+            result = @last[:line]
+            @last = @queue.shift
+            result
+          end
+        end
+
+        def pos
+          @last[:pos]
+        end
+
+        def eof?
+          @last[:eof]
+        end
+
+        def close
+          @stop = true
+          # remove an item if the queue is full.  Otherwise, the thread will not
+          # wake up.
+          @queue.shift if @queue.size == @queue.max
+          @thread.join
+        end
+
+        private
+
+        def run
+          until @io.eof?
+            return if @stop
+
+            _readline
+          end
+          _readline
+        end
+
+        def _readline
+          pos = @io.pos
+          eof = @io.eof?
+          line = eof ? nil : @io.readline
+          data = { line: line, pos: pos, eof: eof }
+          @queue << data
+        end
+      end
     end
   end
 end

data/lib/flydata/queueable_thread.rb

@@ -0,0 +1,32 @@
+module Flydata
+
+class QueueableThread
+  MAX_JOBS = 60
+  def initialize(max_jobs = MAX_JOBS)
+    @queue = SizedQueue.new(max_jobs)
+    @stop = false
+    @thread = Thread.new(&method(:run_loop))
+    @thread.abort_on_exception = true
+  end
+
+  def run(&block)
+    @queue << block
+  end
+
+  def join
+    @stop = true
+    @queue << nil if @queue.empty?  # wake up the thread
+    @thread.join
+  end
+
+  private
+
+  def run_loop
+    until @stop && @queue.empty?
+      block = @queue.shift
+      block.call if block
+    end
+  end
+end
+
+end

data/lib/flydata/sync_file_manager.rb

@@ -1,5 +1,6 @@
 require 'mysql2'
 require 'flydata-core/query_job'
+require 'set'
 
 module Flydata
   class SyncFileManager
@@ -298,6 +299,50 @@ module Flydata
       new_rev
     end
 
+    def delete_table_control_files(*tables)
+      return if (tables.nil? or tables.empty?)
+      files_to_delete = [
+          table_position_file_paths(*tables),
+          table_binlog_pos_paths(*tables),
+          table_binlog_pos_init_paths(*tables),
+          table_rev_file_paths(*tables),
+          table_ddl_file_paths(*tables)
+      ]
+      files_to_delete.flatten.each do |path|
+        FileUtils.rm(path) if File.exists?(path)
+      end
+    end
+
+    def tables_from_positions_dir
+      all_table_control_files = Dir.glob(File.join(table_positions_dir_path, '*.{pos,generated_ddl,init,rev}'))
+      return if all_table_control_files.nil?
+      tables = Set.new
+      all_table_control_files.each do |control_file|
+        file_name = File.basename(control_file)
+        file_name = file_name.slice(0...(file_name.index('.')))
+        tables << file_name
+      end
+      tables.to_a
+    end
+
+    def delete_non_table_control_files(delete_binlog= false)
+      files_to_delete = [
+          dump_file_path,
+          dump_pos_path,
+          mysql_table_marshal_dump_path,
+          sync_info_file,
+          stats_path
+      ]
+      if delete_binlog
+        files_to_delete << binlog_path
+        files_to_delete << sent_binlog_path
+        files_to_delete << lock_pid_file
+      end
+      files_to_delete.flatten.each do |file_to_delete|
+        FileUtils.rm(file_to_delete) if File.exists?(file_to_delete)
+      end
+    end
+
     def delete_table_binlog_pos(table_name)
       file = File.join(table_positions_dir_path, table_name + ".binlog.pos")
       if File.exists?(file)

data/spec/flydata/command/start_spec.rb

@@ -14,7 +14,7 @@ module Flydata
         expect(sender).to receive(:start)
         expect(Flydata::Command::Helper).to receive(:new).and_return(helper)
         expect(helper).to receive(:stop)
-        expect(helper).to receive(:start)
+        expect(subject).to receive(:system) # Use Kernal#system to launch the helper
         subject.run
 
       end

data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb

@@ -19,9 +19,9 @@ module Mysql
       before do
         parser_class = ParserProvider.parser(:mysql, :mysql_alter_table)
         allow(parser_class).to receive(:new).and_return(parser)
-        allow(subject).to receive(:acceptable_table?).and_return(true)
-        allow(subject).to receive(:acceptable_db?).and_return(true)
-        allow(subject).to receive(:check_empty_binlog)
+        allow_any_instance_of(described_class).to receive(:check_empty_binlog)
+        record.delete('table_name')
+        record['normalized_query'] = "truncate table #{table};"
       end
       shared_examples "a process method receiving an exception" do
         let(:an_error) { "an error" }
@@ -54,6 +54,13 @@ module Mysql
           expect(subject.process(record)).to eq(nil)
         end
       end
+      context "when event binlog is older than table binlog.pos" do
+        it 'skip sending event' do
+          expect(sync_fm).to receive(:get_table_binlog_pos).and_return("mysql-bin.000067\t120").once
+          expect(ParserProvider).not_to receive(:parser)
+          expect(subject.process(record)).to eq(nil)
+        end
+      end
     end
   end
 end

data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb

@@ -28,20 +28,20 @@ module Mysql
       r
     end
     let(:query) { "a_query" }
-    let(:normalized_query) { double('normalized_query') }
+    let(:normalized_query) { 'a_query' }
     let(:event_length) { 20 }
     let(:next_position) { 200 }
     let(:timestamp) { 1427973738 }
     let(:record) do
-      r = double('record')
-      allow(r).to receive(:[]).with("db_name").and_return(database)
-      allow(r).to receive(:[]).with("query").and_return(query)
-      allow(r).to receive(:[]).with("table_name").and_return(table)
-      allow(r).to receive(:[]).with("normalized_query").and_return(normalized_query)
-      allow(r).to receive(:[]).with("next_position").and_return(next_position)
-      allow(r).to receive(:[]).with("event_length").and_return(event_length)
-      allow(r).to receive(:[]).with("timestamp").and_return("#{timestamp}")
-      r
+      {
+        'db_name' => database,
+        'query' => query,
+        'table_name' => table,
+        'normalized_query' => normalized_query,
+        'next_position' => next_position,
+        'event_length' => event_length,
+        'timestamp' => "#{timestamp}",
+      }
     end
     subject { described_class.new(context) }
   end

data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb

@@ -10,8 +10,9 @@ module Mysql
       let(:truncate_query) { "TRUNCATE table foo" }
 
       before do
-        allow(record).to receive(:[]).with("query").and_return(truncate_query)
-        allow(record).to receive(:[]).with("normalized_query").and_return(truncate_query)
+        record['query'] = truncate_query
+        record['normalized_query'] = truncate_query
+        record.delete('table_name')
       end
 
       shared_examples "process truncate queries correctly" do
@@ -74,6 +75,14 @@ module Mysql
           include_examples "skip processing queries"
         end
       end
+
+      context 'when per-table binlog pos exists' do
+        let(:truncate_query) { "TRUNCATE #{table}" }
+        before do
+          allow(sync_fm).to receive(:get_table_binlog_pos).and_return("mysql-bin.000067\t120")
+        end
+        include_examples "skip processing queries"
+      end
     end
   end
 end

data/spec/flydata/output/forwarder_spec.rb

@@ -68,12 +68,12 @@ module Flydata
           end
         end
         context 'when the buffer size exceeds threthold' do
-          it do
+          it 'returns true but does not flush buffer' do
             expect(forwarder.emit(record)).to be(false)
             expect(forwarder.emit(record)).to be(false)
             expect(forwarder.buffer_record_count).to be(2)
-            expect(forwarder.emit(record)).to eq(sent_stats)
-            expect(forwarder.buffer_record_count).to be(0)
+            expect(forwarder.emit(record)).to eq(true)
+            expect(forwarder.buffer_record_count).to be(3)
           end
         end
         context 'when the error happens during the data sending' do
@@ -91,7 +91,7 @@ module Flydata
           it 'retry and succeed sending data' do
             forwarder.emit(record)
             forwarder.emit(record)
-            expect(forwarder.emit(record)).to eq(sent_stats)
+            expect(forwarder.emit(record)).to eq(true)
           end
         end
       end