flydata 0.6.14 → 0.7.0

data/lib/flydata/plugin_support/sync_record_emittable.rb

@@ -18,13 +18,14 @@ module PluginSupport
     #   "records" : A record or records for emitting
     #               Each record needs to be Hash
     #   "options"
+    #     type:     : (required) type (insert, update, delete)
     #     tag       : (optional) tag (default: @context.tag)
     #     timestamp : (optional) timestamp (default: current timestamp)
     #     src_pos   : (required) source position (used for sync:repair)
     #     table     : (optional) table name
     #     increment_table_rev : (optional) set true when incrementing table revision
     def emit_sync_records(records, options)
-      return if records.nil? # skip
+      return if records.nil? || records.empty? # skip
       records = [records] unless records.kind_of?(Array)
 
       # Check options

data/lib/flydata/query_based_sync/client.rb

@@ -0,0 +1,101 @@
+require 'thread'
+require 'flydata-core/logger'
+
+module Flydata
+module QueryBasedSync
+
+  class Client
+    include FlydataCore::Logger
+
+    DEFAULT_FETCH_INTERVAL = 60 # 1 minute
+    DEFAULT_RETRY_INTERVAL = 30 # 1 minute
+
+    # params
+    #   fetch_interval
+    #   resource_names
+    def initialize(context)
+      @context = context
+      @resource_requester = self.class::RESOURCE_REQUESTER_CLASS.new(context)
+      @response_handler = self.class::RESPONSE_HANDLER_CLASS.new(context)
+
+      context.params.tap do |c|
+        @fetch_interval = c.nil? ? DEFAULT_FETCH_INTERVAL : c[:fetch_interval]
+        @retry_interval = c.nil? ? DEFAULT_RETRY_INTERVAL : c[:retry_interval]
+      end
+    end
+
+    attr_reader :context
+    attr_reader :resource_requester
+    attr_reader :response_handler
+
+    def start
+      raise "Already started - thread:#{@running_thread}" if @running_thread
+      raise "Already stop requested" if @stop_requested
+      @running_thread = Thread.current
+      run_loop
+    end
+
+    def stop_request
+      @finish = true
+      if @running_thread && @running_thread.alive?
+        @running_thread.run
+      end
+    end
+
+    def stop_requested?
+      !!@finish
+    end
+
+    def handle_response(response)
+      log_debug("Handling response", table_name: response.table_name, count:response.records.count,
+                new_snapshot: response.new_source_pos, base_snapshot: context.table_meta.current_snapshot)
+      @response_handler.handle(response)
+    end
+
+    private
+
+    def run_loop
+      begin
+        until stop_requested?
+          reset_log_context
+          run_once
+          break if stop_requested?
+          sleep @fetch_interval
+        end
+      rescue => e
+        #TODO: Introduce retryable error notions
+        log_error_with_backtrace("Unexpected error occured", error: e)
+        sleep @retry_interval
+        retry
+      end
+    end
+
+    def run_once
+      resource_requester.start do |req|  # open connection
+        context.tables.each do |table_name|
+          add_log_context_items(table_name: table_name)
+          return if stop_requested?
+          req.each_response(table_name) do |response|
+            return if stop_requested?
+            handle_response(response)
+
+            unless response.empty?
+              log_info("Emitted records", count: response.record_count, src_pos:response.new_source_pos)
+            end
+
+            return if stop_requested?
+          end
+
+          delete_log_context_item(:table_name)
+        end
+      end
+
+      # Set the current snapshot to master(resume) binlog.pos file
+      context.cur_src_pos_file.save(context.table_meta.current_snapshot)
+
+      log_info("Updated source position -", resume_pos:context.table_meta.current_snapshot)
+    end
+  end
+
+end
+end

data/lib/flydata/query_based_sync/record_size_estimator.rb

@@ -0,0 +1,39 @@
+module Flydata
+module QueryBasedSync
+
+  class RecordSizeEstimator
+    # Data size for the static part of sent record
+    # {"row":{},"type":"update","respect_order":true,
+    #  "src_pos":"10051:10051:\t10520:10052:\t{\"id\":\"1000000000\"}",
+    #  "v":2,"seq":10000000,"table_name":"","table_rev":1}
+    BASE_DATA_SIZE = 162
+
+    def initialize(table_name, num_columns)
+      @base_record_size = calc_base_record_size(table_name, num_columns)
+    end
+
+    # Calculate a record size for emitting
+    def calc_record_size(record)
+      @base_record_size + record.values.inject(0){|r, v| r+=v.to_s.size}
+    end
+
+    private
+
+    def calc_base_record_size(table_name, num_columns)
+      col_cnt = num_columns
+      # static part of {"1":"16","2":"cc", ...}
+      col_size_for_static = 6 * col_cnt - 1 + 2
+      col_size_for_key = 0
+      col_cnt_length = col_cnt.to_s.size  # get position of number, ex: 9->1, 12->2, 132->3
+      # calc size of each key per position, 1, 10x, 100x,... ex: 123 -> calc size 1-9 and 10-99
+      1.upto(col_cnt_length-1){|i| col_size_for_key += ( i * (10**i - (10**(i-1))) )}
+      # calc size of the key for the hight position
+      col_size_for_key += (col_cnt - 10**(col_cnt_length-1) + 1) * col_cnt_length
+      col_size = col_size_for_static + col_size_for_key
+      BASE_DATA_SIZE + table_name.size + col_size
+    end
+  end
+
+end
+end
+

data/lib/flydata/query_based_sync/resource_requester.rb

@@ -0,0 +1,70 @@
+require 'flydata-core/logger'
+
+module Flydata
+module QueryBasedSync
+
+class ResourceRequester
+  include FlydataCore::Logger
+
+  def initialize(context)
+    @context = context
+    @resource_client = create_resource_client(context)
+  end
+
+  attr_reader :context
+
+  # Start fetching resources by keeping a same connection
+  # Caller needs to call methods using the connection with block
+  #
+  # ex:
+  #   requester.start do |req|
+  #     req.fetch_resources(table_name) do |response|
+  #       handle(response)
+  #     end
+  #   end
+  def start
+    context.table_meta.reload(@resource_client)
+    yield self
+  ensure
+    @resource_client.close
+  end
+
+  # Fetch resources for a table
+  # It may call the callback multiple times in the following cases
+  #   - The resource size is bigger than the max size per request
+  #   - Resume to fetch resources if resume info exists in per-table position file
+  def each_response(table_name, interval = 1)
+    latest_src_pos = context.source_pos_class.new(context.table_meta.current_snapshot)
+
+    loop do
+      responses = fetch_responses_once(table_name, latest_src_pos)
+      break if responses.nil? || responses.empty?
+      responses.each do |response|
+        yield response
+      end
+      break if responses.last.new_source_pos >= latest_src_pos
+      sleep interval  # to avoid rush
+    end
+
+    nil
+  end
+
+  # Override
+  # Return a resource client object
+  # ex: pg_client, mysql_connection
+  def create_resource_client(context)
+    raise "Not implemented"
+  end
+
+  # Override
+  # Fetch the resource and return a response object
+  # Returned object must be kind of Flydata::QueryBasedSync::Response
+  def fetch_responses_once(table_name, latest_src_pos)
+    raise "Not implemented"
+  end
+
+end
+
+end
+end
+

data/lib/flydata/query_based_sync/response.rb

@@ -0,0 +1,122 @@
+require 'flydata/query_based_sync/record_size_estimator'
+
+module Flydata
+module QueryBasedSync
+
+class Response
+
+  DEFAULT_EMIT_CHUNK_LIMIT = 64 * (1024**2) # 64mb
+
+  def initialize(context, table_name, records, query_cond = {})
+    @table_name = table_name
+    @records = records
+    @context = context
+    @query_cond = query_cond
+
+    @table_meta = context.table_meta[table_name.to_sym]
+  end
+
+  attr_reader :context, :table_name, :src_result, :query_cond
+
+  # Set records for sending as flydata sync records
+  # The format is an array of Hash objects
+  # {"1"=>"1000", "2"=>"Andy", ....}
+  attr_reader :records
+
+  # Store the source position for the resume
+  # This needs to be saved at the per-table binlog.pos after sending the records
+  attr_reader :new_source_pos
+
+  # Set per-table source position for resume
+  def set_new_source_pos(required_pk_values = false)
+    pk_values =
+      # Set pk_values if next_response exists or returned row size is the same as the specified limit size.
+      # It is for resuming in the middle of the transaction.
+      if required_pk_values
+        values = []
+        pk_positions = @table_meta[:pk_positions]
+        @table_meta[:primary_keys].each_with_index do |pk, index|
+          pk_pos = pk_positions[index]
+          values << { pk.to_s => @records.last[pk_pos.to_s] }
+        end
+        values
+      end
+
+    args =
+      if pk_values
+        [@query_cond[:from_sid], @query_cond[:to_sid], pk_values]
+      else
+        [@query_cond[:to_sid]]
+      end
+
+    @new_source_pos = @context.source_pos_class.new(*args)
+  end
+
+  def record_count
+    records.nil? ? 0 : records.count
+  end
+
+  def empty?
+    record_count == 0
+  end
+
+  def self.create_responses(context, table_name, raw_result, query_cond = {})
+    emit_chunk_limit = context.params[:emit_chunk_limit] || DEFAULT_EMIT_CHUNK_LIMIT
+    table_meta = context.table_meta[table_name.to_sym]
+    record_size_estimator = RecordSizeEstimator.new(table_name, table_meta[:columns].count)
+
+    all_records = convert_result_to_hash(raw_result)
+    responses = []
+
+    if all_records.nil? || all_records.empty?
+      res = self.new(context, table_name, [], query_cond)
+      res.set_new_source_pos
+      return [res]
+    end
+
+    start_index = 0
+    cur_chunk_size = 0
+
+    # Split records and create response object
+    all_records.each_with_index do |r, i|
+      record_size = record_size_estimator.calc_record_size(r)
+      # Set another response object, if the chunk size exceeds the emit_chunk_limit
+      if cur_chunk_size > 0 && (cur_chunk_size + record_size) > emit_chunk_limit
+        responses << self.new(context, table_name, all_records[start_index..i-1], query_cond)
+        start_index = i
+        cur_chunk_size = record_size
+      else
+        cur_chunk_size += record_size
+      end
+    end
+
+    if cur_chunk_size > 0
+      responses << self.new(context, table_name, all_records[start_index..-1], query_cond)
+    end
+
+    # Set new source positions for each response
+    responses[0..-2].each{|res| res.set_new_source_pos(true)}
+    more_response = (all_records.size == table_meta[:max_num_rows_per_query])
+    responses.last.set_new_source_pos(more_response)
+
+    responses
+  end
+
+  # Convert to sync record style
+  # ex)
+  #   input:   {"id"=>1, "name"=>"Andy"}
+  #   output:  {"1"=>1, "2"=>"Andy"}
+  def self.convert_result_to_hash(result)
+    first_record = result.first
+    return result if first_record.kind_of?(Hash) && first_record.has_key?('1')
+
+    result.collect do |r|
+      h = {}
+      r.values.each.with_index(1){|v, i| h[i.to_s] = v }
+      h
+    end
+  end
+end
+
+end
+end

data/lib/flydata/query_based_sync/response_handler.rb

@@ -0,0 +1,30 @@
+require 'flydata/plugin_support/sync_record_emittable'
+
+module Flydata
+module QueryBasedSync
+
+  class ResponseHandler
+    include Flydata::PluginSupport::SyncRecordEmittable
+
+    def initialize(context)
+      @context = context
+    end
+
+    def handle(response)
+      records = response.records.collect{|r| {row: r} }
+
+      emit_sync_records(records,
+                        type: :update,  # upsert only
+                        src_pos: response.new_source_pos.to_s,
+                        table: response.table_name)
+
+      # TODO: Revisit and check if transaction management is needed
+      # However, since we use 'upsert' for query based sync,
+      # duplicate event will not cause an issue.
+      context.table_src_pos_files[response.table_name.to_sym].
+        save(response.new_source_pos)
+    end
+  end
+
+end
+end

data/lib/flydata/source/sync_generate_table_ddl.rb

@@ -41,7 +41,7 @@ class SyncGenerateTableDdl < Component
     if missing_tables
       missing_tables.each {|missing_table| error_list << { error: "table does not exist in the #{data_source_type_display_name}", table: missing_table } }
     end
- 
+
     [flydata_tabledefs, error_list]
   end
 

data/lib/flydata/source_mysql/plugin_support/binlog_record_dispatcher.rb

@@ -24,14 +24,14 @@ module PluginSupport
 
   class FlydataBinlogRecordDispatcher < BinlogRecordDispatcher
     def initialize(context)
-      context.current_binlog_file = ""
+      context.cur_src_pos_file = ""
       @context = context
       @query_dispatcher = FlydataBinlogQueryDispatcher.new(context)
       @dml_record_handler = DmlRecordHandler.new(context)
     end
 
     def on_rotate(record)
-      @context.current_binlog_file = record["binlog_file"]
+      @context.cur_src_pos_file = record["binlog_file"]
     end
 
     def on_write_rows(record)

data/lib/flydata/source_mysql/plugin_support/binlog_record_handler.rb

@@ -16,19 +16,13 @@ module PluginSupport
       @table_meta = @context.table_meta
 
       # Load per-table binlog position
-      @table_binlog_pos = {}
-      @context.tables.each do |table_name|
-        table_binlog_pos_str = @context.sync_fm.get_table_source_raw_pos(table_name)
-        if table_binlog_pos_str
-          @table_binlog_pos[table_name] = FlydataCore::Mysql::BinlogPos.new(table_binlog_pos_str)
-        end
-      end
+      @table_binlog_pos = @context.table_binlog_pos
     end
 
     private
 
     def binlog_pos(record)
-      "#{@context.current_binlog_file}\t#{record['next_position'] - record['event_length']}"
+      "#{@context.cur_src_pos_file}\t#{record['next_position'] - record['event_length']}"
     end
 
     def supported_database
@@ -81,7 +75,7 @@ module PluginSupport
 
     def check_empty_binlog
       #Log one warning per consecutive records that have empty binlog filename
-      if @context.current_binlog_file.to_s.empty?
+      if @context.cur_src_pos_file.to_s.empty?
         if @first_empty_binlog
           $log.warn "Binlog file name is empty. Rotate event not received!"
           @first_empty_binlog = false

data/lib/flydata/source_mysql/plugin_support/context.rb

@@ -1,7 +1,31 @@
 require 'flydata/plugin_support/context'
+require 'flydata-core/mysql/binlog_pos'
+
+module Flydata
+module SourceMysql
+module PluginSupport
 
-module Flydata::SourceMysql::PluginSupport
   class Context < ::Flydata::PluginSupport::Context
-    register_mandatory_opts :database, :table_meta
+    register_mandatory_opts :database
+
+    def initialize(opts)
+      super
+      set_table_binlog_pos
+    end
+
+    attr_reader :table_binlog_pos
+
+    def set_table_binlog_pos
+      @table_binlog_pos = {}
+      tables.each do |table_name|
+        table_binlog_pos_str = sync_fm.get_table_source_raw_pos(table_name)
+        if table_binlog_pos_str
+          @table_binlog_pos[table_name] = ::FlydataCore::Mysql::BinlogPos.load(table_binlog_pos_str)
+        end
+      end
+    end
   end
+
+end
+end
 end