flydata 0.6.3 → 0.6.4

data/lib/flydata/command/stop.rb

@@ -8,6 +8,7 @@ module Flydata
       def self.slop
         Slop.new do
           on 'f', 'full', 'Stop all the processes'
+          on 'force-run', 'Run forcefully, ignoring exclusive run info'
         end
       end
       def run

data/lib/flydata/command/sync.rb

@@ -1,5 +1,4 @@
 require 'msgpack'
-require 'mysql2'
 require 'rest_client'
 require 'sys/filesystem'
 require 'flydata/agent'
@@ -8,17 +7,16 @@ require 'flydata/command/conf'
 require 'flydata/command/sender'
 require 'flydata/compatibility_check'
 require 'flydata/errors'
+require 'flydata/source/errors'
 require 'flydata/helpers'
 require 'flydata/json'
 require 'flydata/queueable_thread'
 require 'flydata/output/forwarder'
-require 'flydata/parser/mysql/dump_parser'
+require 'flydata/parser'
 require 'flydata/preference/data_entry_preference'
 require 'flydata/sync_file_manager'
 require 'flydata-core/table_def'
-require 'flydata-core/mysql/binlog_pos'
-require 'flydata/mysql/table_ddl'
-require 'flydata-core/mysql/command_generator'
+require 'flydata/table_ddl'
 require 'flydata/event/api_event_sender'
 require 'flydata-core/event/event_dictionary'
 require 'sigdump/setup'
@@ -41,7 +39,7 @@ module Flydata
       STATUS_COMPLETE = 'COMPLETE'
 
       attr_reader :full_initial_sync,     # true if full initial sync
-                  :full_tables,           # all tables (same as data_entry['mysql_data_entry_preference']['tables'])
+                  :full_tables,           # all tables (same as the value of `tables` data entry preference)
                   :new_tables,            # tables which is not finihed initial-sync(pos file doesn't exist)
                   :ddl_tables,            # tables generated ddl
                   :input_tables           # tables which user put
@@ -49,9 +47,6 @@ module Flydata
                   #target_tables          # target tables for current command(sync/reset/generate_table_ddl)
                   #target_tables_for_api  # target tables for calling api(tables parameter needs to be empty for full_initial_sync)
 
-      class SyncDataEntryError < StandardError
-      end
-
       # Command: flydata sync
       #  - Arguments
       def self.slop
@@ -59,8 +54,8 @@ module Flydata
           on 'c', 'skip-cleanup', 'Skip server cleanup'
           on 'f', 'skip-flush', 'Skip server flush'
           on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
-          on 'd', 'dump-file', 'Dump mysqldump into a file. Use this for debugging after making sure the free space.' # dummy for compatibility
-          on 's', 'dump-stream', 'Dump mysqldump stream instead of saving dump file. It might cause timeout error if db size is larger than 10GB.'
+          on 'd', 'dump-file', 'Save dump result into a file.'
+          on 's', 'dump-stream', 'Stream dump result to a pipe instead of saving dump file. It might cause timeout error if db size is larger than 10GB.'
           on 'n', 'no-flydata-start', 'Don\'t start the flydata agent after initial sync.'
           #TODO : This option is temp! Should remove soon.
           on 'ff', 'Skip checking query queue and flush'
@@ -76,10 +71,9 @@ module Flydata
 
       # Public method
       #  - Called from Sender#start/restart
-      def try_mysql_sync(options)
-        # Start initial sync
-        handle_mysql_sync(nil, options)
-      rescue SyncDataEntryError
+      def try_initial_sync(options)
+        handle_initial_sync(nil, options) if source.sync.supported?
+      rescue Source::UnsupportedSourceError
         return
       end
 
@@ -89,6 +83,7 @@ module Flydata
         Slop.new do
           on 'f', 'skip-flush', 'Skip server flush'
           on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
+          on 'force-run', 'Run forcefully, ignoring exclusive run info'
         end
       end
 
@@ -118,6 +113,8 @@ EOS
           on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
           on 'a', 'all', 'Resets Sync for all tables'
           on 'i', 'init', 'Resets unfinished initial sync'
+          on 'force-run', 'Run forcefully, ignoring exclusive run info'
+          on 'f', 'force', "Resets tables including append only tables.  Use this option only when you want to stop sync for the append only table permanently."
         end
       end
 
@@ -143,6 +140,9 @@ EOS
           reset_init = false
         end
         sync_resumed = set_current_tables(tables, resume: !opts[:all])
+        target_tables = opts[:all] ? @full_tables : @input_tables
+        target_append_only_tables = target_tables & @append_only_tables
+        target_full_sync_tables = target_tables - @append_only_tables
 
         return if !sync_resumed && reset_init
 
@@ -152,6 +152,26 @@ EOS
 ERROR!  You cannot reset tables because the previous initial sync has not been completed.  Reset the unfinished initial sync first with the following command:
 
   flydata sync:reset --init
+
+EOS
+          return
+        end
+
+        if target_append_only_tables.size > 0 && !opts[:force]
+          log_info_stdout <<EOS
+ERROR!  Reset failed because it includes append only table(s).  Sync can no longer continue if you reset an append only table.
+
+  Append only table(s): #{target_append_only_tables.join(", ")}
+
+If you really want to reset append-only tables, use '--force' option.
+
+EOS
+
+          log_info_stdout <<EOS unless target_full_sync_tables.empty?
+To reset all tables except for append only tables, run the following command.
+
+  flydata sync:reset #{target_full_sync_tables.join(" ")}
+
 EOS
           return
         end
@@ -203,11 +223,11 @@ EOS
       def skip
         de = data_entry
         sync_fm = create_sync_file_manager(de)
-        binlog_path = sync_fm.binlog_path
+        source_pos_path = sync_fm.source_pos_path
         sync_fm.close
-        `touch #{binlog_path}`
-        log_info_stdout("Created an empty binlog position file.")
-        log_info_stdout("-> #{binlog_path}")
+        `touch #{source_pos_path}`
+        log_info_stdout("Created an empty source position file.")
+        log_info_stdout("-> #{source_pos_path}")
         log_info_stdout("Run 'flydata start' to start continuous sync.")
       end
       run_exclusive :skip
@@ -220,6 +240,7 @@ EOS
           on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
           on 's', 'skip-primary-key-check', 'Skip primary key check when generating DDL'
           on 'all-tables', 'Generate all table schema'
+          #no 'force-run' option. because stdout is often redirected to a file.
         end
       end
 
@@ -228,70 +249,22 @@ EOS
       def generate_table_ddl(*tables)
         # Compatibility check
         de = data_entry
-        dp = flydata.data_port.get
-        Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference']).check
+        context = source.sync_generate_table_ddl(flydata.data_port.get, opts)
+        context.run_compatibility_check
 
         # Set instance variables
         set_current_tables(tables, include_all_tables: true)
 
-        do_generate_table_ddl(de)
+        do_generate_table_ddl(context, de)
       end
       run_exclusive :generate_table_ddl
 
-      # Command: flydata sync:fix_binlogpos
-      #  - Arguments
-      def self.slop_fix_binlogpos
-        Slop.new do
-          on 'f', 'force', 'update sent binlog position file forcibly'
-        end
-      end
-
-      # Command: flydata sync:fix_binlogpos
-      # Set binlog path
-      #  - Entry method
-      def fix_binlogpos
-        de = data_entry
-        sync_fm = create_sync_file_manager(de)
-
-        if File.exists?(sync_fm.sent_binlog_path) && !opts.force?
-          log_info_stdout("Skip creating sent binlogpos because sent position file is exist already. (#{sync_fm.sent_binlog_path})")
-          return
-        end
-
-        if Flydata::Command::Sender.new.process_exist?
-          log_warn_stderr("flydata is running. flydata process needs to be stopped with 'flydata stop'.")
-          return
-        end
-
-        binlog_info = sync_fm.load_binlog
-        if binlog_info.nil?
-          log_info_stdout("Skip creating sent binlogpos because binlog position file is empty or invalid. (#{sync_fm.sent_binlog_path})")
-          return
-        end
-
-        say("Updating binlog position files...")
-        log_info("Updating binlog position files... Original binlog_info:#{binlog_info}")
-
-        # Update binlog.sent.pos file
-        #   -1 is because the position in binlog.pos is the next event's position.
-        #   on the other hand the position in sent position indicates already processed.
-        binlog_info[:pos] -= 1
-        log_info("Updating sent position file. #{binlog_info} -> #{sync_fm.sent_binlog_path}")
-        sync_fm.save_sent_binlog(binlog_info)
-
-        # Update binlog.pos file to start from head of the current binlog file
-        new_binlog_info = binlog_info.dup.tap{|h| h[:pos] = 4}  # 4 is the first position of binlog file.
-        log_info("Updating original position file. #{new_binlog_info} -> #{sync_fm.binlog_path}")
-        sync_fm.save_binlog(new_binlog_info)
-        log_info_stdout("Done!")
-      end
-      run_exclusive :fix_binlogpos
-
       # Command: flydata sync:repair
       #  - Arguments
       def self.slop_repair
         Slop.new do
           on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
+          on 'force-run', 'Run forcefully, ignoring exclusive run info'
         end
       end
 
@@ -301,7 +274,8 @@ EOS
       run_exclusive :repair
 
       def check(options = {})
-        status, pos_mismatch_tables, gap_tables = _check(options)
+        context = source.source_pos
+        status, pos_mismatch_tables, gap_tables = _check(context, options)
 
         if status.include? :OK
           message = "\nNo errors are found.  Sync is clean.\n"
@@ -349,7 +323,7 @@ EOS
         JSON.generate(h)
       end
 
-      def _check(options = {})
+      def _check(source_pos_context, options = {})
         options[:stop_agent] ||= false
 
         set_current_tables
@@ -361,14 +335,15 @@ EOS
         abnormal_shutdown = false
         begin
           begin
-            flush_buffer_and_stop(@full_tables, force: false, timeout: 7200) # 2 hrs.  It can take more but 2 hours are long enough to pull someone's attention.
+            flush_buffer_and_stop(@full_tables, force: false, timeout: 55) # A short timeout is set.  Otherwise, the check command and the repair command takes forever to complete.
           rescue ServerDataProcessingTimeout => e
             data_stuck_at = e.state
           end
 
           # Agent is stopped but locked.  There was an abnormal shutdown.
           abnormal_shutdown = sender.agent_locked?
-          table_status_hash = get_table_status(@full_tables)
+          table_status_hash = get_table_status(@full_tables, source_pos_context)
+          corrupt_master_pos_files = check_master_position_files
           pos_mismatch_tables = check_position_files(table_status_hash)
           gap_tables = check_gaps(table_status_hash)
         ensure
@@ -382,6 +357,9 @@ EOS
         if data_stuck_at == :UPLOAD
           status << :STUCK_AT_UPLOAD
         end
+        if corrupt_master_pos_files
+          status << :CORRUPT_MASTER_POS
+        end
         if gap_tables
           status << :TABLE_GAPS
         end
@@ -394,15 +372,18 @@ EOS
         if status.empty?
           status << :OK
         end
-        [status, pos_mismatch_tables, gap_tables, table_status_hash]
+        [status, corrupt_master_pos_files, pos_mismatch_tables, gap_tables, table_status_hash]
       end
 
       def _repair
         de = data_entry
+        sync_fm = create_sync_file_manager(de)
+        context = source.source_pos
         set_current_tables
         # Stop agent.  Check sync and make sure the state is :STUCK_AT_UPLOAD
         # Get table status for the tables.
-        status, pos_mismatch_tables, gap_tables, table_status_hash = _check(stop_agent:true)
+        status, corrupt_master_pos_files, pos_mismatch_tables, gap_tables, table_status_hash = _check(context, stop_agent:true)
+
         if status.include? :STUCK_AT_PROCESS
           e = AgentError.new("Data is stuck while processing")
           e.description = <<EOS
@@ -417,6 +398,17 @@ EOS
           return
         end
 
+        if status.include?(:ABNORMAL_SHUTDOWN) && status.uniq.length == 1
+          # abnormal shutdown occurred, but all other conditions are healthy.
+          log_info_stdout ""
+          log_info_stdout "Agent process was shut down unexpectedly"
+          log_info_stdout "but sync is in good condition.  Nothing to repair."
+
+          # Remove the lock file if exists.
+          File.delete(FLYDATA_LOCK) if File.exists?(FLYDATA_LOCK)
+          return
+        end
+
         gt = []
         pt = []
         gt = gap_tables.collect{|bt| bt[:table] } if gap_tables
@@ -433,7 +425,11 @@ EOS
      #{tables.join("\n     ")}
 
 EOS
+        log_info_stdout <<EOS if corrupt_master_pos_files
+ - Sync is broken due to corrupt master position files
+     #{corrupt_master_pos_files.join("\n     ")}
 
+EOS
         log_info_stdout <<EOS if status.include? :ABNORMAL_SHUTDOWN
  - Agent process was not shut down correctly.  Files may be corrupt.
 
@@ -441,74 +437,73 @@ EOS
 
         return unless ask_yes_no("Proceed?")
 
-        oldest_binlog = get_oldest_available_binlog
+        oldest_source_pos = get_oldest_available_source_pos
         unrepairable_tables = []
-        # Determine the master binlog positions
-        sent_binlog_pos = nil
+        # Determine the master source positions
+        sent_source_pos = nil
         @full_tables.each do |table|
-          binlog_str = table_status_hash[table]["src_pos"]
-          if binlog_str.nil?  || binlog_str == "-"
+          table_source_pos = table_status_hash[table]["source_pos"]
+
+          if table_source_pos.nil? || table_source_pos.empty?
             # the table status has no src_pos (which is rare.)  Skip the table
             next
           end
-          binlog_pos = FlydataCore::Mysql::BinlogPos.new(binlog_str)
           if tables.empty?
-            if sent_binlog_pos.nil? || sent_binlog_pos < binlog_pos
-              sent_binlog_pos = binlog_pos
+            if sent_source_pos.nil? || sent_source_pos < table_source_pos
+              sent_source_pos = table_source_pos
             end
           else
             if tables.include?(table)
-              if sent_binlog_pos.nil? || sent_binlog_pos > binlog_pos
-                if oldest_binlog && binlog_pos < oldest_binlog
+              if sent_source_pos.nil? || sent_source_pos > table_source_pos
+                if oldest_source_pos && table_source_pos < oldest_source_pos
                   unrepairable_tables << table
                 else
-                  sent_binlog_pos = binlog_pos
+                  sent_source_pos = table_source_pos
                 end
               end
             end
           end
         end
-        # if sent_binlog_pos is nil, it means sync has started for none of tables.  No need
+        # if sent_source_pos is nil, it means sync has started for none of tables.  No need
         # to repair positions nor clean buffer data.
-        if sent_binlog_pos
-          if oldest_binlog && sent_binlog_pos < oldest_binlog
-            e = AgentError.new("Repair failed due to expired binlog")
+        if sent_source_pos
+          if oldest_source_pos && sent_source_pos < oldest_source_pos
+            e = AgentError.new("Repair failed due to expired source position")
             e.description = <<EOS
-  Repair failed because the starting binlog position `#{sent_binlog_pos} no longer exists.  Run full initial sync instead.
+  Repair failed because the starting source position `#{sent_source_pos} no longer exists.  Run full initial sync instead.
 EOS
             raise e
           end
-          master_binlog_pos = FlydataCore::Mysql::BinlogPos.new(sent_binlog_pos.filename, 4)
-  
+          master_source_pos = context.resume_pos(sent_source_pos)
+
           # Delete agent buffer
           log_info_stdout "Deleting data in the agent buffer..."
           files = Flydata::Agent.new(FLYDATA_HOME).delete_buffer_files
           unless files.empty?
             $log.debug "Deleted buffer files\n  " + files.join("\n  ")
           end
-  
+
           # Delete query queue items for the tables
           log_info_stdout "Deleting data stuck in the server buffer..."
           cleanup_sync_server(de, tables, queue_only: true) unless tables.empty?
-  
-          # Save the positions (binlog and seq)
+
+          # Save the positions (source_pos and seq)
           log_info_stdout "Fixing table positions..."
           @full_tables.each do |table|
-             binlog_str = table_status_hash[table]["src_pos"]
-             if binlog_str.nil? || binlog_str == "-"
+             table_source_pos = table_status_hash[table]["source_pos"]
+             if table_source_pos.nil? || table_source_pos.empty?
                # no init sync has happened to the table.  No need to set positions.
                next
              end
-             binlog_pos = FlydataCore::Mysql::BinlogPos.new(binlog_str)
              pos = table_status_hash[table]["seq"]
-             old_binlog_pos, old_pos = save_table_positions(table, binlog_pos, pos)
+             old_source_pos, old_pos = save_table_positions(table, table_source_pos, pos, context)
              if pos.to_i != old_pos.to_i && !tables.include?(table)
                $log.debug "Fixed broken table position.  table:#{table} pos:#{old_pos} -> #{pos}"
              end
           end
-  
+
           log_info_stdout "Fixing the master position files..."
-          save_master_binlog_positions(master_binlog_pos, sent_binlog_pos)
+          save_master_source_positions(master_source_pos, sent_source_pos, context)
         end
 
         # Remove the lock file if exists.
@@ -519,13 +514,11 @@ EOS
 
       # Initial sync
 
-      def handle_mysql_sync(tables = nil, options = {})
+      def handle_initial_sync(tables = nil, options = {})
         unless Flydata::Command::Sender.new.wait_until_server_ready
           raise Timeout::Error, "Timed out to wait for the server side to become active. Pleae try again later."
         end
 
-        de = data_entry
-
         # Setup instance variables
         sync_resumed = set_current_tables(tables, resume: true)
 
@@ -533,7 +526,7 @@ EOS
           # skip confirmation prompts and resume sync right away.
           # #initial_sync knows where to resume from.
           log_info_stdout("Resuming the initial sync...")
-          initial_sync(de, options.merge(sync_resumed: true))
+          initial_sync(options.merge(sync_resumed: true))
         elsif !@unsynced_tables.empty?
           show_purpose_name
           unsynced_table_message = "We've noticed that these tables have not been synced yet: #{@unsynced_tables.join(", ")}\n"
@@ -546,7 +539,7 @@ EOS
           end
           log_info_stdout(unsynced_table_message)
           if ask_yes_no("Do you want to run initial sync on all of these tables now?")
-            initial_sync(de, options.merge(sync_resumed: false))
+            initial_sync(options.merge(sync_resumed: false))
           else
             #If generate_table_ddl has not been run for these tables, warn user
             unless @ddl_tables.empty?
@@ -559,7 +552,8 @@ EOS
         end
       end
 
-      def initial_sync(de, opt)
+      def initial_sync(opt)
+        de = data_entry
         # Load sync information from file
         validate_initial_sync_status
         begin
@@ -576,7 +570,7 @@ EOS
             # flush is unnecessary for full initial sync because it's guaranteed
             # that agent is stopped with no leftover buffer.
           end
-          sync_mysql_to_redshift(de, opt)
+          perform_initial_sync(de, opt)
         rescue ServerDataProcessingTimeout => e
           ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
           ee.description = <<EOS
@@ -593,7 +587,7 @@ EOS
       end
 
       # Don't call the method unless init sync needs to be run for a table
-      def sync_mysql_to_redshift(de, options = {})
+      def perform_initial_sync(de, options = {})
         dp = flydata.data_port.get
         sync_fm = create_sync_file_manager(de)
 
@@ -602,60 +596,54 @@ EOS
           Flydata::Command::Conf.new.copy_templates
         end
 
-        generate_mysqldump(de, sync_fm, !opts.dump_stream?, options) do |mysqldump_io, binlog_pos, db_bytesize|
-          parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, binlog_pos, db_bytesize)
+        generate_source_dump(de, sync_fm, !opts.dump_stream?, options) do |dump_io, source_pos|
+          parse_dump_and_send(dump_io, dp, de, sync_fm, source_pos)
         end
-        complete_mysqldump_processing(dp, de, sync_fm)
+        complete_dump_processing(sync_fm)
       end
 
-      def generate_mysqldump(de, sync_fm, file_dump = true, options = {},
-                             &block)
-        # validate parameter
-        %w(host username database).each do |k|
-          if de['mysql_data_entry_preference'][k].to_s.empty?
-            raise "'#{k}' is required. Set the value in the conf file " +
-                  "-> #{Flydata::Preference::DataEntryPreference.conf_path(de)}"
-          end
-        end
+      def generate_source_dump(de, sync_fm, file_dump = true, options = {},
+                               &dump_ready_callback)
+        dp = flydata.data_port.get
+        context = source.generate_source_dump(dp, options)
 
         # Status is parsed or complete -> skip dump and parse
         dump_pos_info = sync_fm.load_dump_pos
         if dump_pos_info[:status] == STATUS_PARSED || dump_pos_info[:status] == STATUS_COMPLETE
-          initialize_binlog_positions_and_call_callback(
-                  nil, options[:binlog_ready_callback], sync_fm)
+          initialize_source_positions_and_call_callback(
+                  nil, options[:source_pos_ready_callback], sync_fm)
+          return
         end
 
-        # mysqldump file exists -> skip dump
-        dp = flydata.data_port.get
+        # dump file exists -> skip dump
         fp = sync_fm.dump_file_path
         if file_dump && File.exists?(fp) && File.size(fp) > 0
-          initialize_binlog_positions_and_call_callback(
-                  nil, options[:binlog_ready_callback], sync_fm)
-          return call_block_or_return_io(fp, &block)
+          initialize_source_positions_and_call_callback(
+                  nil, options[:source_pos_ready_callback], sync_fm)
+          return call_block_or_return_io(fp, &dump_ready_callback)
         end
 
-        log_info_stdout("Checking MySQL server connection and configuration...")
-        Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], dump_dir: fp, backup_dir: sync_fm.backup_dir).check
+        log_info_stdout("Checking the data source connection and configuration...")
+        context.run_compatibility_check(fp, sync_fm.backup_dir)
         log_info_stdout("Checking the database size...")
 
-        db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).get_db_bytesize
+        db_bytesize = context.dump_size(target_tables)
 
         tables = target_tables
         tables ||= '<all tables>'
-        data_servers = de['mysql_data_entry_preference']['data_servers'] ? "\n  data servers: #{de['mysql_data_entry_preference']['data_servers']}" : ""
+        data_servers = source.sync.data_servers
+        data_servers = data_servers ? "\n  data servers: #{data_servers}" : ""
 
         confirmation_text = <<-EOM
 
 FlyData Sync will start synchronizing the following database tables
-  host:         #{de['mysql_data_entry_preference']['host']}
-  port:         #{de['mysql_data_entry_preference']['port']}
-  username:     #{de['mysql_data_entry_preference']['username']}
-  database:     #{de['mysql_data_entry_preference']['database']}
-  tables:       #{tables.join(", ")}#{data_servers}
 EOM
+        context.confirmation_items.each do |name, value|
+          confirmation_text << "  %-14s%s\n" % ["#{name}:", value.to_s]
+        end
 
-        confirmation_text << <<-EOM if de['mysql_data_entry_preference']['ssl_ca']
-  ssl:          Yes
+        confirmation_text << <<-EOM
+  tables:       #{tables.join(", ")}#{data_servers}
 EOM
         confirmation_text << <<-EOM if file_dump
   dump file:    #{fp}
@@ -674,7 +662,7 @@ EOM
               log_warn_stderr("!!WARNING There may not be enough disk space for a DB dump. We recommend 1GB free disk space after the dump. free disk space:#{as_size(free_disk_bytesize)}(#{free_disk_bytesize} byte) /" +
                               " db size:#{as_size(db_bytesize)}(#{db_bytesize} byte)")
               unless ask_yes_no('Do you want to continue?')
-                log_warn_stderr("To change the dump file directory, delete '#' and modify the path of 'mysqldump_dir:' in '#{Preference::DataEntryPreference.conf_path(de)}'")
+                log_warn_stderr("To change the dump file directory, delete '#' and modify the path of 'dump_dir:' in '#{Preference::DataEntryPreference.conf_path(de)}'")
                 exit 1
               end
             end
@@ -692,18 +680,17 @@ EOM
               data_entry_id: de['id'],
               data_port_id: de['data_port_id'])
           end
-          dump_generator = Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.new(de['mysql_data_entry_preference'].merge('tables' => target_tables))
           if file_dump
-            binlog_pos = nil
+            source_pos = nil
             begin
-              dump_generator.dump(fp) do |_io, _binlog_pos|
-                binlog_pos = _binlog_pos
-                initialize_binlog_positions_and_call_callback(binlog_pos, options[:binlog_ready_callback], sync_fm)
+              context.dump(target_tables, fp) do |_io, _source_pos|
+                source_pos = _source_pos
+                initialize_source_positions_and_call_callback(source_pos, options[:source_pos_ready_callback], sync_fm)
               end
               log_info_stdout("  -> Database dump done")
             rescue Exception => e
               #Catch all exceptions including SystemExit and Interrupt.
-              log_info_stdout "Quit while running mysqldump, deleting dump file..."
+              log_info_stdout "Quit while running dump, deleting dump file..."
               sync_fm.delete_dump_file
               log_info_stdout "Dump file deleted. To restart the FlyData Agent, Following tables may need reset : #{target_tables}"
               if (target_tables.nil? or target_tables.empty?)
@@ -713,11 +700,11 @@ EOM
               end
               raise e
             end
-            call_block_or_return_io(fp, binlog_pos, &block)
+            call_block_or_return_io(fp, source_pos, &dump_ready_callback)
           else
-            dump_generator.dump do |io, binlog_pos|
-              initialize_binlog_positions_and_call_callback(binlog_pos, options[:binlog_ready_callback], sync_fm)
-              block.call(io, binlog_pos, db_bytesize)
+            context.dump(target_tables) do |io, source_pos|
+              initialize_source_positions_and_call_callback(source_pos, options[:source_pos_ready_callback], sync_fm)
+              dump_ready_callback.call(io, source_pos)
             end
           end
         else
@@ -725,16 +712,16 @@ EOM
         end
       end
 
-      def initialize_binlog_positions_and_call_callback(binlog_pos, callback, sync_fm)
-        if binlog_pos
-          initialize_positions(sync_fm, binlog_pos)
+      def initialize_source_positions_and_call_callback(source_pos, callback, sync_fm)
+        if source_pos
+          initialize_positions(sync_fm, source_pos)
         else
-          # no binlog_pos was given because dump was completed in the
+          # no source_pos was given because dump was completed in the
           # previous init sync attempt.  Position files must be there already
           # so no initialization is necessary.
         end
         if callback
-          callback.call(binlog_pos)
+          callback.call
         end
       end
 
@@ -744,11 +731,11 @@ EOM
         stat.block_size * stat.blocks_available
       end
 
-      def call_block_or_return_io(fp, binlog_pos = nil, &block)
-        if block
+      def call_block_or_return_io(fp, source_pos= nil, &source_pos_block)
+        if source_pos_block
           f_io = open_file_io(fp)
           begin
-            block.call(f_io, binlog_pos)
+            source_pos_block.call(f_io, source_pos)
             return nil
           ensure
             f_io.close rescue nil
@@ -781,35 +768,35 @@ EOM
       #   <- checkpoint
       #...
       #CREATE TABLE ...
-      def parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, binlog_pos, db_bytesize = nil)
+      def parse_dump_and_send(dump_io, dp, de, sync_fm, source_pos)
+
+        context = source.parse_dump_and_send
         # Prepare forwarder
         de_tag_name = de["tag_name#{env_suffix}"]
         server_port = dp['server_port']
-        servers = if de['mysql_data_entry_preference']['data_servers']
-                    de['mysql_data_entry_preference']['data_servers'].split(',')
-                  else
-                    dp["servers#{env_suffix}"].collect{|s| "#{s}:#{server_port}"}
-                  end
-        forwarder_type = de['mysql_data_entry_preference']['forwarder'] ||
+        data_servers = source.sync.data_servers
+        servers = data_servers ?  data_servers.split(',') :
+                   dp["servers#{env_suffix}"].collect{|s| "#{s}:#{server_port}"}
+        forwarder_type = source.sync.forwarder ||
           (dp['ssl_enabled'] ? 'sslforwarder' : 'tcpforwarder')
         forwarder = Flydata::Output::ForwarderFactory.create(forwarder_type, de_tag_name, servers)
 
         # Load dump.pos file for resume
         dump_pos_info = sync_fm.load_dump_pos
-        option = dump_pos_info || {}
-        if option[:table_name] && option[:last_pos].to_i != -1
-          binlog_pos = option[:binlog_pos]
-          log_info_stdout("Resuming... Last processed table: #{option[:table_name]}")
+        dump_pos_info ||= {}
+        if dump_pos_info[:table_name] && dump_pos_info[:last_pos].to_i != -1
+          source_pos = dump_pos_info[:source_pos]
+          log_info_stdout("Resuming... Last processed table: #{dump_pos_info[:table_name]}")
         else
-          option[:binlog_pos] = binlog_pos
+          dump_pos_info[:source_pos] = source_pos
         end
-        if option[:mysql_table] && option[:mysql_table].value_converters.nil?
-          # Old marshal dumped mysql_table object may not have value_converters
-          option[:mysql_table].set_value_converters(FlydataCore::TableDef::MysqlTableDef::VALUE_CONVERTERS)
+        if dump_pos_info[:source_table] && dump_pos_info[:source_table].value_converters.nil?
+          # Old marshal dumped source_table object may not have value_converters
+          dump_pos_info[:source_table].set_value_converters(context.value_converters)
         end
-        if option[:mysql_table] && option[:mysql_table].column_names.nil?
-          # Old marshal dumped mysql_table object may not have column_names
-          option[:mysql_table].set_column_names
+        if dump_pos_info[:source_table] && dump_pos_info[:source_table].column_names.nil?
+          # Old marshal dumped source_table object may not have column_names
+          dump_pos_info[:source_table].set_column_names
         end
         log_info_stdout("Sending data to FlyData Server...")
 
@@ -831,40 +818,41 @@ EOM
         send_record_counts_threads = []
         begin
           RubyProf.start if RUN_PROFILE and defined?(RubyProf) and not RubyProf.running?
-          Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
-            mysqldump_io,
+          context.parse_dump(
+            dump_pos_info,
+            dump_io,
             # create table
-            Proc.new { |mysql_table|
-              mysql_table.set_value_converters(FlydataCore::TableDef::MysqlTableDef::VALUE_CONVERTERS)
-              mt = mysql_table.clone
-              mt.value_converters = {}
+            Proc.new { |source_table|
+              source_table.set_value_converters(context.value_converters)
+              st = source_table.clone
+              st.value_converters = {}
               # don't let the closure hold these objects as it causes memory leak
-              mysql_table = nil
+              source_table = nil
               output_thread.run do
                 tmp_num_inserted_record = 0
-                # dump mysql_table for resume
+                # dump source_table for resume
                 #TODO: make it option
-                sync_fm.save_mysql_table_marshal_dump(mt)
-                log_info_stdout("  -> Started sending data for table '#{mt.table_name}'")
+                sync_fm.save_source_table_marshal_dump(st)
+                log_info_stdout("  -> Started sending data for table '#{st.table_name}'")
               end
             },
             # insert record
-            Proc.new { |mysql_table, values_set|
-              mysql_table_name = mysql_table.table_name
+            Proc.new { |source_table, values_set|
+              source_table_name = source_table.table_name
               records = values_set.collect do |values|
-                convert_to_flydata_values(mysql_table, values)
-                json = JSON.generate_kv_pairs(mysql_table.column_names, values)
+                convert_to_flydata_values(source_table, values)
+                json = JSON.generate_kv_pairs(source_table.column_names, values)
                 values.clear
                 unless json.start_with? '{'
                   raise DumpParseError.new("Broken JSON record json:#{json[0..100]}")
                 end
-                {table_name: mysql_table_name, log: json}
+                {table_name: source_table_name, log: json}
               end
               # release resources to prevent the closure from keeping it
               values_set.clear
               values_set = nil
-              mysql_table = nil
-              mysql_table_name = nil
+              source_table = nil
+              source_table_name = nil
               output_thread.run do
                 forwarder.emit(records)
                 tmp_num_inserted_record += 1
@@ -873,9 +861,9 @@ EOM
               true
             },
             # checkpoint
-            Proc.new { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
-              table_name = mysql_table.nil? ? '' : mysql_table.table_name
-              mysql_table = nil
+            Proc.new { |source_table, last_pos, bytesize, source_pos, state, substate|
+              table_name = source_table.nil? ? '' : source_table.table_name
+              source_table = nil
               output_thread.run do
                 skip = skip_checkpoint
                 skip_checkpoint = false
@@ -894,12 +882,12 @@ EOM
                   end
 
                   # save check point
-                  sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, binlog_pos, state, substate)
+                  sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, source_pos, state, substate)
 
                   # send record count for the table
                   if table_name.to_s != '' &&
-                     state == Flydata::Parser::Mysql::MysqlDumpParser::State::CREATE_TABLE
-                    # all records for `mysql_table` have been sent
+                     state == Flydata::Parser::State::CREATE_TABLE
+                    # all records for `source_table` have been sent
                     send_record_counts(de, sync_fm, table_name,
                                        send_record_counts_threads)
                     log_info_stdout("  -> Finished sending data for table '#{table_name}'...")
@@ -928,7 +916,7 @@ EOM
         log_info_stdout("  -> Done")
         #log_info_stdout("  -> Records sent to the server")
         #log_info_stdout("  -> #{sync_fm.load_stats}")
-        sync_fm.save_dump_pos(STATUS_PARSED, '', dump_file_size, binlog_pos)
+        sync_fm.save_dump_pos(STATUS_PARSED, '', dump_file_size, source_pos)
 
         if ENV['FLYDATA_BENCHMARK']
           bench_end_time = Time.now
@@ -937,15 +925,15 @@ EOM
         end
       end
 
-      def complete_mysqldump_processing(dp, de, sync_fm)
+      def complete_dump_processing(sync_fm)
         return if ENV['FLYDATA_BENCHMARK']
 
         # Status is not parsed -> don't complete
         dump_pos_info = sync_fm.load_dump_pos
         return unless dump_pos_info[:status] == STATUS_PARSED
-        binlog_pos = dump_pos_info[:binlog_pos]
+        source_pos = dump_pos_info[:source_pos]
 
-        sync_fm.save_dump_pos(STATUS_COMPLETE, '', -1, binlog_pos)
+        sync_fm.save_dump_pos(STATUS_COMPLETE, '', -1, source_pos)
       end
 
       # option: timeout, tables
@@ -1015,8 +1003,8 @@ EOM
 
       def complete(de)
         sync_fm = create_sync_file_manager(de)
-        info = sync_fm.load_dump_pos
-        if info[:status] == STATUS_COMPLETE
+        dump_pos_info = sync_fm.load_dump_pos
+        if dump_pos_info[:status] == STATUS_COMPLETE
           send_record_counts(de, sync_fm)
           sync_fm.delete_dump_file
           sync_fm.backup_dump_dir
@@ -1058,18 +1046,18 @@ EOM
         end
       end
 
-      def initialize_positions(sync_fm, binlog_pos)
-        sync_fm.save_table_binlog_pos(target_tables, binlog_pos)
+      def initialize_positions(sync_fm, source_pos)
+        sync_fm.save_table_source_pos(target_tables, source_pos)
 
         if @full_initial_sync
-          sync_fm.save_binlog(binlog_pos)
+          sync_fm.save_source_pos(source_pos)
         end
-        sync_fm.install_table_binlog_files(target_tables)
+        sync_fm.install_table_source_pos_files(target_tables)
         sync_fm.reset_table_position_files(target_tables)
       end
 
-      def convert_to_flydata_values(mysql_table, values)
-        vc = mysql_table.value_converters
+      def convert_to_flydata_values(source_table, values)
+        vc = source_table.value_converters
         return if vc.empty?
 
         vc.each_pair do |index, converter|
@@ -1127,40 +1115,26 @@ EOM
 
       # Generate table ddl
 
-      def do_generate_table_ddl(de)
-        if `which mysqldump`.empty?
-          raise "mysqldump is not installed.  mysqldump is required to run the command"
-        end
-
+      def do_generate_table_ddl(context, de)
         error_list = []
         schema_name = (de['schema_name'] || nil)
-        mp = de['mysql_data_entry_preference']
 
         tables = opts.all_tables? ? @full_tables : (@input_tables.empty? ? @unsynced_tables : @input_tables)
 
         raise "There are no valid unsynced tables, if you want to just get ddl for all tables, please run \`flydata sync:generate_table_ddl --all-tables\`" if tables.empty?
 
-        %w(host username database).each do |conf_name|
-          raise "MySQL `#{conf_name}` is neither defined in the data entry nor the local config file" if mp[conf_name].to_s.empty?
-        end
         if tables.empty?
           raise "`tables` (or `tables_append_only`) is neither defined in the data entry nor the local config file"
         end
 
+        flydata_tabledefs, error_list = context.generate_flydata_tabledef(tables,
+                        skip_primary_key_check: opts.skip_primary_key_check?)
         create_flydata_ctl_table = true
-        option = {skip_primary_key_check: opts.skip_primary_key_check?}.merge(mp)
-        missing_tables = FlydataCore::Mysql::CommandGenerator.each_mysql_tabledef(tables, option) do |mysql_tabledef, error|
-          if error
-            error_list << error.err_hash
-            next
-          end
-          flydata_tabledef = mysql_tabledef.to_flydata_tabledef
-          puts FlydataCore::TableDef::SyncRedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
+        append_only = tables & @append_only_tables
+        flydata_tabledefs.each do |flydata_tabledef|
+          puts FlydataCore::TableDef::SyncRedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?, skip_drop_table: append_only.include?(flydata_tabledef[:table_name]))
           create_flydata_ctl_table = false
         end
-        if missing_tables
-          missing_tables.each {|missing_table| error_list << { error: 'table does not exist in the MySQL database', table: missing_table } }
-        end
         table_validity_hash = {}
         tables_without_error = tables
         unless error_list.empty?
@@ -1190,11 +1164,10 @@ EOS
         flydata.data_entry.update_table_validity(de['id'], {updated_tables: table_validity_hash}) unless table_validity_hash.empty?
 
         sync_fm = create_sync_file_manager(de)
-        sync_fm.save_generated_ddl(tables_without_error, Mysql::TableDdl::VERSION)
+        sync_fm.save_generated_ddl(tables_without_error, TableDdl::VERSION)
         sync_fm.close
       end
 
-
       ALL_DONE_MESSAGE_TEMPLATE =  <<-EOM
 
 Congratulations! FlyData has started synchronizing your database tables.
@@ -1228,20 +1201,21 @@ Thank you for using FlyData!
       # Utility methods
 
       def set_current_tables(input_tables = nil, options = {})
-        de = data_entry
-        sync_fm = create_sync_file_manager(de)
+        sync_fm = create_sync_file_manager
         sync_info = sync_fm.load_sync_info
         sync_resumed = options[:resume] && !!sync_info
 
+        table_lists = source.sync.table_lists
+
         #full_tables will either include all tables including invalid tables or all valid tables that aren't new tables
 
         # The 'new_tables' list may include tables which has already been
         # synced for backward compatibility reason.
         # Filter out such tables so that we get a list of 'new' tables with no
         # position file.
-        real_new_tables = sync_fm.get_new_table_list(de['mysql_data_entry_preference']['new_tables'], "pos")
-        @full_tables = options[:include_all_tables] ? de['mysql_data_entry_preference']['tables'] + de['mysql_data_entry_preference']['invalid_tables'] :
-                                                      de['mysql_data_entry_preference']['tables'] - real_new_tables
+        real_new_tables = sync_fm.get_new_table_list(table_lists['new_tables'], "pos")
+        @full_tables = options[:include_all_tables] ? table_lists['tables'] + table_lists['invalid_tables'] :
+                                                      table_lists['tables'] - real_new_tables
 
         @unsynced_tables = sync_fm.get_new_table_list(@full_tables, "pos") # Get list of tables that do not have a .pos file
         @ddl_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl")
@@ -1250,6 +1224,8 @@ Thank you for using FlyData!
         @input_tables ||= []
         @full_initial_sync = sync_resumed ? sync_info[:initial_sync] :
                                             (@unsynced_tables == @full_tables)
+        @append_only_tables = table_lists['tables_append_only']
+        @append_only_tables ||= []
 
         sync_fm.close
 
@@ -1292,38 +1268,15 @@ Thank you for using FlyData!
       end
 
       def data_entry
-        @de ||= retrieve_sync_data_entry
-      end
-
-      def retrieve_sync_data_entry
-        de = retrieve_data_entries.first
-        raise "There are no data entries." unless de
-        case de['type']
-        when 'RedshiftMysqlDataEntry'
-          mp = de['mysql_data_entry_preference']
-
-          if mp['tables_append_only']
-            mp['tables'] = (mp['tables'].split(",") + mp['tables_append_only'].split(",")).uniq
-          else
-            mp['tables'] = mp['tables'].split(",").uniq
-          end
-          mp['invalid_tables'] = mp['invalid_tables'].kind_of?(String) ? mp['invalid_tables'].split(",").uniq : []
-          mp['new_tables'] = mp['new_tables'].kind_of?(String) ? mp['new_tables'].split(",").uniq : []
-
-          unless mp['ssl_ca_content'].to_s.strip.empty?
-            sync_fm = create_sync_file_manager(de)
-            sync_fm.save_ssl_ca(mp['ssl_ca_content'])
-            mp['ssl_ca'] = sync_fm.ssl_ca_path
-            mp['sslca'] = mp['ssl_ca']
-          end
-        else
-          raise SyncDataEntryError, "No supported data entry. Only mysql-redshift sync is supported."
+        unless @sync_de
+          @sync_de = super
+          source.sync.setup # updates the data entry contents
         end
-        de
+        @sync_de
       end
 
       def create_sync_file_manager(de = data_entry)
-        SyncFileManager.new(de)
+        SyncFileManager.new(de, source)
       end
 
       def verify_input_tables(input_tables, all_tables)
@@ -1335,6 +1288,19 @@ Thank you for using FlyData!
         raise "These tables are not registered tables: #{inval_table.join(", ")}" unless inval_table.empty?
       end
 
+      def check_master_position_files
+        de = data_entry
+        sync_fm = create_sync_file_manager(de)
+        old_master_source_pos = sync_fm.load_source_pos
+        old_sent_source_pos = sync_fm.load_sent_source_pos
+
+        corrupt_files = []
+        corrupt_files << sync_fm.source_pos_path if old_master_source_pos.nil?
+        corrupt_files << sync_fm.sent_source_pos_path if old_sent_source_pos.nil?
+
+        corrupt_files.empty? ? nil : corrupt_files
+      end
+
       def check_position_files(table_status_hash)
         de = data_entry
         sync_fm = create_sync_file_manager(de)
@@ -1359,73 +1325,79 @@ Thank you for using FlyData!
         gap_tables.empty? ? nil : gap_tables
       end
 
-      def save_table_positions(table, binlog_pos, pos)
+      def save_table_positions(table, source_pos, pos, context)
         de = data_entry
         sync_fm = create_sync_file_manager(de)
-        s = sync_fm.get_table_binlog_pos(table)
-        old_binlog_pos = s ? FlydataCore::Mysql::BinlogPos.new(s) : nil
+        old_source_pos = sync_fm.get_table_source_pos(table)
         old_pos = sync_fm.get_table_position(table)
         if pos.to_i != old_pos.to_i
           sync_fm.save_table_position(table, pos)
           $log.debug "table pos updated.  table:#{table} pos:#{old_pos} -> #{pos}"
         end
-        if binlog_pos != old_binlog_pos
-          sync_fm.save_table_binlog_pos(table, binlog_pos.to_s, destination: :positions)
-          $log.debug "table binlog updated.  table:#{table} binlog:`#{old_binlog_pos}` -> `#{binlog_pos}`"
+        if source_pos != old_source_pos
+          sync_fm.save_table_source_pos(table, source_pos, destination: :positions)
+          $log.debug "table source_pos updated.  table:#{table} source_pos:`#{old_source_pos}` -> `#{source_pos}`"
         end
-        [old_binlog_pos, old_pos]
+        [old_source_pos, old_pos]
       end
 
-      def save_master_binlog_positions(master_binlog_pos, sent_binlog_pos)
+      def save_master_source_positions(master_source_pos, sent_source_pos, context)
         de = data_entry
         sync_fm = create_sync_file_manager(de)
-        s = sync_fm.load_binlog
-        old_master_binlog_pos = s ? FlydataCore::Mysql::BinlogPos.new(s) : nil
-        s = sync_fm.load_sent_binlog
-        old_sent_binlog_pos = s ? FlydataCore::Mysql::BinlogPos.new(s) : nil
-        if master_binlog_pos != old_master_binlog_pos
-          sync_fm.save_binlog(master_binlog_pos.to_s)
-          $log.debug "master binlog positions updated. `#{old_master_binlog_pos}` -> `#{master_binlog_pos}`"
-        end
-        if sent_binlog_pos != old_sent_binlog_pos
-          sync_fm.save_sent_binlog(sent_binlog_pos.to_s)
-          $log.debug "sent binlog positions updated. `#{old_sent_binlog_pos}` -> `#{sent_binlog_pos}`"
-        end
-        [old_master_binlog_pos, old_sent_binlog_pos]
+
+        old_master_source_pos = sync_fm.load_source_pos
+        old_sent_source_pos = sync_fm.load_sent_source_pos
+
+        if master_source_pos != old_master_source_pos
+          sync_fm.save_source_pos(master_source_pos)
+          $log.debug "master source positions updated. `#{old_master_source_pos}` -> `#{master_source_pos}`"
+        end
+        if sent_source_pos != old_sent_source_pos
+          sync_fm.save_sent_source_pos(sent_source_pos.to_s)
+          $log.debug "sent source positions updated. `#{old_sent_source_pos}` -> `#{sent_source_pos}`"
+        end
+
+        [old_master_source_pos, old_sent_source_pos]
       end
 
-      def get_table_status(tables)
+      def get_table_status(tables, source_pos_context)
         de = data_entry
         sync_fm = create_sync_file_manager(de)
         result = flydata.data_entry.table_status(de['id'], mode: env_mode, tables: tables)
         result = result["table_status"]
 
         table_status_hash = result.inject({}){|h, ts| h[ts["table_name"]] = ts; h}
+        source_pos = nil
+        table_status_hash.keys.each do |table|
+          src_pos_str = table_status_hash[table].delete("src_pos")
+
+          source_pos =  src_pos_str ? source_pos_context.create_source_pos(src_pos_str) : nil
+          table_status_hash[table].merge!( { "source_pos" => source_pos } )
+        end
         missing_tables = tables - table_status_hash.keys
         unless missing_tables.empty?
           raise "table status is not available for these table(s): #{missing_tables.join(",")}"
         end
-
-        populate_initial_binlog_positions(table_status_hash, sync_fm)
+        populate_initial_source_positions(table_status_hash, sync_fm)
         table_status_hash
       end
 
-      # table_status has no binlog position for sequence "0".  Populate the info
+      # table_status has no source position for sequence "0".  Populate the info
       # from 'table.binlog.pos.init' file.
-      def populate_initial_binlog_positions(table_status_hash, sync_fm)
+      def populate_initial_source_positions(table_status_hash, sync_fm)
         table_status_hash.keys.each do |table|
-          src_pos = table_status_hash[table]["src_pos"]
-          if src_pos.nil? || src_pos == "-"
-            init_binlog_pos =  sync_fm.get_table_binlog_pos_init(table)
-            if init_binlog_pos
-              table_status_hash[table]["src_pos"] = init_binlog_pos
+          table_source_pos = table_status_hash[table]["source_pos"]
+          if table_source_pos.nil? || table_source_pos.empty?
+            init_source_pos = sync_fm.get_table_source_pos_init(table)
+            if init_source_pos
+              table_status_hash[table]["source_pos"] = init_source_pos
             end
           end
         end
       end
 
       # TODO implement
-      def get_oldest_available_binlog
+      def get_oldest_available_source_pos
         nil
       end
     end