flydata 0.6.3 → 0.6.4

data/lib/flydata/source/errors.rb

@@ -0,0 +1,7 @@
+module Flydata
+module Source
+
+class UnsupportedSourceError < RuntimeError; end
+
+end
+end

data/lib/flydata/source/generate_source_dump.rb

@@ -0,0 +1,72 @@
+require 'flydata/source'
+require 'flydata/source/component'
+require 'flydata/source/errors'
+
+module Flydata
+module Source
+
+class GenerateSourceDump < Component
+  def self.inherited(child_class)
+    Source.register(child_class, self)
+  end
+
+  def initialize(source, dp, options)
+    super(source, options)
+    @dp = dp
+  end
+  attr_reader :dp
+
+  # Public Interface: Run compatibility check
+  #
+  # Run whatever check (compatibility, connectivity, privilege, etc) to ensure
+  # that initial sync can run without an issue.
+  #
+  # dump_dir: A directory path string to the dump directory
+  # backup_dir: A directory path string to the backup directory
+  #
+  # Raises exception when check fails
+  def run_compatibility_check(dump_dir, backup_dir)
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+
+  # Public Interface: Confirmation items
+  #
+  # Returns a hash of items to be shown as the final confirmation before
+  # initial sync.
+  #  Example: Return value {"host" => "ubertas","port" => 3306}
+  #           will be displayed as
+  #               host: ubertas
+  #               port: 3306
+  def confirmation_items
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+
+  # Public Interface: Dump size
+  #
+  # tables: An array of table names to be dumped.
+  #
+  # Returns an approximate size of dump in bytes.  The value may be a
+  # best-effort estimate.  It doesn't have to be accurate.
+  def dump_size(tables)
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+
+  # Public Interface: Dump data
+  #
+  # tables: An array of tables to be dumped.
+  # file_path: A file path string of the dump file to which data is written.
+  #         This value may be nil, in which case contents are written to a
+  #         pipe.
+  # src_pos_callback: A callback called when the source position of the dump
+  #         becomes available. The callback takes the following arguments.
+  #           io: Input IO to the dump.
+  #           src_pos: Source position of the dump
+  #
+  # Returns none
+  def dump(tables, file_path = nil, &src_pos_callback)
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+end
+
+end
+end

data/lib/flydata/source/parse_dump_and_send.rb

@@ -0,0 +1,52 @@
+require 'flydata/source'
+require 'flydata/source/component'
+require 'flydata/source/errors'
+
+module Flydata
+module Source
+
+class ParseDumpAndSend < Component
+  def self.inherited(child_class)
+    Source.register(child_class, self)
+  end
+
+  # Public Interface: Value Converters
+  #
+  # Returns a hash of value converters which convert source values to FlyData values.
+  # Hash key is a data type for which the converter is called.  A conveter takes a
+  # source value and returns a converted value.
+  def value_converters
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+
+  # Public Interface: Parse Dump
+  #
+  # dump_pos_info: A hash containing the dump position from where the parse
+  #                starts
+  # dmpio: A read IO object to the dump
+  # create_table_block: A callback called with a table info before its data gets
+  #                     sent
+  #   callback parameters:
+  #     source_table: A SourceTable object
+  # insert_record_block: A callback called with parsed records
+  #   callback parameters:
+  #     source_table: A SourceTable object
+  #     values_set: An array of arrays.  Each array has values for a row.
+  # check_point_block: A callback called between transactions.  Dump position
+  #                    gets saved for resume at this timing.
+  #   callback_parameters:
+  #     source_table: A SourceTable object
+  #     last_pos: An IO location as of the check point
+  #     bytesize: Number of bytes which have been processed
+  #     source_pos: Source position of the dump
+  #     state: Current dump state
+  #     substate: Current dump sub-state
+  #
+  # Returns none
+  def parse_dump(dump_pos_info, dmpio, create_table_block, insert_record_block, check_point_block)
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+end
+
+end
+end

data/lib/flydata/source/setup.rb

@@ -0,0 +1,31 @@
+require 'flydata/source'
+require 'flydata/source/component'
+require 'flydata/source/errors'
+
+module Flydata
+module Source
+
+class Setup < Component
+  def self.inherited(child_class)
+    Source.register(child_class, self)
+  end
+
+  # Public Interface: Tells if Agent process needs restart.
+  #
+  # Returns true if Agent process needs restart at the end of  #initial_run.
+  # Returns false otherwise.
+  def initial_run_need_restart?
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+
+  # Public Interface: initial_run complete message
+  #
+  # Returns a string which is displayed as the completion message at the
+  # end of #initial_run, which gets called at the end of Agent install.
+  def initial_run_complete_message
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+end
+
+end
+end

data/lib/flydata/source/source_pos.rb

@@ -0,0 +1,45 @@
+require 'flydata/source'
+require 'flydata/source/component'
+require 'flydata/source/errors'
+
+module Flydata
+module Source
+
+class SourcePos < Component
+  def self.inherited(child_class)
+    Source.register(child_class, self)
+  end
+
+  def initialize(source)
+    super(source)
+  end
+  #attr_reader :Should not use any attr_reader/attr_accessor if the variable is Source dependent
+
+  # Public Interface: Create source pos
+  #
+  # Returns a context-dependent position object that has the following methods:
+  # (includeng Comparable mixin is preferable)
+  # - all comparison operators (<=>, <, <=, >=, >, ==, !=)
+  # - to_s
+  #
+  def create_source_pos(source_pos_str)
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+
+  # Public Interface: Back to last known safe source pos
+  #
+  # Returns a source position object
+  #
+  # A source pos may not be restart-able from the very position because
+  # there may be preceding records necessary to restart the source log
+  # playback. This method returns such a 'safe' position from which
+  # the source log can be played back.
+  #
+  def resume_pos(source_pos)
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+
+end
+
+end
+end

data/lib/flydata/source/sync.rb

@@ -0,0 +1,56 @@
+require 'flydata/source'
+require 'flydata/source/component'
+require 'flydata/source/errors'
+
+module Flydata
+module Source
+
+class Sync < Component
+  def self.inherited(child_class)
+    Source.register(child_class, self)
+  end
+
+  # Public Interface: Set up data source
+  #
+  # Called right after the source instance is created.  Perform one-time setup
+  # which is necessary to initialize the source for sync (and sync only)
+  #
+  # Raises exception when the source does not support sync
+  def setup
+    raise UnsupportedSourceError,"subclass must implement"
+  end
+
+  # Public Interface: Tells if the source support sync or not
+  #
+  # Returns true if the source supports sync.  No otherwise.
+  def supported?
+    raise UnsupportedSourceError, "subclass must implement"
+  end
+
+  # Public Interface: Table lists
+  #
+  # Returns lists of tables in a hash.  The following lists will be returned
+  #   "tables" : An array of tables currently in sync
+  #   "new_tables" : An array of tables for which no generate_table_ddl has been run yet
+  #   "invalid_tables" : An array of tables that had an issue starting sync
+  def table_lists
+    raise UnsupportedSourceError, "subclass must implement"
+  end
+
+  # Public Interface: Data Servers
+  #
+  # Returns a comma separated list of data servers to which the agent sends data
+  def data_servers
+    raise UnsupportedSourceError, "subclass must implement"
+  end
+
+  # Public Interface: Forwarder
+  #
+  # Returns a forwarder type in string.  Values are 'tcpforwarder' or 'sslforwarder'.
+  def forwarder
+    raise UnsupportedSourceError, "subclass must implement"
+  end
+end
+
+end
+end

data/lib/flydata/source/sync_generate_table_ddl.rb

@@ -0,0 +1,43 @@
+require 'flydata/source'
+require 'flydata/source/component'
+require 'flydata/source/errors'
+
+module Flydata
+module Source
+
+class SyncGenerateTableDdl < Component
+  def self.inherited(child_class)
+    Source.register(child_class, self)
+  end
+
+  def initialize(source, dp, options)
+    super(source, options)
+    @dp = dp
+  end
+  attr_reader :dp
+
+  # Public Interface: Run compatibility check
+  #
+  # Run whatever check (compatibility, connectivity, privilege, etc) to ensure
+  # that the 'sync:generate_table_ddl' command can run without an issue.
+  #
+  # Raises exception when check fails
+  def run_compatibility_check
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+
+  # Public Interface: Generate FlyData table definitions for given tables
+  #
+  # tables - An array of table names
+  # options - A hash of options
+  #
+  # Returns flydata_tabledefs, errors
+  #  flydata_tablesdefs - An array of FlyData tabledefs
+  #  errors - An array of error hashes for tables whose tabledef generation failed
+  def generate_flydata_tabledef(tables, options)
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+end
+
+end
+end

data/lib/flydata/source_file/setup.rb

@@ -0,0 +1,17 @@
+require 'flydata/source/setup'
+
+module Flydata
+module SourceFile
+
+class Setup < Source::Setup
+  def initial_run_need_restart?
+    true
+  end
+
+  def initial_run_complete_message
+    :all_done
+  end
+end
+
+end
+end

data/lib/flydata/source_file/sync.rb

@@ -0,0 +1,14 @@
+require 'flydata/source/sync'
+require 'flydata/source/errors'
+
+module Flydata
+module SourceFile
+
+class Sync < Source::Sync
+  def setup
+    raise Source::UnsupportedSourceError, "This data entry does not support 'sync' commands"
+  end
+end
+
+end
+end

data/lib/flydata/{command → source_mysql/command}/mysql.rb

@@ -1,6 +1,7 @@
 require 'flydata/command/sync'
-require 'flydata/command/mysql_command_base'
+require 'flydata/source_mysql/command/mysql_command_base'
 
+# Command class must be in module Flydata::Command
 module Flydata
   module Command
     class Mysql < Sync

data/lib/flydata/{command → source_mysql/command}/mysql_command_base.rb

@@ -1,12 +1,13 @@
 require 'open3'
 require 'flydata-core/mysql/command_generator'
 
+# Command class must be in module Flydata::Command
 module Flydata
   module Command
 
     module MysqlCommandBase
       def run(*args)
-        de = retrieve_sync_data_entry
+        de = data_entry
         cmd = generate_command(de['mysql_data_entry_preference'], args)
         return if cmd.to_s.empty?
         $stderr.puts "command:#{cmd}" if FLYDATA_DEBUG
@@ -37,10 +38,7 @@ module Flydata
       def reset; end
       def skip; end
       def generate_table_ddl; end
-      def fix_binlogpos; end
     end
 
   end
 end
-
-

data/lib/flydata/{command → source_mysql/command}/mysqlbinlog.rb

@@ -1,6 +1,7 @@
 require 'flydata/command/sync'
-require 'flydata/command/mysql_command_base'
+require 'flydata/source_mysql/command/mysql_command_base'
 
+# Command class must be in module Flydata::Command
 module Flydata
   module Command
     class Mysqlbinlog < Sync

data/lib/flydata/{command → source_mysql/command}/mysqldump.rb

@@ -1,6 +1,7 @@
 require 'flydata/command/sync'
-require 'flydata/command/mysql_command_base'
+require 'flydata/source_mysql/command/mysql_command_base'
 
+# Command class must be in module Flydata::Command
 module Flydata
   module Command
     class Mysqldump < Sync

data/lib/flydata/source_mysql/generate_source_dump.rb

@@ -0,0 +1,53 @@
+require 'flydata/source/generate_source_dump'
+require 'flydata/preference/data_entry_preference'
+require 'flydata/source_mysql/mysql_compatibility_check'
+require 'flydata/source_mysql/parser/dump_parser'
+require 'flydata-core/mysql/binlog_pos'
+
+module Flydata
+module SourceMysql
+
+class GenerateSourceDump < Source::GenerateSourceDump
+  def run_compatibility_check(dump_dir, backup_dir)
+    %w(host username database).each do |k|
+      if de['mysql_data_entry_preference'][k].to_s.empty?
+        raise "'#{k}' is required. Set the value in the conf file " +
+              "-> #{Flydata::Preference::DataEntryPreference.conf_path(de)}"
+      end
+    end
+
+    MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], dump_dir: dump_dir, backup_dir: backup_dir).check
+  end
+
+  def confirmation_items
+    mp = de['mysql_data_entry_preference']
+    items = {
+      "host"     => mp['host'],
+      "port"     => mp['port'],
+      "username" => mp['username'],
+      "database" => mp['database'],
+    }
+    items["ssl"] = "Yes" if mp['ssl_ca']
+
+    items
+  end
+
+  def dump_size(tables)
+    opts = de['mysql_data_entry_preference'].merge({"tables" => tables})
+    Parser::DatabaseSizeCheck.new(opts).get_db_bytesize
+  end
+
+  def dump(tables, file_path = nil, &src_pos_callback)
+    opts = de['mysql_data_entry_preference'].merge({"tables" => tables})
+    dump_generator = Parser::MysqlDumpGeneratorNoMasterData.new(opts)
+
+    dump_generator.dump(file_path) do |io, binlog_hash|
+      source_pos = FlydataCore::Mysql::BinlogPos.new(binlog_hash)
+      src_pos_callback.call(io, source_pos)
+    end
+    nil
+  end
+end
+
+end
+end