flydata 0.7.12 → 0.7.13

data/lib/flydata/command/sync.rb

@@ -436,6 +436,7 @@ EOS
         sync_fm = create_sync_file_manager(de)
         context = source.source_pos
         set_current_tables
+
         # Stop agent.  Check sync and make sure the state is :STUCK_AT_UPLOAD
         # Get table status for the tables.
         status, corrupt_master_pos_files, pos_mismatch_tables, gap_tables, table_status_hash =
@@ -538,9 +539,19 @@ EOS
             end
           end
         end
-        # if sent_source_pos is nil, it means sync has started for none of tables.  No need
-        # to repair positions nor clean buffer data.
+
+        unless unrepairable_tables.empty?
+          # Notify expired source position tables through error logs to us
+          log_error_stderr "[error]: Failed to repair tables due to expired source position. These tables need to be re-synced - #{unrepairable_tables.join(", ")}"
+        end
+
+        # If sent_source_pos is nil, it means:
+        #   - Sync has started for none of tables
+        #   - None of tables are broken
+        #   - All of broken tables have an expired source position
+        # No need to repair positions nor clean buffer data.
         if sent_source_pos
+          # This logic is unreachable since sent_source_pos cannot be more than oldest_source_pos
           if oldest_source_pos && sent_source_pos < oldest_source_pos
             e = AgentError.new("Repair failed due to expired source position")
             e.description = <<EOS
@@ -1512,9 +1523,8 @@ Thank you for using FlyData!
         end
       end
 
-      # TODO implement
       def get_oldest_available_source_pos
-        nil
+        source.sync_repair.get_oldest_available_source_pos
       end
     end
   end

data/lib/flydata/source.rb

@@ -24,6 +24,7 @@ module Source
     "FileDataEntry" => :source_file,
     "RedshiftPostgresqlDataEntry" => :source_postgresql,
     "RedshiftZendeskDataEntry" => :source_zendesk,
+    "RedshiftOracleDataEntry" => :source_oracle,
   }
   def self.component_class_for(component_sym, de)
     source_sym = DATA_ENTRY_TYPE_MAP[de['type']]

data/lib/flydata/source/sync_repair.rb

@@ -0,0 +1,25 @@
+require 'flydata/source'
+require 'flydata/source/component'
+require 'flydata/source/errors'
+
+module Flydata
+module Source
+
+class SyncRepair < Component
+  def self.inherited(child_class)
+    Source.register(child_class, self)
+  end
+
+  # Public Interface: Get oldest avilable source pos information
+  #
+  # Called from sync:repair command to determine a master position for repair.
+  # If this returns nil, sync:repair sets any oldest source position to source pos.
+  #
+  # Raises exception when failing to get source positions
+  def self.get_oldest_available_source_pos
+    nil
+  end
+end
+
+end
+end

data/lib/flydata/source_mysql/generate_source_dump.rb

@@ -3,6 +3,7 @@ require 'flydata/preference/data_entry_preference'
 require 'flydata/source_mysql/mysql_compatibility_check'
 require 'flydata/source_mysql/parser/dump_parser'
 require 'flydata-core/mysql/binlog_pos'
+require 'flydata/source_mysql/sync_database_size_check'
 
 module Flydata
 module SourceMysql
@@ -34,7 +35,7 @@ class GenerateSourceDump < Source::GenerateSourceDump
 
   def dump_size(tables)
     opts = de['mysql_data_entry_preference'].merge({"tables" => tables})
-    Parser::DatabaseSizeCheck.new(opts).get_db_bytesize
+    SyncDatabaseSizeCheck.new(opts).get_db_bytesize
   end
 
   def dump(tables, file_path = nil, &src_pos_callback)

data/lib/flydata/source_mysql/mysql_accessible.rb

@@ -0,0 +1,30 @@
+require 'mysql2'
+require 'flydata-core/mysql/config'
+
+module Flydata
+  module SourceMysql
+    module MysqlAccessible
+      def initialize(conf)
+        @conf = conf
+        @mysql_db_opts = build_mysql_db_opts(conf)
+      end
+
+      def exec_mysql_query(query)
+        cli = mysql_client
+        cli.query(query)
+      ensure
+        if cli
+          cli.close rescue nil
+        end
+      end
+
+      def build_mysql_db_opts(conf)
+        FlydataCore::Mysql::Config.build_mysql_db_opts(conf)
+      end
+
+      def mysql_client
+        Mysql2::Client.new(@mysql_db_opts)
+      end
+    end
+  end
+end

data/lib/flydata/source_mysql/parser/dump_parser.rb

@@ -12,18 +12,6 @@ module Flydata
   module SourceMysql
     module Parser
 
-      module MysqlAccessible
-        def mysql_conf(conf)
-          @mysql_conf = FlydataCore::Mysql::Config.build_mysql_db_opts(conf)
-        end
-
-        def mysql_cli(conf = nil)
-          mysql_conf(conf) if conf
-          return FlydataMysqlClient.new(@mysql_conf) if @mysql_conf
-          nil
-        end
-      end
-
       module DumpStreamIO
         # return position
         # sync command doesn't resume if pos is -1 in dump position file
@@ -625,34 +613,6 @@ EOS
         end
       end
 
-      class DatabaseSizeCheck
-        include MysqlAccessible
-
-        SIZE_CHECK_QUERY = <<EOT
-SELECT
-  SUM(data_length) bytesize
-FROM
-  information_schema.tables
-WHERE
-  table_schema = '%s' AND table_name in (%s);
-EOT
-
-        def initialize(de_conf)
-          @de_conf = de_conf
-          @database = de_conf['database']
-          @tables = de_conf['tables']
-          @query = SIZE_CHECK_QUERY % [@database, @tables.collect{|t| "'#{t}'"}.join(',')]
-        end
-
-        def get_db_bytesize
-          client = mysql_cli(@de_conf)
-          result = client.query(@query)
-          return result.first['bytesize'].to_i
-        ensure
-          client.close rescue nil
-        end
-      end
-
       # Read and buffer data in a separate thread
       class AsyncIO
         MAX_ITEMS = 200

data/lib/flydata/source_mysql/sync_database_size_check.rb

@@ -0,0 +1,29 @@
+require 'flydata/source_mysql/mysql_accessible'
+
+module Flydata
+  module SourceMysql
+    class SyncDatabaseSizeCheck
+      include MysqlAccessible
+
+      SIZE_CHECK_QUERY = <<EOT
+SELECT
+  SUM(data_length) bytesize
+FROM
+  information_schema.tables
+WHERE
+  table_schema = '%s' AND table_name in (%s);
+EOT
+
+      def initialize(de_conf)
+        super
+        @query = SIZE_CHECK_QUERY % [
+          de_conf['database'],
+          tables = de_conf['tables'].collect{|t| "'#{t}'"}.join(',')]
+      end
+
+      def get_db_bytesize
+        exec_mysql_query(@query).first['bytesize'].to_i
+      end
+    end
+  end
+end

data/lib/flydata/source_mysql/sync_repair.rb

@@ -0,0 +1,26 @@
+require 'flydata/source/sync_repair'
+require 'flydata/source_mysql/mysql_accessible'
+require 'flydata/source_mysql/source_pos'
+
+module Flydata
+module SourceMysql
+
+class SyncRepair < Source::SyncRepair
+  def get_oldest_available_source_pos
+    binary_logs = BinarylogsCheck.new(de[Sync::SOURCE_PREFERENCE_NAME]).get_binary_logs
+    oldest_binlog_str = binary_logs.first.values.join("\t")
+    oldest_binlog =  source.source_pos.create_source_pos(oldest_binlog_str)
+  end
+
+  class BinarylogsCheck
+    include MysqlAccessible
+
+    SHOW_BINARY_LOGS_QUERY = "SHOW BINARY LOGS;"
+    def get_binary_logs
+      exec_mysql_query(SHOW_BINARY_LOGS_QUERY)
+    end
+  end
+end
+
+end
+end

data/lib/flydata/source_oracle/data_entry.rb

@@ -0,0 +1,24 @@
+require 'flydata/source/data_entry'
+
+module Flydata
+module SourceOracle
+
+class DataEntry < Source::DataEntry
+  CONFIG_PARAMS = {
+    oracle_data_entry_preference: {
+      database: {},
+      tables: {},
+      tables_append_only: {},
+      pk_override: {},
+      table_attributes: {},
+      host: {},
+      port: {},
+      username: {},
+      password: {encrypted: true},
+      schema: {},
+    }
+  }
+end
+
+end
+end

data/lib/flydata/source_oracle/generate_source_dump.rb

@@ -0,0 +1,184 @@
+require 'flydata/source/generate_source_dump'
+require 'flydata/preference/data_entry_preference'
+require 'flydata/source_oracle/oracle_component'
+require 'flydata/source_oracle/query_based_sync/diff_query_generator'
+require 'flydata/source_oracle/table_meta'
+require 'flydata-core/oracle/source_pos'
+require 'flydata-core/oracle/oracle_client'
+require 'flydata-core/oracle/query_helper'
+require 'msgpack'
+
+module Flydata
+module SourceOracle
+
+class GenerateSourceDump < Source::GenerateSourceDump
+  include OracleComponent
+
+  def run_compatibility_check(dump_dir, backup_dir)
+    %w(host username database).each do |k|
+      if de_prefs[k].to_s.empty?
+        raise "'#{k}' is required. Set the value in the conf file " +
+              "-> #{Flydata::Preference::DataEntryPreference.conf_path(de)}"
+      end
+    end
+  end
+
+  def confirmation_items
+    items = {
+      "host"     => de_prefs['host'],
+      "port"     => de_prefs['port'],
+      "username" => de_prefs['username'],
+      "database" => de_prefs['database'],
+      "schema"   => de_prefs['schema'],
+    }
+
+    items
+  end
+
+  DUMP_SIZE_QUERY = <<EOS
+SELECT
+  sum(bytes) as total_size
+FROM
+(SELECT segment_name table_name, owner, bytes
+FROM dba_segments
+WHERE segment_type IN ('TABLE','TABLE PARTITION','TABLE SUBPARTITION')
+UNION ALL
+SELECT i.table_name, i.owner, s.bytes
+FROM dba_indexes i, dba_segments s
+WHERE s.segment_name = i.index_name
+AND s.owner = i.owner
+AND s.segment_type IN ('INDEX','INDEX PARTITION','INDEX SUBPARTITION')
+UNION ALL
+SELECT l.table_name, l.owner, s.bytes
+FROM dba_lobs l, dba_segments s
+WHERE s.segment_name = l.segment_name
+AND s.owner = l.owner
+AND s.segment_type IN ('LOBSEGMENT','LOB PARTITION')
+UNION ALL
+SELECT l.table_name, l.owner, s.bytes
+FROM dba_lobs l, dba_segments s
+WHERE s.segment_name = l.index_name
+AND s.owner = l.owner
+AND s.segment_type = 'LOBINDEX')
+WHERE owner = %{schema} and table_name in (%{tables})
+EOS
+
+  def dump_size(tables)
+    cli = FlydataCore::Oracle::OracleClient.new(de_prefs)
+
+    query = DUMP_SIZE_QUERY % {
+      schema: FlydataCore::Oracle::QueryHelper.schema_as_value(de_prefs['schema'],
+                                                               de_prefs['username']),
+      tables: FlydataCore::Oracle::QueryHelper.tables_as_value(tables)
+    }
+
+    cursor = cli.query(query)
+    cursor.fetch_hash['TOTAL_SIZE'].to_i
+  ensure
+    cursor.close rescue nil if cursor
+    cli.close if cli
+  end
+
+  def dump(tables, file_path = nil, &src_pos_callback)
+    io = nil
+    if file_path
+      io = File.open(file_path, "w")
+    else
+      raise "dump via pipe has not been implemented yet"
+    end
+
+    table_meta = Flydata::SourceOracle::TableMeta.new(de_prefs, tables)
+    cli = FlydataCore::Oracle::OracleClient.new(de_prefs)
+    table_meta.reload(cli)
+
+    tables_missing_meta = tables.select{|t| tm = table_meta[t]; tm.nil? || tm.empty?}
+    unless tables_missing_meta.empty?
+      raise "Tables are not available.  Check if the following table(s) exist and are visible: #{tables_missing_meta.join(",")}"
+    end
+
+    context = source.sync_generate_table_ddl(dp, nil)
+    source_pos = get_source_pos(table_meta.current_scn, &src_pos_callback)
+
+    options = de_prefs.merge(table_meta: table_meta)
+    missing_tables = context.each_source_tabledef(tables, options) do |tabledef, error|
+      dump_table(tabledef, source_pos, io, cli) if tabledef
+    end
+
+    nil
+  ensure
+    cli.close if cli
+    io.close if io
+  end
+
+  private
+
+  def get_source_pos(snapshot, &src_pos_callback)
+    src_pos = FlydataCore::Oracle::SourcePos.new(snapshot)
+    src_pos_callback.call(nil, src_pos)
+
+    src_pos
+  end
+
+  NUM_ROWS = 50000
+
+  def dump_table(tabledef, source_pos, io, cli)
+    dump_source_table(tabledef, io)
+
+    last_pks = nil
+    loop do
+      num_rows, last_pks =
+        dump_table_chunk(tabledef.table_name, de_prefs['schema'], source_pos, NUM_ROWS,
+                         tabledef, tabledef.pk_columns, last_pks, io, cli)
+      break if num_rows < NUM_ROWS
+    end
+  end
+
+  def dump_source_table(tabledef, io)
+    columns = {}
+    tabledef.column_def.each do |column_name, columndef_str|
+      columndef_hash = eval columndef_str
+      columns[column_name] = {"column_name" => column_name,
+                              "format_type" => columndef_hash["data_type"]}
+    end
+    source_table_hash = { "table_name" => tabledef.table_name,
+                          "columns" => columns }
+
+    io.write(source_table_hash.to_msgpack)
+  end
+
+  def dump_table_chunk(table, schema, source_pos, num_rows,
+                       tabledef, pk_columns, last_pks, io, cli)
+    query = Flydata::SourceOracle::QueryBasedSync::DiffQueryGenerator.new(
+              table, schema,
+              columns: tabledef.columns,
+              to_scn: source_pos.scn,
+              pk_columns: pk_columns,
+              last_pks: last_pks,
+              limit: num_rows).build_query
+
+    column_names = tabledef.columns.inject([]) do |ary, h|
+       ary << h[:column]
+    end
+
+    res = cli.query(query)
+    count = 0
+    last_row = nil
+    while (row = res.fetch_hash)
+      count += 1
+      last_row = row
+      dump_row(row, column_names, io)
+    end
+
+    last_pks = last_row ? pk_columns.collect{|col| last_row[col]} : nil
+
+    [count, last_pks]
+  end
+
+  def dump_row(row, columns, io)
+    data = columns.collect{|col| row[col] }.to_msgpack
+    io.write(data)
+  end
+end
+
+end
+end