flydata 0.7.12 → 0.7.13

data/lib/flydata/source_oracle/oracle_component.rb

@@ -0,0 +1,12 @@
+module Flydata
+module SourceOracle
+
+# Postgresql specific Component helper methods
+module OracleComponent
+  def de_prefs
+    de['oracle_data_entry_preference']
+  end
+end
+
+end
+end

data/lib/flydata/source_oracle/parse_dump_and_send.rb

@@ -0,0 +1,128 @@
+require 'flydata/source/parse_dump_and_send'
+require 'flydata/source_oracle/oracle_component'
+require 'flydata-core/table_def/oracle_table_def'
+
+module Flydata
+module SourceOracle
+
+class ParseDumpAndSend < Source::ParseDumpAndSend
+  include OracleComponent
+
+  def value_converters
+    FlydataCore::TableDef::OracleTableDef::VALUE_CONVERTERS
+  end
+
+  # dump format
+  # dump file is in msgpack.  Each table data starts with a source table hash
+  # followed by row arrays.
+  #
+  # {"table_name"=>"users", "columns"=>{"id"=>{"column_name"=>"id", "format_type"=>"bigint"}, "name"=>{"column_name"=>"name", "format_type"=>"character varying"}, "another_id"=>{"column_name"=>"another_id", "format_type"=>"integer"}}}
+  # ["2", "hay", "1"]
+  # ["3", "hoe", "2"]
+  def parse_dump(dump_pos_info, dmpio, create_table_block, insert_record_block,
+                 check_point_block)
+    parser = DumpParser.new(dump_pos_info, dmpio, create_table_block,
+                            insert_record_block, check_point_block)
+    parser.parse_all
+  end
+end
+
+class DumpParser
+  MAX_ROW_BYTES = 1 * 1024 * 1024 # Parser holds rows until the total byte size
+                                  # reaches this number
+
+  def initialize(dump_pos_info, dmpio, create_table_block, insert_record_block,
+                 check_point_block)
+    @source_pos = dump_pos_info[:source_pos]
+    raise ArgumentError.new("source position is required") unless @source_pos
+    @current_table = nil
+    @last_pos = 0
+    @row_head_pos = nil
+    @rows = []
+    @dmpio = dmpio
+    @create_table_block = create_table_block
+    @insert_record_block = insert_record_block
+    @check_point_block = check_point_block
+
+    resume(dump_pos_info)
+  end
+
+  def parse_all
+    u = MessagePack::Unpacker.new(@dmpio)
+    u.each do |obj|
+      @last_pos = @dmpio.pos - u.buffer.size
+      parse(obj)
+    end
+    close
+  end
+
+  def parse(obj)
+    obj.kind_of?(Hash) ? handle_table_info(obj) : handle_data_row(obj)
+  end
+
+  def close
+    unless @rows.empty?
+      call_insert_record_block
+      # Core logic expects a check point callback with CREATE_TABLE at the
+      # end of table data insertion of each table.  #handle_table_info takes
+      # care of all tables but the last table.  This is for the last table.
+      call_check_point_block(Parser::State::CREATE_TABLE)
+    end
+  end
+
+
+  private
+
+  def resume(dump_pos_info)
+    last_pos = dump_pos_info[:last_pos] ? dump_pos_info[:last_pos].to_i : -1
+    if last_pos == -1
+      # no resume point
+      return
+    end
+
+    @last_pos = last_pos
+    @dmpio.pos = last_pos
+    @current_table = dump_pos_info[:source_table]
+  end
+
+  # {"table_name"=>"users", "columns"=>{"id"=>{"column_name"=>"id", "format_type"=>"bigint"}, "name"=>{"column_name"=>"name", "format_type"=>"character varying"}, "another_id"=>{"column_name"=>"another_id", "format_type"=>"integer"}}}
+  def handle_table_info(table_info)
+    call_insert_record_block # flush rows if any
+    call_check_point_block(Parser::State::CREATE_TABLE)
+    columns = table_info["columns"].inject({}) do |h, (k, v)|
+                h[k] = %w(column_name format_type).inject({}) do |hh, kk|
+                         hh[kk.to_sym] = v[kk]
+                         hh
+                       end
+                h
+              end
+    @current_table = Parser::SourceTable.new(table_info["table_name"], columns)
+    @create_table_block.call(@current_table)
+    call_check_point_block(Parser::State::INSERT_RECORD)
+  end
+
+  def handle_data_row(row)
+    @row_head_pos = @last_pos unless @row_head_pos
+    @rows << row
+    if @last_pos - @row_head_pos > MAX_ROW_BYTES
+      call_insert_record_block
+    end
+  end
+
+  def call_check_point_block(state)
+    @check_point_block.call(@current_table, @last_pos, @last_pos, @source_pos,
+                            state)
+  end
+
+  def call_insert_record_block
+    return if @rows.empty?
+    if @insert_record_block.call(@current_table, @rows)
+      call_check_point_block(Parser::State::INSERT_RECORD)
+    end
+    @rows = []
+    @row_head_pos = nil
+  end
+end
+
+end
+end

data/lib/flydata/source_oracle/plugin_support/context.rb

@@ -0,0 +1,13 @@
+require 'flydata/plugin_support/context'
+require 'flydata-core/oracle/source_pos'
+
+module Flydata
+module SourceOracle
+module PluginSupport
+  class Context < ::Flydata::PluginSupport::Context
+    register_mandatory_opts :database, :dbconf
+    SOURCE_POS_CLASS = ::FlydataCore::Oracle::SourcePos
+  end
+end
+end
+end

data/lib/flydata/source_oracle/plugin_support/source_position_file.rb

@@ -0,0 +1,14 @@
+require 'flydata/plugin_support/source_position_file'
+require 'flydata-core/oracle/source_pos'
+
+module Flydata
+module SourceOracle
+
+module PluginSupport
+  class SourcePositionFile < Flydata::PluginSupport::SourcePositionFile
+    SOURCE_POS_CLASS = FlydataCore::Oracle::SourcePos
+  end
+end
+
+end
+end

data/lib/flydata/source_oracle/query_based_sync/diff_query_generator.rb

@@ -0,0 +1,122 @@
+require 'flydata-core/oracle/oracle_client'
+require 'flydata-core/table_def/oracle_table_def'
+require 'flydata-core/table_def/value_conv'
+
+module Flydata
+module SourceOracle
+module QueryBasedSync
+
+class DiffQueryGenerator
+
+  DEFAULT_LIMIT = 10000
+
+  FETCH_RECORDS_SQL = <<EOS
+SELECT %{columns} FROM %{table_name} AS OF SCN %{scn} WHERE %{where_clause} ORDER BY %{order_by_clause}
+EOS
+
+  def initialize(table, schema, query_cond = {})
+    @table = table
+    @schema = schema
+
+    @columns = query_cond[:columns] or raise ArgumentError, "columns is required"
+    @to_scn = query_cond[:to_scn] or raise ArgumentError, "to_scn is required"
+    @pk_columns = query_cond[:pk_columns] or raise ArgumentError, "pk_columns is required"
+    @last_pks = query_cond[:last_pks]
+    @limit = query_cond[:limit] || DEFAULT_LIMIT
+  end
+
+  def build_query(binding_param_enabled = true)
+    column_names, types = @columns.inject([[],[]]) do |arys, h|
+      arys[0] << h[:column]; arys[1] << h[:type].gsub(/\(.+?\)/,''); arys
+    end
+    columns = column_list(column_names, types)
+
+    query = FETCH_RECORDS_SQL % {
+      columns: columns,
+      table_name: fq_table_name(@table, @schema),
+      scn: @to_scn,
+      where_clause: build_where_clause(@pk_columns, @last_pks, @limit),
+      order_by_clause: build_order_by_clause(@pk_columns),
+    }
+
+  end
+
+  private
+
+  def build_where_clause(pk_columns, last_pks, limit, binding_param_enabled = false)
+    clauses = []
+    if last_pks && !last_pks.empty?
+      clauses << pk_conditions(pk_columns, last_pks, binding_param_enabled)
+    end
+    clauses << "ROWNUM <= #{limit}"
+    clauses.join(" AND ")
+  end
+
+  def build_order_by_clause(pk_columns)
+    "#{pk_columns.collect{|col| %Q|"#{col}"| }.join(",")}"
+  end
+
+  def pk_conditions(pk_columns, last_pks, binding_param_enabled)
+    i = pk_columns.size - 1
+    str = nil
+    while i >= 0
+      colname = pk_columns[i]
+      val = value_in_where(i, last_pks, binding_param_enabled)
+      if str
+        str = %Q|"#{colname}" > #{val} OR ("#{colname}" = #{val} AND (#{str}))|
+      else
+        str = %Q|"#{colname}" > #{val}|
+      end
+      i -= 1
+    end
+    "(#{str})"
+  end
+
+  def value_in_where(index, last_pks, binding_param_enabled)
+    if binding_param_enabled
+      "$#{index + 1}"
+    else
+      %Q|'#{last_pks[index]}'|
+    end
+  end
+
+  def fq_table_name(table, schema)
+    !schema.to_s.empty? ? %Q|"#{schema.upcase}"."#{table}"| : %Q|"#{table}"|
+  end
+
+  # return a list of columns with necessary conversion as a string
+  QUERY_VAL_CONV_RULES = Hash.new do |h, k|
+    h[k] =
+      case k
+        when "bytea"; %Q['0x' || encode("%<column>s", 'hex') AS "%<column>s"]
+        else %Q["%<column>s"]
+      end
+  end
+
+  VALUE_OVERRIDERS = Hash.new do |h, k|
+    h[k] =
+      case k
+      when "bit", "varbit"; -> (v) { v ? '%d' % v.to_i(2) : nil }
+      when "boolean"; -> (v) { FlydataCore::TableDef::OracleTableDef::to_boolean(v) }
+      when "money"; -> (v) { v ? FlydataCore::TableDef::ValueConv.strip_currency_format(v) : nil }
+      else
+        nil
+      end
+  end
+
+  def column_list(columns, types)
+    columns.each_with_index.collect{|c, i| QUERY_VAL_CONV_RULES[types[i]] % {column:c}}.join(", ")
+  end
+
+  def build_value_overriders(columns, types)
+    columns.each_with_index.inject({}) do |h, (c, i)|
+      overrider = VALUE_OVERRIDERS[types[i]]
+      h[c] = overrider if overrider
+      h
+    end
+  end
+end
+
+end
+end
+end

data/lib/flydata/source_oracle/setup.rb

@@ -0,0 +1,24 @@
+require 'flydata/source/setup'
+require 'flydata/sync_file_manager'
+
+module Flydata
+module SourceOracle
+
+class Setup < Source::Setup
+  def initial_run_need_restart?
+    overinstall?
+  end
+
+  def initial_run_complete_message
+    overinstall? ? :all_done : :initial_sync
+  end
+
+  private
+
+  def overinstall?
+    File.exists?(Flydata::SyncFileManager.new(de).source_pos_path)
+  end
+end
+
+end
+end

data/lib/flydata/source_oracle/source_pos.rb

@@ -0,0 +1,18 @@
+require 'flydata/source/source_pos'
+require 'flydata-core/oracle/source_pos'
+
+module Flydata
+module SourceOracle
+
+class SourcePos < Source::SourcePos
+  def create_source_pos(source_pos_str)
+    FlydataCore::Oracle::SourcePos.load(source_pos_str)
+  end
+
+  def resume_pos(source_pos)
+    source_pos
+  end
+end
+
+end
+end

data/lib/flydata/source_oracle/sync.rb

@@ -0,0 +1,15 @@
+require 'flydata/source/sync'
+
+module Flydata
+module SourceOracle
+
+class Sync < Source::Sync
+  SOURCE_PREFERENCE_NAME = 'oracle_data_entry_preference'
+
+  def supported?
+    true
+  end
+end
+
+end
+end

data/lib/flydata/source_oracle/sync_generate_table_ddl.rb

@@ -0,0 +1,64 @@
+require 'flydata/source/sync_generate_table_ddl'
+require 'flydata/source_oracle/table_meta'
+
+module Flydata
+module SourceOracle
+
+class SyncGenerateTableDdl < Source::SyncGenerateTableDdl
+  def run_compatibility_check
+    # do nothing for now
+  end
+
+  def data_source_type_display_name
+    "Oracle database"
+  end
+
+  def data_entry_prefs
+    de['oracle_data_entry_preference']
+  end
+
+  def each_source_tabledef(tables, options, &block)
+    # Oracle options.
+    tables = tables.clone
+    if tables.to_s == '' || tables.to_s == '[]'
+      raise ArgumentError, "tables is nil or empty"
+    end
+    _each_tabledef(tables, options, &block)
+  end
+
+  private
+
+  def table_meta(tables, options)
+    Flydata::SourceOracle::TableMeta.new(options, tables)
+  end
+
+  def _each_tabledef(tables, options, &block)
+    all_table_meta = options[:table_meta]
+    unless all_table_meta
+      all_table_meta = table_meta(tables, options)
+      all_table_meta.reload
+    end
+
+    missing_tables = []
+
+    tables.each do |table|
+      t_meta = all_table_meta[table]
+      unless t_meta && !t_meta.empty?
+        missing_tables << table
+        next
+      end
+
+      table_def = t_meta[:table_def]
+      if table_def
+        yield(table_def, nil)
+      else
+        yield(nil, t_meta[:table_def_err])
+      end
+    end
+
+    missing_tables
+  end
+end
+
+end
+end

data/lib/flydata/source_oracle/table_meta.rb

@@ -0,0 +1,220 @@
+require 'flydata/table_meta'
+require 'flydata-core/oracle/oracle_client'
+require 'flydata-core/table_def/oracle_table_def'
+
+
+module Flydata
+module SourceOracle
+
+  # Fetch and keep table meta information
+  #
+  #  <table-name(Symbol)>:
+  #    table_name: <String>               # Table name
+  #    table_schema: <String> or nil      # Schema name
+  #    primary_keys: <Array of String>    # Set primary key names. ex: ['group_id', 'category_id']
+  #    pk_positions: <Array of Integer>   # Set the ordinal position of primary keys. ex: [1, 3]
+  #    max_row_size: <Integer>            # byte, calculated based on column size
+  #    max_num_rows_per_query>: <Integer> # max number of rows per query
+  #    raw_columns: <Hash>                # raw information schema data
+  #    columns: table_def.columns
+  #    table_def:
+  #      columns:
+  #        table:
+  #        column:
+  #        type:
+  #        not_null:
+  #        primary_key:
+  #        default:
+  #        column_size: (new)  # Set in `OracleTableDef.parse_one_column_def`
+  #
+  class TableMeta < Flydata::TableMeta
+
+    GET_TABLE_META_SQL = <<EOS
+SELECT
+   tbls.OWNER                        AS schema
+  ,cols.TABLE_NAME                   AS table_name
+  ,cols.COLUMN_NAME                  AS column_name
+  ,cols.COLUMN_ID                    AS ordinal_position
+  ,(CASE
+    WHEN pk_cols.CONSTRAINT_NAME IS NULL THEN
+      'f'
+    ELSE
+      't'
+    END)                             AS is_primary
+  ,pk_cols.CONSTRAINT_NAME           AS pk_constraint_name
+  ,pk_cols.KEY_ORDINAL               AS pk_key_ordinal
+  ,(CASE
+    WHEN uq_cols.CONSTRAINT_NAME IS NULL THEN
+      'f'
+    ELSE
+      't'
+   END)                              AS is_unique
+  ,uq_cols.CONSTRAINT_NAME           AS uq_constraint_name
+  ,uq_cols.KEY_ORDINAL               AS uq_key_ordinal
+  ,cols.DATA_TYPE                    AS column_data_type
+  ,cols.DATA_DEFAULT                 AS column_default
+  ,cols.NULLABLE                     AS is_nullable
+  ,cols.DATA_LENGTH                  AS max_length
+  ,cols.DATA_PRECISION               AS precision
+  ,cols.DATA_SCALE                   AS scale
+FROM
+  (SELECT * FROM ALL_TABLES WHERE OWNER = %{schema}) tbls
+  INNER JOIN ALL_TAB_COLUMNS cols
+    ON  cols.OWNER      = tbls.OWNER
+    AND cols.TABLE_NAME = tbls.TABLE_NAME
+  LEFT OUTER JOIN (
+    SELECT
+       cons_inner.CONSTRAINT_NAME  AS constraint_name
+      ,cons_columns_inner.POSITION AS key_ordinal
+      ,tbls_inner.TABLE_NAME       AS table_name
+      ,cols_inner.COLUMN_NAME      AS col_name
+      ,cols_inner.COLUMN_ID        AS col_id
+    FROM
+      (SELECT * FROM ALL_TABLES WHERE OWNER = %{schema}) tbls_inner
+      INNER JOIN ALL_TAB_COLUMNS cols_inner
+        ON  cols_inner.OWNER      = tbls_inner.OWNER
+        AND cols_inner.TABLE_NAME = tbls_inner.TABLE_NAME
+      INNER JOIN ALL_CONSTRAINTS cons_inner
+        ON  cons_inner.OWNER           = cols_inner.OWNER
+        AND cons_inner.CONSTRAINT_TYPE = 'P'
+        AND cons_inner.TABLE_NAME      = tbls_inner.TABLE_NAME
+      INNER JOIN ALL_CONS_COLUMNS cons_columns_inner
+        ON  cons_columns_inner.OWNER           = cons_inner.OWNER
+        AND cons_columns_inner.CONSTRAINT_NAME = cons_inner.CONSTRAINT_NAME
+        AND cons_columns_inner.TABLE_NAME      = cons_inner.TABLE_NAME
+        AND cons_columns_inner.COLUMN_NAME     = cols_inner.COLUMN_NAME
+  ) pk_cols
+    ON  pk_cols.TABLE_NAME = cols.TABLE_NAME
+    AND pk_cols.COL_NAME   = cols.COLUMN_NAME
+    AND pk_cols.COL_ID     = cols.COLUMN_ID
+  LEFT OUTER JOIN (
+    SELECT
+       cons_inner.CONSTRAINT_NAME  AS constraint_name
+      ,cons_columns_inner.POSITION AS key_ordinal
+      ,tbls_inner.TABLE_NAME       AS table_name
+      ,cols_inner.COLUMN_NAME      AS col_name
+      ,cols_inner.COLUMN_ID        AS col_id
+    FROM
+      (SELECT * FROM ALL_TABLES WHERE OWNER = %{schema}) tbls_inner
+      INNER JOIN ALL_TAB_COLUMNS cols_inner
+        ON  cols_inner.OWNER      = tbls_inner.OWNER
+        AND cols_inner.TABLE_NAME = tbls_inner.TABLE_NAME
+      INNER JOIN ALL_CONSTRAINTS cons_inner
+        ON  cons_inner.OWNER           = cols_inner.OWNER
+        AND cons_inner.CONSTRAINT_TYPE = 'U'
+        AND cons_inner.TABLE_NAME      = tbls_inner.TABLE_NAME
+      INNER JOIN ALL_CONS_COLUMNS cons_columns_inner
+        ON  cons_columns_inner.OWNER           = cons_inner.OWNER
+        AND cons_columns_inner.CONSTRAINT_NAME = cons_inner.CONSTRAINT_NAME
+        AND cons_columns_inner.TABLE_NAME      = cons_inner.TABLE_NAME
+        AND cons_columns_inner.COLUMN_NAME     = cols_inner.COLUMN_NAME
+   ) uq_cols
+     ON  uq_cols.TABLE_NAME = cols.TABLE_NAME
+     AND uq_cols.COL_NAME   = cols.COLUMN_NAME
+     AND uq_cols.COL_ID     = cols.COLUMN_ID
+ORDER BY
+  table_name, ordinal_position
+EOS
+
+
+    GET_CURRENT_SCN_SQL = "SELECT current_scn from v$database"
+
+    DEFAULT_MAX_FETCH_RECORD_SIZE = 50000
+    #DEFAULT_MAX_FETCH_RECORD_SIZE = 8
+
+    def initialize(options, tables, schema = nil)
+      @options = options
+      @database = options[:dbname] || options[:database] || options['database']
+      @tables = tables
+      username = options[:username] || options['username']
+      @schema = (schema || options[:schema] || options['schema'] || username).to_s.strip.upcase
+    end
+
+    attr_reader :current_scn
+
+    def reload(oracle_client = nil)
+      schema = {
+        database: "'#{@database}'",
+        schema: "'#{@schema}'",
+        tables: @tables.collect{|t| "'#{t}'"}.join(','),
+      }
+
+      conn = oracle_client
+      if conn.nil?
+        local_conn = conn = FlydataCore::Oracle::OracleClient.new(@options)
+      end
+
+      sql = GET_TABLE_META_SQL % schema
+
+      # Set table_meta
+      columns = conn.query(sql)
+      @table_meta = build_table_meta(columns)
+
+      # Set current scn
+      current_scn_str = conn.query(GET_CURRENT_SCN_SQL).fetch.first
+
+      raise "Invalid scn is retrieved" if current_scn_str.nil?
+      @current_scn = current_scn_str.to_i
+
+      self
+    ensure
+      local_conn.close rescue nil if local_conn
+    end
+
+    def build_table_meta(columns)
+      ret = Hash.new{|h,k| h[k]={} }
+
+      # Put ret[<table-name-sym>][:raw_columns]
+      while (col = columns.fetch_hash)
+        column_name = col['COLUMN_NAME'].to_sym
+        table_name = col['TABLE_NAME'].to_sym
+        t_meta = ret[table_name]
+        t_meta[:raw_columns] = Hash.new {|h,k| h[k] = []} unless t_meta[:raw_columns]
+        t_meta[:raw_columns][column_name] << col
+      end
+
+      ret.each do |table_name, t_meta|
+        begin
+          table_def = FlydataCore::TableDef::OracleTableDef.create(
+                      t_meta[:raw_columns].values, @options)
+        rescue FlydataCore::TableDefError => e
+          t_meta.merge!(
+            table_name: table_name,
+            table_def_err: e,
+          )
+          # Skip when getting an error when parsing the columns
+          next
+        end
+
+        primary_keys = []
+        pk_positions = []
+        table_def.columns.each.with_index(1) do |col, index|
+          col_name = col[:column]
+          if col["CONSTRAINT_TYPE"] == 'P'
+            primary_keys << col_name
+            pk_positions << index.to_s
+          end
+        end
+
+        t_meta.merge!(
+          table_name: table_name.to_s,
+          table_schema: @schema,
+          primary_keys: primary_keys,
+          pk_positions: pk_positions,
+          max_num_rows_per_query: DEFAULT_MAX_FETCH_RECORD_SIZE,
+          columns: table_def.columns,
+          table_def: table_def,
+        )
+      end
+
+      ret
+    end
+
+    def calc_column_size(column)
+      #TODO: Implement the check logic based on column type
+      124
+    end
+  end
+
+end
+end