flydata 0.7.12 → 0.7.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77a3df344141ff66884b5deecb9b947c4616b64d
-  data.tar.gz: d3a7bbaa5ba44fb27d24d0d37e6844b4be0b809a
+  metadata.gz: 30343349c710d665850b71f57d39308b6db3ac9c
+  data.tar.gz: 5c4ecdd659a4dc2e5a20bd12746bd2903315e71d
 SHA512:
-  metadata.gz: dcec46188c13aa32bdb56974f81642b71da51cfe1a4de1cd08eade5fcbb8b70d1a5ad329cbc1cd7ddc020571deee70bf346dc1d845362ba262dc79d5cde40a93
-  data.tar.gz: f8d1f84ebcea486f2aa30cafbdef5851c8876b6f9d21ff3220c976fd4b7306d42faa94d53ad7df9f250fd0b6f922484fda867d4bae563db828680c8a50b9d353
+  metadata.gz: cffa5e91a09f5a7e9091dcd198db4e15d3de9c9666d753b888e06b1a8735de494371733353b1503f07a0dff41c8704ed29b8b143177d46649397f6580e7af548
+  data.tar.gz: c13cb4a781ab2a263a9c88f52700a4d21e8ee11c1ab38b5c92620f2718846eae13c28394d9714e20ff58b6b81c6c8eb9611a764e00567541f33f83b91fdef0db

data/Gemfile

@@ -30,5 +30,6 @@ group :development do
   gem 'activerecord', '~> 4.0', '>= 4.0.0'
   gem 'protected_attributes', '~> 1.0', '>= 1.0.8'
   gem 'pry'
+  #gem 'ruby-oci8'
   gem 'rake-compiler', '~> 0.9', '>= 0.9.5'
 end

data/VERSION

@@ -1 +1 @@
-0.7.12
+0.7.13

data/flydata-core/lib/flydata-core/oracle/config.rb

@@ -0,0 +1,25 @@
+module FlydataCore
+  module Oracle
+    class Config
+      def self.build_oci_uri(db_conf)
+        db_opts = [:uri,
+                   :host,
+                   :port,
+                   :username,
+                   :password,
+                   :database,
+                   ].inject({}) { |h, sym|
+                     if db_conf.has_key?(sym)
+                       h[sym] = db_conf[sym]
+                     elsif db_conf[sym.to_s]
+                       h[sym] = db_conf[sym.to_s]
+                     end
+                     h
+        }
+
+        db_opts[:uri] ||
+        "#{db_opts[:username]}/#{db_opts[:password]}@#{db_opts[:host]}:#{db_opts[:port]}/#{db_opts[:database]}"
+      end
+    end
+  end
+end

data/flydata-core/lib/flydata-core/oracle/oracle_client.rb

@@ -0,0 +1,48 @@
+require 'oci8'
+require 'flydata-core/oracle/config'
+
+module FlydataCore
+module Oracle
+
+class OracleClient
+  def initialize(dbconf)
+    @dbconf = dbconf
+  end
+
+  attr_reader :dbconf
+
+  def establish_connection
+    @conn = create_connection if @conn.nil?
+  end
+
+  def query(query, params = {})
+    establish_connection
+
+    cursor = @conn.parse(query)
+    case params
+    when Hash
+      params.each {|k, value| cursor.bind_param(k, value) }
+    when Array
+      params.each.with_index(1) {|value, i| cursor.bind_param(i, value) }
+    end
+    cursor.exec
+    cursor
+  end
+
+  def close
+    if @conn
+      @conn.logoff
+      @conn = nil
+    end
+  end
+
+  private
+
+  def create_connection
+    uri = FlydataCore::Oracle::Config.build_oci_uri(@dbconf)
+    OCI8.new(uri)
+  end
+end
+
+end
+end

data/flydata-core/lib/flydata-core/oracle/query_helper.rb

@@ -0,0 +1,20 @@
+module FlydataCore
+module Oracle
+
+class QueryHelper
+  # Set default schema if a schmema name is not given
+  def self.schema_as_value(schema, user)
+    s = schema.to_s.strip
+    if s.empty?
+      s = user.to_s.strip
+    end
+    "'#{s.upcase}'"
+  end
+
+  def self.tables_as_value(tables)
+    tables.collect{|t| "'#{t.upcase}'"}.join(",")
+  end
+end
+
+end
+end

data/flydata-core/lib/flydata-core/oracle/source_pos.rb

@@ -0,0 +1,63 @@
+require 'json'
+
+module FlydataCore
+module Oracle
+
+class SourcePos
+  include Comparable
+
+  # Source Position data for Oracle
+  #
+  # Use system change number (SCN) for managing a consitent check point in
+  # Oracle database. SCN is a unsigned integer value.
+  def initialize(scn_or_obj, pk_values = nil)
+    if scn_or_obj.kind_of?(self.class)
+      scn_or_obj.tap do |s|
+        @scn = s.scn
+        @pk_values = s.pk_values
+      end
+    else
+      @scn = scn_or_obj.to_i if scn_or_obj
+      @pk_values = pk_values  # must be array or nil
+    end
+  end
+
+  attr_reader :scn
+  attr_reader :pk_values
+
+  def empty?
+    @scn.to_s.empty?
+  end
+
+  def to_s
+    pk_values = @pk_values ? @pk_values.to_json : ''
+    "#{@scn}\t#{pk_values}"
+  end
+
+  def <=>(other)
+    if @scn != other.scn
+      return @scn <=> other.scn
+    elsif @pk_values.nil? && !other.pk_values.nil?
+      1
+    elsif !@pk_values.nil? && other.pk_values.nil?
+      -1
+    elsif @pk_values == other.pk_values
+      0
+    else
+      @pk_values.to_s <=> other.pk_values.to_s
+    end
+  end
+
+  def self.load(str)
+    scn, pk_values = str.split("\t").collect{|v| v.strip}
+    pk_values = if pk_values.to_s.empty?
+                  nil
+                else
+                  JSON.parse(pk_values)
+                end
+    self.new(scn, pk_values)
+  end
+end
+
+end
+end

data/flydata-core/lib/flydata-core/table_def/oracle_table_def.rb

@@ -0,0 +1,167 @@
+require 'flydata-core/table_def/base'
+
+module FlydataCore
+module TableDef
+
+class OracleTableDef < Base
+
+  VALUE_CONVERTERS = {}
+
+  #TODO: Check supported data format one by one
+  TYPE_MAP_O2F = {
+    'NUMBER' => {type: 'int8'},
+    'BIGINT' => {type: 'int8'},
+    'INT8' => {type: 'int8'},
+    'SERIAL8' => {type: 'serial8'},
+    'CHARACTER' => {type: 'varchar', width_attrs:["MAX_LENGTH"], def_width:[1]},
+    'CHARACTER varying' => {type: 'varchar',
+                            width_attrs:["MAX_LENGTH"],
+                            def_width:[1]},
+    'VARCHAR' => {type: 'varchar',
+                            width_attrs:["MAX_LENGTH"],
+                            def_width:[1]},
+    'VARCHAR2' => {type: 'varchar',
+                            width_attrs:["MAX_LENGTH"],
+                            def_width:[1]},
+    'DATE' => {type: 'date'},
+    'DOUBLE PRECISION' => {type: 'float8'},
+    'FLOAT8' => {type: 'float8'},
+    'INTEGER' => {type: 'int4'},
+    'INT' => {type: 'int4'},
+    'INT4' => {type: 'int4'},
+    'NUMERIC' => {type: 'numeric',
+                  width_attrs:["PRECISION", "SCALE"]
+                  #can be no width values
+                 },
+    'REAL' => {type: 'float4'},
+    'FLOAT4' => {type: 'float4'},
+    'SMALLINT' => {type: 'int2'},
+    'INT2' => {type: 'int2'},
+    'SMALLSERIAL' => {type: 'serial2'},
+    'TEXT' => {type: 'text'},
+    'TIMESTAMP' => {type: 'datetime'},
+    :default => {type: '_unsupported'},
+  }
+
+  def self.convert_to_flydata_type(information_schema_columns)
+    ora_type = information_schema_columns["COLUMN_DATA_TYPE"]
+    raise "Unknown Oracle type or internal error.  type:#{ora_type}" unless ora_type
+    unless TYPE_MAP_O2F.has_key?(ora_type)
+      ora_type = :default
+    end
+    type_hash = TYPE_MAP_O2F[ora_type]
+    flydata_type = type_hash[:type]
+    ret_type = flydata_type
+
+    width_values = get_width_values(information_schema_columns, type_hash)
+    if width_values
+      ret_type += "(#{width_values.join(",")})"
+    end
+
+    if type_hash[:override]
+      ret_type = type_hash[:override].call(ret_type, ora_type, flydata_type)
+    end
+    ret_type
+  end
+
+  private
+
+  def self.get_width_values(information_schema_columns, type_hash)
+    values = []
+    if type_hash.has_key?(:width_attrs)
+      values = type_hash[:width_attrs].collect{|attr| information_schema_columns[attr].to_i }
+    end
+
+    if type_hash.has_key?(:def_width)
+      if values.nil? || values.size != type_hash[:def_width].size
+        raise "The number of the default values must match the number of width attributes. column:#{information_schema_columns[:column_name]} type:#{type_hash[:type]} def_width:#{type_hash[:def_width].inspect} width_attrs:#{type_hash[:width_attrs].inspect}"
+      end
+      values = values.each_with_index.collect {|v, i| v ? v : type_hash[:def_width][i]}
+    end
+    values.pop until values.empty? || values.last # remove trailing nil
+    if values.any?{|v| v.nil?}
+       raise "nil value is not allowed"
+    end
+    values.empty? ? nil : values
+  end
+
+  def self._create(information_schema_columns, options)
+    table_def = information_schema_columns.collect {|iscol| iscol.first}.inspect
+    table_name = nil
+    columns = []
+    column_def = {}
+    # TODO: Set UTF-8 to NLS_LANG whne creating a connection
+    # Otherwise the user will see "Warning: NLS_LANG is not set. fallback to US7ASCII"
+    #
+    # SourceOracle uses UTF8 client encoding no matter what the server
+    # encoding is.
+    default_charset = 'UTF_8'
+    default_charset_oracle = 'UTF8'
+    comment = nil
+    unique_keys_hash = Hash.new {|h, k| h[k] = []}
+
+    information_schema_columns.each do |iscol_arr|
+      # An iscol_arr represents a column. Column information in all elements in iscol_arr is the same.
+      # Only difference between elements is index information.
+      iscol = iscol_arr.first
+      column = parse_one_column_def(iscol)
+      if table_name
+        unless table_name == column[:table]
+          raise "Table name must match through all columns.  Got `#{table_name}` and `#{column[:table]}`"
+        end
+      else
+        table_name = column[:table]
+      end
+      columns << column
+      coldef = iscol.dup.tap{|c|
+        c.each{|k, v| c[k] = v.to_i if v.kind_of?(BigDecimal) }
+      }.inspect
+      column_def[column[:column]] = coldef
+
+      # TODO: Handle unique keys
+      #iscol_arr.each do |iscol|
+      #  # gather information for unique keys that this column belongs to.
+      #  if iscol['IS_PRIMARY'] == 'f' && iscol['IS_UNIQUE'] == 't'
+      #    unique_keys_hash[iscol['CONSTRAINT_NAME'].to_sym] << iscol['COLUMN_NAME']
+      #  end
+      #end
+    end
+    unique_keys = unique_keys_hash.values
+
+    [table_def, table_name, columns, column_def, unique_keys, default_charset, default_charset_oracle, comment]
+  end
+
+  def self.parse_one_column_def(information_schema_column)
+    column = {}
+    column[:table] = information_schema_column["TABLE_NAME"]
+    column[:column] = information_schema_column["COLUMN_NAME"]
+    column[:type] = convert_to_flydata_type(information_schema_column)
+    column[:not_null] = true if information_schema_column["IS_NULLABLE"] == "N"
+    column[:primary_key] = true if information_schema_column["IS_PRIMARY"] == "t"
+    column[:default] = case column[:type]
+      when 'boolean'
+        to_boolean(information_schema_column["COLUMN_DEFAULT"])
+      when '_unsupported'
+        # omit default value
+        nil
+      else
+        information_schema_column["COLUMN_DEFAULT"] # TODO nil handling
+      end
+    column
+  end
+
+  #TODO: Revisit how to handle boolean default value
+  def self.to_boolean(col_value)
+    return nil if col_value.nil?
+    # Catch all possible literal for boolean type in Oracle.
+    # Actual col_value coming in here is:
+    #   'true' or 'false' for default value
+    #   't' or 'f' for column value
+    return true if col_value.to_s =~ /^(t|true|y|yes|on|1)$/i
+    return false if col_value.to_s =~ /^(f|false|n|no|off|0)$/i
+    raise "Invalid default value for Oracle boolean type:`#{col_value}`"
+  end
+end
+
+end
+end

data/flydata-core/spec/oracle/config_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+require 'flydata-core/oracle/config'
+
+module FlydataCore
+  module Oracle
+    describe Config do
+      describe '.build_oci_uri' do
+        subject { described_class.build_oci_uri(conf) }
+
+        context 'with uri conf' do
+          let(:expected_uri) { "admin/welcome@localhost:1521/ORCL" }
+          let(:conf) {
+            {
+              uri: expected_uri,
+              host: 'localhost',
+              port: 1234,
+              username: 'testuser',
+              password: 'password',
+              database: 'testdb',
+              user: 'testuser',
+              dbname: 'testdb',
+            }
+          }
+          it { is_expected.to eq(expected_uri) }
+        end
+
+        context 'with conf having dbname' do
+          let(:expected_uri) { "testuser/password@localhost:1234/testdb" }
+          let(:conf) {
+            {
+              host: 'localhost',
+              port: 1234,
+              username: 'testuser',
+              password: 'password',
+              database: 'testdb',
+              user: 'testuser',
+              dbname: 'testdb',
+            }
+          }
+          it { is_expected.to eq(expected_uri) }
+        end
+      end
+    end
+  end
+end

data/flydata-core/spec/oracle/source_pos_spec.rb

@@ -0,0 +1,101 @@
+require 'flydata-core/oracle/source_pos'
+
+module FlydataCore
+module Oracle
+
+describe SourcePos do
+  let(:subject_object) { described_class.new(scn, pk_values) }
+
+  let(:scn) { "123456" }
+  let(:pk_values) { nil }
+
+  describe '#initialize' do
+    context 'with scn and pk_values' do
+      it { expect(subject_object.scn).to eq(scn.to_i) }
+      it { expect(subject_object.pk_values).to eq(pk_values) }
+    end
+
+    context 'with source pos object' do
+      let(:scn) { '123456' }
+      let(:pk_values) { [{'id' => '1000'}] }
+      let(:obj) { described_class.new(scn, pk_values) }
+      subject { described_class.new(obj) }
+
+      it { expect(subject.scn).to eq(scn.to_i) }
+      it { expect(subject.pk_values).to eq(pk_values) }
+    end
+  end
+
+  describe '#to_s' do
+    subject { subject_object.to_s }
+
+    context 'when pk_values is nil' do
+      let(:pk_values) { nil }
+      it { is_expected.to eq %Q|#{scn}\t| }
+    end
+    context 'when pk_values is not nil' do
+      let(:pk_values) { { "user_id" => 1, "address_id" => 3 } }
+      it { is_expected.to eq %Q|#{scn}\t{"user_id":1,"address_id":3}| }
+    end
+  end
+
+  describe '.load' do
+    subject { described_class.load(str) }
+
+    context 'without pk_values' do
+      let(:str) { %Q|#{scn}\t| }
+      it { is_expected.to eq described_class.new(scn) }
+    end
+    context 'with pk_values' do
+      let(:str) { %Q|#{scn}\t{"user_id":1,"address_id":3}| }
+      it do
+        is_expected.to eq described_class.new(scn,
+                                             "user_id" => 1, "address_id" => 3)
+      end
+    end
+  end
+
+  describe '#<=>' do
+    def src_pos(scn, pk_values = nil)
+      described_class.new(scn, pk_values)
+    end
+    context 'when both have no pk_values' do
+      it { expect(src_pos('1111') == src_pos('1111')).to be(true) }
+      it { expect(src_pos('1111') == src_pos('1112')).to be(false) }
+      it { expect(src_pos('1111') < src_pos('1112')).to be(true) }
+      it { expect(src_pos('1111') < src_pos('1110')).to be(false) }
+      it { expect(src_pos('1111') > src_pos('1110')).to be(true) }
+      it { expect(src_pos('1111') > src_pos('1112')).to be(false) }
+      it { expect(src_pos('1111') > src_pos('999')).to be(true) }
+    end
+
+    context 'when one of them has pk_values' do
+      let(:pkv) { [{'id'=>'10'}] }
+      it { expect(src_pos('1111',pkv) == src_pos('1111')).to be(false) }
+      it { expect(src_pos('1111',pkv) == src_pos('1112')).to be(false) }
+      it { expect(src_pos('1111',pkv) < src_pos('1111')).to be(true) }
+      it { expect(src_pos('1111',pkv) < src_pos('1112')).to be(true) }
+      it { expect(src_pos('1111',pkv) < src_pos('1110')).to be(false) }
+      it { expect(src_pos('1111',pkv) > src_pos('1110')).to be(true) }
+      it { expect(src_pos('1111',pkv) > src_pos('1112')).to be(false) }
+    end
+
+    context 'when both have pk_values' do
+      let(:pkv1) { [{'id'=>'10'}] }
+      let(:pkv2) { [{'name'=>'akira'}] }
+      it { expect(src_pos('1111',pkv1) == src_pos('1111',pkv1)).to be(true) }
+      it { expect(src_pos('1111',pkv1) == src_pos('1111',pkv2)).to be(false) }
+      it { expect(src_pos('1111',pkv1) == src_pos('1112',pkv2)).to be(false) }
+      it { expect(src_pos('1111',pkv1) < src_pos('1111',pkv2)).to be(true) }
+      it { expect(src_pos('1111',pkv1) < src_pos('1111',pkv1)).to be(false) }
+      it { expect(src_pos('1111',pkv1) < src_pos('1112',pkv2)).to be(true) }
+      it { expect(src_pos('1111',pkv1) < src_pos('1110',pkv2)).to be(false) }
+      it { expect(src_pos('1111',pkv1) > src_pos('1110',pkv2)).to be(true) }
+      it { expect(src_pos('1111',pkv1) > src_pos('1112',pkv2)).to be(false) }
+      it { expect(src_pos('1111',pkv1) > src_pos('1111',pkv2)).to be(false) }
+    end
+  end
+end
+
+end
+end