flydata 0.7.12 → 0.7.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +1 -0
  3. data/VERSION +1 -1
  4. data/flydata-core/lib/flydata-core/oracle/config.rb +25 -0
  5. data/flydata-core/lib/flydata-core/oracle/oracle_client.rb +48 -0
  6. data/flydata-core/lib/flydata-core/oracle/query_helper.rb +20 -0
  7. data/flydata-core/lib/flydata-core/oracle/source_pos.rb +63 -0
  8. data/flydata-core/lib/flydata-core/table_def/oracle_table_def.rb +167 -0
  9. data/flydata-core/spec/oracle/config_spec.rb +45 -0
  10. data/flydata-core/spec/oracle/source_pos_spec.rb +101 -0
  11. data/flydata.gemspec +0 -0
  12. data/lib/flydata/command/sync.rb +14 -4
  13. data/lib/flydata/source.rb +1 -0
  14. data/lib/flydata/source/sync_repair.rb +25 -0
  15. data/lib/flydata/source_mysql/generate_source_dump.rb +2 -1
  16. data/lib/flydata/source_mysql/mysql_accessible.rb +30 -0
  17. data/lib/flydata/source_mysql/parser/dump_parser.rb +0 -40
  18. data/lib/flydata/source_mysql/sync_database_size_check.rb +29 -0
  19. data/lib/flydata/source_mysql/sync_repair.rb +26 -0
  20. data/lib/flydata/source_oracle/data_entry.rb +24 -0
  21. data/lib/flydata/source_oracle/generate_source_dump.rb +184 -0
  22. data/lib/flydata/source_oracle/oracle_component.rb +12 -0
  23. data/lib/flydata/source_oracle/parse_dump_and_send.rb +128 -0
  24. data/lib/flydata/source_oracle/plugin_support/context.rb +13 -0
  25. data/lib/flydata/source_oracle/plugin_support/source_position_file.rb +14 -0
  26. data/lib/flydata/source_oracle/query_based_sync/diff_query_generator.rb +122 -0
  27. data/lib/flydata/source_oracle/setup.rb +24 -0
  28. data/lib/flydata/source_oracle/source_pos.rb +18 -0
  29. data/lib/flydata/source_oracle/sync.rb +15 -0
  30. data/lib/flydata/source_oracle/sync_generate_table_ddl.rb +64 -0
  31. data/lib/flydata/source_oracle/table_meta.rb +220 -0
  32. data/lib/flydata/source_postgresql/sync_repair.rb +13 -0
  33. data/spec/flydata/source_mysql/generate_source_dump_spec.rb +2 -2
  34. metadata +27 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 77a3df344141ff66884b5deecb9b947c4616b64d
4
- data.tar.gz: d3a7bbaa5ba44fb27d24d0d37e6844b4be0b809a
3
+ metadata.gz: 30343349c710d665850b71f57d39308b6db3ac9c
4
+ data.tar.gz: 5c4ecdd659a4dc2e5a20bd12746bd2903315e71d
5
5
  SHA512:
6
- metadata.gz: dcec46188c13aa32bdb56974f81642b71da51cfe1a4de1cd08eade5fcbb8b70d1a5ad329cbc1cd7ddc020571deee70bf346dc1d845362ba262dc79d5cde40a93
7
- data.tar.gz: f8d1f84ebcea486f2aa30cafbdef5851c8876b6f9d21ff3220c976fd4b7306d42faa94d53ad7df9f250fd0b6f922484fda867d4bae563db828680c8a50b9d353
6
+ metadata.gz: cffa5e91a09f5a7e9091dcd198db4e15d3de9c9666d753b888e06b1a8735de494371733353b1503f07a0dff41c8704ed29b8b143177d46649397f6580e7af548
7
+ data.tar.gz: c13cb4a781ab2a263a9c88f52700a4d21e8ee11c1ab38b5c92620f2718846eae13c28394d9714e20ff58b6b81c6c8eb9611a764e00567541f33f83b91fdef0db
data/Gemfile CHANGED
@@ -30,5 +30,6 @@ group :development do
30
30
  gem 'activerecord', '~> 4.0', '>= 4.0.0'
31
31
  gem 'protected_attributes', '~> 1.0', '>= 1.0.8'
32
32
  gem 'pry'
33
+ #gem 'ruby-oci8'
33
34
  gem 'rake-compiler', '~> 0.9', '>= 0.9.5'
34
35
  end
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.7.12
1
+ 0.7.13
@@ -0,0 +1,25 @@
1
+ module FlydataCore
2
+ module Oracle
3
+ class Config
4
+ def self.build_oci_uri(db_conf)
5
+ db_opts = [:uri,
6
+ :host,
7
+ :port,
8
+ :username,
9
+ :password,
10
+ :database,
11
+ ].inject({}) { |h, sym|
12
+ if db_conf.has_key?(sym)
13
+ h[sym] = db_conf[sym]
14
+ elsif db_conf[sym.to_s]
15
+ h[sym] = db_conf[sym.to_s]
16
+ end
17
+ h
18
+ }
19
+
20
+ db_opts[:uri] ||
21
+ "#{db_opts[:username]}/#{db_opts[:password]}@#{db_opts[:host]}:#{db_opts[:port]}/#{db_opts[:database]}"
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,48 @@
1
+ require 'oci8'
2
+ require 'flydata-core/oracle/config'
3
+
4
+ module FlydataCore
5
+ module Oracle
6
+
7
+ class OracleClient
8
+ def initialize(dbconf)
9
+ @dbconf = dbconf
10
+ end
11
+
12
+ attr_reader :dbconf
13
+
14
+ def establish_connection
15
+ @conn = create_connection if @conn.nil?
16
+ end
17
+
18
+ def query(query, params = {})
19
+ establish_connection
20
+
21
+ cursor = @conn.parse(query)
22
+ case params
23
+ when Hash
24
+ params.each {|k, value| cursor.bind_param(k, value) }
25
+ when Array
26
+ params.each.with_index(1) {|value, i| cursor.bind_param(i, value) }
27
+ end
28
+ cursor.exec
29
+ cursor
30
+ end
31
+
32
+ def close
33
+ if @conn
34
+ @conn.logoff
35
+ @conn = nil
36
+ end
37
+ end
38
+
39
+ private
40
+
41
+ def create_connection
42
+ uri = FlydataCore::Oracle::Config.build_oci_uri(@dbconf)
43
+ OCI8.new(uri)
44
+ end
45
+ end
46
+
47
+ end
48
+ end
@@ -0,0 +1,20 @@
1
+ module FlydataCore
2
+ module Oracle
3
+
4
+ class QueryHelper
5
+ # Set default schema if a schmema name is not given
6
+ def self.schema_as_value(schema, user)
7
+ s = schema.to_s.strip
8
+ if s.empty?
9
+ s = user.to_s.strip
10
+ end
11
+ "'#{s.upcase}'"
12
+ end
13
+
14
+ def self.tables_as_value(tables)
15
+ tables.collect{|t| "'#{t.upcase}'"}.join(",")
16
+ end
17
+ end
18
+
19
+ end
20
+ end
@@ -0,0 +1,63 @@
1
+ require 'json'
2
+
3
+ module FlydataCore
4
+ module Oracle
5
+
6
+ class SourcePos
7
+ include Comparable
8
+
9
+ # Source Position data for Oracle
10
+ #
11
+ # Use system change number (SCN) for managing a consitent check point in
12
+ # Oracle database. SCN is a unsigned integer value.
13
+ def initialize(scn_or_obj, pk_values = nil)
14
+ if scn_or_obj.kind_of?(self.class)
15
+ scn_or_obj.tap do |s|
16
+ @scn = s.scn
17
+ @pk_values = s.pk_values
18
+ end
19
+ else
20
+ @scn = scn_or_obj.to_i if scn_or_obj
21
+ @pk_values = pk_values # must be array or nil
22
+ end
23
+ end
24
+
25
+ attr_reader :scn
26
+ attr_reader :pk_values
27
+
28
+ def empty?
29
+ @scn.to_s.empty?
30
+ end
31
+
32
+ def to_s
33
+ pk_values = @pk_values ? @pk_values.to_json : ''
34
+ "#{@scn}\t#{pk_values}"
35
+ end
36
+
37
+ def <=>(other)
38
+ if @scn != other.scn
39
+ return @scn <=> other.scn
40
+ elsif @pk_values.nil? && !other.pk_values.nil?
41
+ 1
42
+ elsif !@pk_values.nil? && other.pk_values.nil?
43
+ -1
44
+ elsif @pk_values == other.pk_values
45
+ 0
46
+ else
47
+ @pk_values.to_s <=> other.pk_values.to_s
48
+ end
49
+ end
50
+
51
+ def self.load(str)
52
+ scn, pk_values = str.split("\t").collect{|v| v.strip}
53
+ pk_values = if pk_values.to_s.empty?
54
+ nil
55
+ else
56
+ JSON.parse(pk_values)
57
+ end
58
+ self.new(scn, pk_values)
59
+ end
60
+ end
61
+
62
+ end
63
+ end
@@ -0,0 +1,167 @@
1
+ require 'flydata-core/table_def/base'
2
+
3
+ module FlydataCore
4
+ module TableDef
5
+
6
+ class OracleTableDef < Base
7
+
8
+ VALUE_CONVERTERS = {}
9
+
10
+ #TODO: Check supported data format one by one
11
+ TYPE_MAP_O2F = {
12
+ 'NUMBER' => {type: 'int8'},
13
+ 'BIGINT' => {type: 'int8'},
14
+ 'INT8' => {type: 'int8'},
15
+ 'SERIAL8' => {type: 'serial8'},
16
+ 'CHARACTER' => {type: 'varchar', width_attrs:["MAX_LENGTH"], def_width:[1]},
17
+ 'CHARACTER varying' => {type: 'varchar',
18
+ width_attrs:["MAX_LENGTH"],
19
+ def_width:[1]},
20
+ 'VARCHAR' => {type: 'varchar',
21
+ width_attrs:["MAX_LENGTH"],
22
+ def_width:[1]},
23
+ 'VARCHAR2' => {type: 'varchar',
24
+ width_attrs:["MAX_LENGTH"],
25
+ def_width:[1]},
26
+ 'DATE' => {type: 'date'},
27
+ 'DOUBLE PRECISION' => {type: 'float8'},
28
+ 'FLOAT8' => {type: 'float8'},
29
+ 'INTEGER' => {type: 'int4'},
30
+ 'INT' => {type: 'int4'},
31
+ 'INT4' => {type: 'int4'},
32
+ 'NUMERIC' => {type: 'numeric',
33
+ width_attrs:["PRECISION", "SCALE"]
34
+ #can be no width values
35
+ },
36
+ 'REAL' => {type: 'float4'},
37
+ 'FLOAT4' => {type: 'float4'},
38
+ 'SMALLINT' => {type: 'int2'},
39
+ 'INT2' => {type: 'int2'},
40
+ 'SMALLSERIAL' => {type: 'serial2'},
41
+ 'TEXT' => {type: 'text'},
42
+ 'TIMESTAMP' => {type: 'datetime'},
43
+ :default => {type: '_unsupported'},
44
+ }
45
+
46
+ def self.convert_to_flydata_type(information_schema_columns)
47
+ ora_type = information_schema_columns["COLUMN_DATA_TYPE"]
48
+ raise "Unknown Oracle type or internal error. type:#{ora_type}" unless ora_type
49
+ unless TYPE_MAP_O2F.has_key?(ora_type)
50
+ ora_type = :default
51
+ end
52
+ type_hash = TYPE_MAP_O2F[ora_type]
53
+ flydata_type = type_hash[:type]
54
+ ret_type = flydata_type
55
+
56
+ width_values = get_width_values(information_schema_columns, type_hash)
57
+ if width_values
58
+ ret_type += "(#{width_values.join(",")})"
59
+ end
60
+
61
+ if type_hash[:override]
62
+ ret_type = type_hash[:override].call(ret_type, ora_type, flydata_type)
63
+ end
64
+ ret_type
65
+ end
66
+
67
+ private
68
+
69
+ def self.get_width_values(information_schema_columns, type_hash)
70
+ values = []
71
+ if type_hash.has_key?(:width_attrs)
72
+ values = type_hash[:width_attrs].collect{|attr| information_schema_columns[attr].to_i }
73
+ end
74
+
75
+ if type_hash.has_key?(:def_width)
76
+ if values.nil? || values.size != type_hash[:def_width].size
77
+ raise "The number of the default values must match the number of width attributes. column:#{information_schema_columns[:column_name]} type:#{type_hash[:type]} def_width:#{type_hash[:def_width].inspect} width_attrs:#{type_hash[:width_attrs].inspect}"
78
+ end
79
+ values = values.each_with_index.collect {|v, i| v ? v : type_hash[:def_width][i]}
80
+ end
81
+ values.pop until values.empty? || values.last # remove trailing nil
82
+ if values.any?{|v| v.nil?}
83
+ raise "nil value is not allowed"
84
+ end
85
+ values.empty? ? nil : values
86
+ end
87
+
88
+ def self._create(information_schema_columns, options)
89
+ table_def = information_schema_columns.collect {|iscol| iscol.first}.inspect
90
+ table_name = nil
91
+ columns = []
92
+ column_def = {}
93
+ # TODO: Set UTF-8 to NLS_LANG whne creating a connection
94
+ # Otherwise the user will see "Warning: NLS_LANG is not set. fallback to US7ASCII"
95
+ #
96
+ # SourceOracle uses UTF8 client encoding no matter what the server
97
+ # encoding is.
98
+ default_charset = 'UTF_8'
99
+ default_charset_oracle = 'UTF8'
100
+ comment = nil
101
+ unique_keys_hash = Hash.new {|h, k| h[k] = []}
102
+
103
+ information_schema_columns.each do |iscol_arr|
104
+ # An iscol_arr represents a column. Column information in all elements in iscol_arr is the same.
105
+ # Only difference between elements is index information.
106
+ iscol = iscol_arr.first
107
+ column = parse_one_column_def(iscol)
108
+ if table_name
109
+ unless table_name == column[:table]
110
+ raise "Table name must match through all columns. Got `#{table_name}` and `#{column[:table]}`"
111
+ end
112
+ else
113
+ table_name = column[:table]
114
+ end
115
+ columns << column
116
+ coldef = iscol.dup.tap{|c|
117
+ c.each{|k, v| c[k] = v.to_i if v.kind_of?(BigDecimal) }
118
+ }.inspect
119
+ column_def[column[:column]] = coldef
120
+
121
+ # TODO: Handle unique keys
122
+ #iscol_arr.each do |iscol|
123
+ # # gather information for unique keys that this column belongs to.
124
+ # if iscol['IS_PRIMARY'] == 'f' && iscol['IS_UNIQUE'] == 't'
125
+ # unique_keys_hash[iscol['CONSTRAINT_NAME'].to_sym] << iscol['COLUMN_NAME']
126
+ # end
127
+ #end
128
+ end
129
+ unique_keys = unique_keys_hash.values
130
+
131
+ [table_def, table_name, columns, column_def, unique_keys, default_charset, default_charset_oracle, comment]
132
+ end
133
+
134
+ def self.parse_one_column_def(information_schema_column)
135
+ column = {}
136
+ column[:table] = information_schema_column["TABLE_NAME"]
137
+ column[:column] = information_schema_column["COLUMN_NAME"]
138
+ column[:type] = convert_to_flydata_type(information_schema_column)
139
+ column[:not_null] = true if information_schema_column["IS_NULLABLE"] == "N"
140
+ column[:primary_key] = true if information_schema_column["IS_PRIMARY"] == "t"
141
+ column[:default] = case column[:type]
142
+ when 'boolean'
143
+ to_boolean(information_schema_column["COLUMN_DEFAULT"])
144
+ when '_unsupported'
145
+ # omit default value
146
+ nil
147
+ else
148
+ information_schema_column["COLUMN_DEFAULT"] # TODO nil handling
149
+ end
150
+ column
151
+ end
152
+
153
+ #TODO: Revisit how to handle boolean default value
154
+ def self.to_boolean(col_value)
155
+ return nil if col_value.nil?
156
+ # Catch all possible literal for boolean type in Oracle.
157
+ # Actual col_value coming in here is:
158
+ # 'true' or 'false' for default value
159
+ # 't' or 'f' for column value
160
+ return true if col_value.to_s =~ /^(t|true|y|yes|on|1)$/i
161
+ return false if col_value.to_s =~ /^(f|false|n|no|off|0)$/i
162
+ raise "Invalid default value for Oracle boolean type:`#{col_value}`"
163
+ end
164
+ end
165
+
166
+ end
167
+ end
@@ -0,0 +1,45 @@
1
+ require 'spec_helper'
2
+ require 'flydata-core/oracle/config'
3
+
4
+ module FlydataCore
5
+ module Oracle
6
+ describe Config do
7
+ describe '.build_oci_uri' do
8
+ subject { described_class.build_oci_uri(conf) }
9
+
10
+ context 'with uri conf' do
11
+ let(:expected_uri) { "admin/welcome@localhost:1521/ORCL" }
12
+ let(:conf) {
13
+ {
14
+ uri: expected_uri,
15
+ host: 'localhost',
16
+ port: 1234,
17
+ username: 'testuser',
18
+ password: 'password',
19
+ database: 'testdb',
20
+ user: 'testuser',
21
+ dbname: 'testdb',
22
+ }
23
+ }
24
+ it { is_expected.to eq(expected_uri) }
25
+ end
26
+
27
+ context 'with conf having dbname' do
28
+ let(:expected_uri) { "testuser/password@localhost:1234/testdb" }
29
+ let(:conf) {
30
+ {
31
+ host: 'localhost',
32
+ port: 1234,
33
+ username: 'testuser',
34
+ password: 'password',
35
+ database: 'testdb',
36
+ user: 'testuser',
37
+ dbname: 'testdb',
38
+ }
39
+ }
40
+ it { is_expected.to eq(expected_uri) }
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,101 @@
1
+ require 'flydata-core/oracle/source_pos'
2
+
3
+ module FlydataCore
4
+ module Oracle
5
+
6
+ describe SourcePos do
7
+ let(:subject_object) { described_class.new(scn, pk_values) }
8
+
9
+ let(:scn) { "123456" }
10
+ let(:pk_values) { nil }
11
+
12
+ describe '#initialize' do
13
+ context 'with scn and pk_values' do
14
+ it { expect(subject_object.scn).to eq(scn.to_i) }
15
+ it { expect(subject_object.pk_values).to eq(pk_values) }
16
+ end
17
+
18
+ context 'with source pos object' do
19
+ let(:scn) { '123456' }
20
+ let(:pk_values) { [{'id' => '1000'}] }
21
+ let(:obj) { described_class.new(scn, pk_values) }
22
+ subject { described_class.new(obj) }
23
+
24
+ it { expect(subject.scn).to eq(scn.to_i) }
25
+ it { expect(subject.pk_values).to eq(pk_values) }
26
+ end
27
+ end
28
+
29
+ describe '#to_s' do
30
+ subject { subject_object.to_s }
31
+
32
+ context 'when pk_values is nil' do
33
+ let(:pk_values) { nil }
34
+ it { is_expected.to eq %Q|#{scn}\t| }
35
+ end
36
+ context 'when pk_values is not nil' do
37
+ let(:pk_values) { { "user_id" => 1, "address_id" => 3 } }
38
+ it { is_expected.to eq %Q|#{scn}\t{"user_id":1,"address_id":3}| }
39
+ end
40
+ end
41
+
42
+ describe '.load' do
43
+ subject { described_class.load(str) }
44
+
45
+ context 'without pk_values' do
46
+ let(:str) { %Q|#{scn}\t| }
47
+ it { is_expected.to eq described_class.new(scn) }
48
+ end
49
+ context 'with pk_values' do
50
+ let(:str) { %Q|#{scn}\t{"user_id":1,"address_id":3}| }
51
+ it do
52
+ is_expected.to eq described_class.new(scn,
53
+ "user_id" => 1, "address_id" => 3)
54
+ end
55
+ end
56
+ end
57
+
58
+ describe '#<=>' do
59
+ def src_pos(scn, pk_values = nil)
60
+ described_class.new(scn, pk_values)
61
+ end
62
+ context 'when both have no pk_values' do
63
+ it { expect(src_pos('1111') == src_pos('1111')).to be(true) }
64
+ it { expect(src_pos('1111') == src_pos('1112')).to be(false) }
65
+ it { expect(src_pos('1111') < src_pos('1112')).to be(true) }
66
+ it { expect(src_pos('1111') < src_pos('1110')).to be(false) }
67
+ it { expect(src_pos('1111') > src_pos('1110')).to be(true) }
68
+ it { expect(src_pos('1111') > src_pos('1112')).to be(false) }
69
+ it { expect(src_pos('1111') > src_pos('999')).to be(true) }
70
+ end
71
+
72
+ context 'when one of them has pk_values' do
73
+ let(:pkv) { [{'id'=>'10'}] }
74
+ it { expect(src_pos('1111',pkv) == src_pos('1111')).to be(false) }
75
+ it { expect(src_pos('1111',pkv) == src_pos('1112')).to be(false) }
76
+ it { expect(src_pos('1111',pkv) < src_pos('1111')).to be(true) }
77
+ it { expect(src_pos('1111',pkv) < src_pos('1112')).to be(true) }
78
+ it { expect(src_pos('1111',pkv) < src_pos('1110')).to be(false) }
79
+ it { expect(src_pos('1111',pkv) > src_pos('1110')).to be(true) }
80
+ it { expect(src_pos('1111',pkv) > src_pos('1112')).to be(false) }
81
+ end
82
+
83
+ context 'when both have pk_values' do
84
+ let(:pkv1) { [{'id'=>'10'}] }
85
+ let(:pkv2) { [{'name'=>'akira'}] }
86
+ it { expect(src_pos('1111',pkv1) == src_pos('1111',pkv1)).to be(true) }
87
+ it { expect(src_pos('1111',pkv1) == src_pos('1111',pkv2)).to be(false) }
88
+ it { expect(src_pos('1111',pkv1) == src_pos('1112',pkv2)).to be(false) }
89
+ it { expect(src_pos('1111',pkv1) < src_pos('1111',pkv2)).to be(true) }
90
+ it { expect(src_pos('1111',pkv1) < src_pos('1111',pkv1)).to be(false) }
91
+ it { expect(src_pos('1111',pkv1) < src_pos('1112',pkv2)).to be(true) }
92
+ it { expect(src_pos('1111',pkv1) < src_pos('1110',pkv2)).to be(false) }
93
+ it { expect(src_pos('1111',pkv1) > src_pos('1110',pkv2)).to be(true) }
94
+ it { expect(src_pos('1111',pkv1) > src_pos('1112',pkv2)).to be(false) }
95
+ it { expect(src_pos('1111',pkv1) > src_pos('1111',pkv2)).to be(false) }
96
+ end
97
+ end
98
+ end
99
+
100
+ end
101
+ end