myreplicator 1.1.1 → 1.1.2
Sign up to get free protection for your applications and to get access to all the features.
- data/app/models/myreplicator/export.rb +5 -1
- data/app/views/myreplicator/exports/_form.html.erb +1 -2
- data/db/migrate/20130213211927_create_myreplicator_vertica_exports.rb~ +8 -0
- data/lib/configuration.rb~ +23 -0
- data/lib/exporter/mysql_exporter.rb~ +6 -0
- data/lib/exporter.rb~ +1 -0
- data/lib/loader/vertica/source_db.rb~ +30 -0
- data/lib/loader/vertica/vertica_loader.rb~ +175 -0
- data/lib/loader/vertica/vertica_sql.rb~ +60 -0
- data/lib/myreplicator/version.rb +1 -1
- data/lib/transporter/parallelizer.rb~ +5 -0
- data/test/dummy/app/controllers/test_controller.rb~ +8 -0
- data/test/dummy/config/database.yml +15 -13
- data/test/dummy/config/database.yml~ +54 -0
- data/test/dummy/config/myreplicator.yml +13 -9
- data/test/dummy/config/myreplicator.yml~ +23 -0
- data/test/dummy/db/development.sqlite3 +0 -0
- data/test/dummy/db/production.sqlite3 +0 -0
- data/test/dummy/log/development.log +2303 -2973
- data/test/dummy/tmp/cache/assets/{E76/790/sprockets%2Fdaf734b1fcabd6ca66b333ad0f0fdbf0 → C2E/D00/sprockets%2F667019818351638709494c01bddb5f68} +0 -0
- data/test/dummy/tmp/cache/assets/{D36/320/sprockets%2Fc662bfc4a04da6af6804481495bfa108 → C7F/970/sprockets%2F03f29761343991c6cd347bbd2184d560} +0 -0
- data/test/dummy/tmp/cache/assets/{D3C/EF0/sprockets%2F3b257fcef04542cf19da19f2f1c62761 → C83/BA0/sprockets%2F701a6339a558e5af28f150c161f43878} +0 -0
- data/test/dummy/tmp/cache/assets/C8B/150/sprockets%2F37163d05e55ad0b31b602ac5330412e3 +0 -0
- data/test/dummy/tmp/cache/assets/C98/250/sprockets%2F4af1200380a1a28632393d61a50f66cf +0 -0
- data/test/dummy/tmp/cache/assets/{DCB/970/sprockets%2Fffc91d7a12aab685d3484b0bfd5c824e → CBF/800/sprockets%2F00142a873933017aaa760316d0e2dcea} +0 -0
- data/test/dummy/tmp/cache/assets/CC5/870/sprockets%2F4e91734f6f02a779d39f8272f627dd24 +0 -0
- data/test/dummy/tmp/cache/assets/{DEC/900/sprockets%2Fe8ff7b68993fbee4dc864f1d5c373fe1 → CC6/7C0/sprockets%2F881234d3e954859c46bf49bb21f72d79} +0 -0
- data/test/dummy/tmp/cache/assets/CC9/1C0/sprockets%2Fa7a2b5a56180e1b21f783a9b30c03167 +0 -0
- data/test/dummy/tmp/cache/assets/CCC/A20/sprockets%2F6f3c2f4ae1f4d0219804a257410ce405 +0 -0
- data/test/dummy/tmp/cache/assets/{D82/050/sprockets%2F3dfff61e335fbf562b9a98f8742d4b90 → CD5/B90/sprockets%2Fc999d13a6a21113981c0d820e8043bdf} +0 -0
- data/test/dummy/tmp/cache/assets/{D06/850/sprockets%2Ff836842d203808ac78fc0cf30b4b0c72 → CD7/030/sprockets%2F9ba4859590582b8b72a650b2b00b6cd2} +0 -0
- data/test/dummy/tmp/cache/assets/CDE/780/sprockets%2F6982ce9303b4e69c26f2a1a246a180e7 +0 -0
- data/test/dummy/tmp/cache/assets/CE0/180/sprockets%2F8b9055a1b853d0a56ec2e94f79f37010 +0 -0
- data/test/dummy/tmp/cache/assets/{D4B/090/sprockets%2F3cfd81b97448f6103d74d0a7b197c7ff → CE5/670/sprockets%2Fe9e4122f1706626a21da6f8457f088ce} +0 -0
- data/test/dummy/tmp/cache/assets/D00/9B0/sprockets%2F2bc203eb4802b393a589093debb65c04 +0 -0
- data/test/dummy/tmp/cache/assets/D00/A90/sprockets%2F3fa6fcf2205c530208681446cbc36bd3 +0 -0
- data/test/dummy/tmp/cache/assets/{CBA/4C0/sprockets%2F54386f4be21f77c788524fa5b11d6850 → D06/5D0/sprockets%2F91850a20c0ddfa3d8814ca91870fb715} +0 -0
- data/test/dummy/tmp/cache/assets/D0B/B30/sprockets%2F55059589d999952597c5c86e5becaf4e +0 -0
- data/test/dummy/tmp/cache/assets/D0D/DA0/sprockets%2F34d6b075a16a5a58ff4050988d8bd4b0 +0 -0
- data/test/dummy/tmp/cache/assets/{D95/F60/sprockets%2F83fff218aa39ccc49352ff3f2a1a06b1 → D14/3A0/sprockets%2Fe59a60053fada52e8185281b4ee887a5} +0 -0
- data/test/dummy/tmp/cache/assets/D1B/D40/sprockets%2F7cc2142509e95f7168a82a652d8d9dea +0 -0
- data/test/dummy/tmp/cache/assets/D1D/700/sprockets%2Ffe9cb975216709e2881c74b3d1d3e35f +0 -0
- data/test/dummy/tmp/cache/assets/{DDC/CD0/sprockets%2Fb4b0d7dc6d4a0bcc2e5600bf5b15d2a1 → D20/A20/sprockets%2Fb503e93ff1966dd94d03e79f291d75c1} +0 -0
- data/test/dummy/tmp/cache/assets/D21/D40/sprockets%2Fe5a6e691816fa904f414ea89a6f0586f +0 -0
- data/test/dummy/tmp/cache/assets/D26/A70/sprockets%2F63d95ae156df465783cd78e95069b2cc +0 -0
- data/test/dummy/tmp/cache/assets/D2B/E60/sprockets%2F8615ecf645b553c959544af9ff8438da +0 -0
- data/test/dummy/tmp/cache/assets/D34/F70/sprockets%2Fb93de9992473bee94e369fe3198c529c +0 -0
- data/test/dummy/tmp/cache/assets/{D7F/D00/sprockets%2F10b0bae19e1ece503f23252ea80b5cf1 → D36/4D0/sprockets%2Fc050a64b5a803a638e155d05dcfe577d} +0 -0
- data/test/dummy/tmp/cache/assets/D3E/310/sprockets%2F333a2a7535eac766267ebb7d5c2ab489 +0 -0
- data/test/dummy/tmp/cache/assets/D3F/A00/sprockets%2F7a803404e1f60b8d672d763cb9ba8af5 +0 -0
- data/test/dummy/tmp/cache/assets/{C6A/110/sprockets%2F8578e16b59362054b8e3f580e45b7589 → D46/AA0/sprockets%2Ffcd7c5d9d04071ee45fe6a4865d53621} +0 -0
- data/test/dummy/tmp/cache/assets/{D3B/060/sprockets%2F049a04139e96acd00a20e6f88b538fbf → D50/570/sprockets%2F6c1d20a178f66e798958d1e437fdb5da} +0 -0
- data/test/dummy/tmp/cache/assets/D57/420/sprockets%2F937ea7429c536578ec7b688dee0cdf41 +0 -0
- data/test/dummy/tmp/cache/assets/{D74/F80/sprockets%2F9409bb9efe926f0130490fdca7c5ea73 → D69/6F0/sprockets%2F94fff7f55bc4c300b25f3f9361ac1a52} +0 -0
- data/test/dummy/tmp/cache/assets/{CA5/3A0/sprockets%2F24338fe3c579f79b67d198549674d48a → D74/DF0/sprockets%2F2db9c6478b8a29583ad20780ebeb4e4d} +0 -0
- data/test/dummy/tmp/cache/assets/D7C/E30/sprockets%2F0ba91e21bddfc7e1de102b22183e1e11 +0 -0
- data/test/dummy/tmp/cache/assets/D80/780/sprockets%2Fbe257c7a5788c2dfd272922fa399d7db +0 -0
- data/test/dummy/tmp/cache/assets/D85/9C0/sprockets%2Ffd0ec212bac1538379d0cc03b1e3bb84 +0 -0
- data/test/dummy/tmp/cache/assets/{F08/560/sprockets%2F5ab2cf5fb087fdef95cfe9fe6fabf9df → D86/D00/sprockets%2Fa4f32b4234d0d1bba272cd75e0d48e1d} +0 -0
- data/test/dummy/tmp/cache/assets/{CE8/8A0/sprockets%2F2ed5b94e22ac7a692e562c767944093a → D8B/B60/sprockets%2Faa32227c440a378ccd21218eefeb80bf} +0 -0
- data/test/dummy/tmp/cache/assets/{DA2/580/sprockets%2Fc8fe3cbc8b57ba600fe4fc3317e70875 → D99/6D0/sprockets%2Fa5e4558ff6808ad03b0f09e03fb4bfb3} +0 -0
- data/test/dummy/tmp/cache/assets/D9F/0B0/sprockets%2Faf0d2e69be3a6b56a76c20bf14d9e468 +0 -0
- data/test/dummy/tmp/cache/assets/DA7/E50/sprockets%2F47bf4f2b4afeac775e6d572a83343fb8 +0 -0
- data/test/dummy/tmp/cache/assets/DA8/910/sprockets%2Fab5775c4a837bd4d97ac394d473cda9b +0 -0
- data/test/dummy/tmp/cache/assets/{CC5/C90/sprockets%2F7a318379ad738c7f47aa44862e58e339 → DAA/060/sprockets%2Facc0d22b9d28123cc1c84d0db630d0ba} +0 -0
- data/test/dummy/tmp/cache/assets/DB0/8E0/sprockets%2F299733fb380a50e6fccf9fb3cd666eb9 +0 -0
- data/test/dummy/tmp/cache/assets/DD2/490/sprockets%2Fa452ee92a092bc2feabc572cce49896d +0 -0
- data/test/dummy/tmp/cache/assets/{D45/7C0/sprockets%2F7935d1666c0f916efdb4e44d10ce087c → DE1/320/sprockets%2F9f44ecdec8ceeef70871e15d88a448b1} +0 -0
- data/test/dummy/tmp/cache/assets/{C36/4A0/sprockets%2F545908d60919384cf74dd2658832a349 → DF8/5D0/sprockets%2Fb815ed34d61cfed96222daa3bfd1d84d} +0 -0
- data/test/dummy/tmp/cache/assets/E16/C70/sprockets%2F21ad93418ff75ceac86c7a85dfffe8f6 +0 -0
- data/test/dummy/tmp/cache/assets/E1C/AC0/sprockets%2Faff544a3a34eb7dab7d46b0cb2cd7b70 +0 -0
- data/test/dummy/tmp/cache/assets/{DF1/DA0/sprockets%2Fa761efb1e3e1c1cadd288b51b0ad68c8 → E2E/1F0/sprockets%2Fa24e3d7bc5ae4d40adf6f1b8fe94e7c3} +0 -0
- data/test/dummy/tmp/cache/assets/{C53/E40/sprockets%2Ff3eadad8548570531548a50069659241 → E35/4F0/sprockets%2F96b1cdf8db6a1c8eb8abcce05958ae74} +0 -0
- data/test/dummy/tmp/cache/assets/{D42/4D0/sprockets%2F11ab8d065b2f678ae997dac015972d7a → E3B/400/sprockets%2F5b0ce0efc07fdef9786e786cad5b84ba} +0 -0
- data/test/dummy/tmp/cache/assets/E57/E40/sprockets%2Ff3c7dda392bb05e4ffac29c5da6ffc11 +0 -0
- data/test/fixtures/myreplicator/vertica_exports.yml +11 -0
- metadata +253 -233
- data/test/dummy/load_exceptions.log +0 -1
- data/test/dummy/tmp/cache/assets/BE9/F00/sprockets%2F71b85f682226652265224563a92d012c +0 -0
- data/test/dummy/tmp/cache/assets/C55/470/sprockets%2F7493d2001e124f4fa090fb8664898327 +0 -0
- data/test/dummy/tmp/cache/assets/C92/350/sprockets%2F147976739b994ea0e5833048aebc5447 +0 -0
- data/test/dummy/tmp/cache/assets/CB0/AD0/sprockets%2F836537535ea3b4aa2d1b6789d11d4926 +0 -0
- data/test/dummy/tmp/cache/assets/CC3/5C0/sprockets%2Fe1d5f04ad1787592629a99b22f8c5567 +0 -0
- data/test/dummy/tmp/cache/assets/CCF/EA0/sprockets%2F5058706a1b7e72b34903fb00a7a0fc39 +0 -0
- data/test/dummy/tmp/cache/assets/CE1/370/sprockets%2Ff1a3b204bb618eba1846687099f640f7 +0 -0
- data/test/dummy/tmp/cache/assets/CE2/AD0/sprockets%2F51321b84e79ca92b09e4e6d0680b4f56 +0 -0
- data/test/dummy/tmp/cache/assets/CE8/5A0/sprockets%2F7367c62ac3a27578b50b924e2fe8b746 +0 -0
- data/test/dummy/tmp/cache/assets/CFB/530/sprockets%2F14190013244760ce4b30b2dbff73a8ef +0 -0
- data/test/dummy/tmp/cache/assets/D13/830/sprockets%2Fe39b0c5cb9a9084e314de7ae97026562 +0 -0
- data/test/dummy/tmp/cache/assets/D1B/360/sprockets%2Fe0e9b41868bf8a806f190df71512f83e +0 -0
- data/test/dummy/tmp/cache/assets/D1B/970/sprockets%2F0e1455d479dbf72d92cd23f28ed75842 +0 -0
- data/test/dummy/tmp/cache/assets/D43/100/sprockets%2Fe10493f07854d6da3a883ddd4e911c9a +0 -0
- data/test/dummy/tmp/cache/assets/D44/470/sprockets%2F9538d7d811d21efe47400772be4d4efb +0 -0
- data/test/dummy/tmp/cache/assets/D4F/E30/sprockets%2F7d4c622d36585e70d9f4ce6d83ae4a58 +0 -0
- data/test/dummy/tmp/cache/assets/D51/2D0/sprockets%2F7457c2dac213b0415022ac55fab5cb9a +0 -0
- data/test/dummy/tmp/cache/assets/D5B/5C0/sprockets%2F4e0d53616225ee8dbd80d06d3fa20ca2 +0 -0
- data/test/dummy/tmp/cache/assets/D6B/F60/sprockets%2Ff897ac3f53a8b248abb51abb812875c1 +0 -0
- data/test/dummy/tmp/cache/assets/D6D/7A0/sprockets%2Fece60a47c5639c873cfe144140cd72eb +0 -0
- data/test/dummy/tmp/cache/assets/D82/D30/sprockets%2Fcdd32b3338793e35a78f58efad64be96 +0 -0
- data/test/dummy/tmp/cache/assets/D91/D90/sprockets%2F60b95ba2a84d6c720cd412c9e0e44afd +0 -0
- data/test/dummy/tmp/cache/assets/D98/360/sprockets%2Fc9bf8e838b5afed8e8479876a9419a5e +0 -0
- data/test/dummy/tmp/cache/assets/D9B/790/sprockets%2F8182e1ebcce8ca3f3d30e26bd931654f +0 -0
- data/test/dummy/tmp/cache/assets/D9C/E60/sprockets%2F788bb2bebc8ed41035ca7cc50c68e553 +0 -0
- data/test/dummy/tmp/cache/assets/DBF/E30/sprockets%2Fd5e3199429abe1e1ce22c233ddbaec25 +0 -0
- data/test/dummy/tmp/cache/assets/DC5/EA0/sprockets%2Fc3821d00ce6bea8ebf265d439d14b6be +0 -0
- data/test/dummy/tmp/cache/assets/DCA/5F0/sprockets%2F15be5a98423136a3c8dcf7dff1ad45ad +0 -0
- data/test/dummy/tmp/cache/assets/E2D/2C0/sprockets%2Ff0c2e561abf62fabc69462c2cfeef3d8 +0 -0
- data/test/dummy/tmp/pids/server.pid +0 -1
@@ -232,9 +232,8 @@ function editInit(){
|
|
232
232
|
}
|
233
233
|
}
|
234
234
|
CronUI.translate();
|
235
|
-
<% if @export.export_to != 'vertica' %>
|
236
235
|
$("#export_table_name").select2("val","<%= @export.table_name %>")
|
237
|
-
<%
|
236
|
+
<% if @export.export_to == 'vertica' %>
|
238
237
|
$("#export_destination_schema").select2("destroy").before('<input type="text" id="export_destination_schema" name="export[destination_schema]" value="<%= @export.destination_schema %>"/>').remove();
|
239
238
|
<% end %>
|
240
239
|
}
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Myreplicator
|
2
|
+
##
|
3
|
+
# Configuration class for avoiding load the yml file everytime and cleaner config settings
|
4
|
+
##
|
5
|
+
class Configuration
|
6
|
+
class << self; attr_accessor :tmp_path, :mysqldump, :mysql end
|
7
|
+
|
8
|
+
# yml = YAML.load(File.read("#{Myreplicator.app_root}/config/myreplicator.yml"))
|
9
|
+
# Kernel.p yml["myreplicator"]
|
10
|
+
|
11
|
+
# @@tmp_path = yml["myreplicator"]["tmp_path"]
|
12
|
+
# @@mysql = yml["myreplicator"]["mysql"]
|
13
|
+
# @@mysqldump = yml["myreplicator"]["mysqldump"]
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.config(&block)
|
17
|
+
@@config ||= Myreplicator::Configuration.new
|
18
|
+
|
19
|
+
yield @@config if block
|
20
|
+
|
21
|
+
return @@config
|
22
|
+
end
|
23
|
+
end
|
data/lib/exporter.rb~
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require ""
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module VerticaUtils
|
2
|
+
class SourceDb < ActiveRecord::Base
|
3
|
+
|
4
|
+
def self.connect db
|
5
|
+
establish_connection(ActiveRecord::Base.configurations[db])
|
6
|
+
end
|
7
|
+
|
8
|
+
##
|
9
|
+
# Returns tables as an Array
|
10
|
+
# releases the connection
|
11
|
+
##
|
12
|
+
def self.get_tables(db)
|
13
|
+
tables = []
|
14
|
+
begin
|
15
|
+
self.connect(db)
|
16
|
+
tables = self.connection.tables
|
17
|
+
self.connection_pool.release_connection
|
18
|
+
rescue Mysql2::Error => e
|
19
|
+
puts "Connection to #{db} Failed!"
|
20
|
+
puts e.message
|
21
|
+
end
|
22
|
+
return tables
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.exec_sql source_db,sql
|
26
|
+
SourceDb.connect(source_db)
|
27
|
+
return SourceDb.connection.execute(sql)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,175 @@
|
|
1
|
+
require 'vertica_utils/types'
|
2
|
+
require 'vertica_utils/source_db'
|
3
|
+
require 'vertica_utils/vertica_sql'
|
4
|
+
|
5
|
+
module VerticaUtils
|
6
|
+
class VerticaLoader
|
7
|
+
class << self
|
8
|
+
def create_table *args
|
9
|
+
options = args.extract_options!
|
10
|
+
columns = []
|
11
|
+
table_definition(options).each(:as => :hash) do |row|
|
12
|
+
columns << row
|
13
|
+
end
|
14
|
+
options[:columns] = columns
|
15
|
+
|
16
|
+
sql = VerticaUtils::VerticaSql.create_table_stmt options
|
17
|
+
end
|
18
|
+
|
19
|
+
def create_all_tables db
|
20
|
+
tables = VerticaUtils::SourceDb.get_tables(db)
|
21
|
+
sqls = {}
|
22
|
+
tables.each do |table|
|
23
|
+
puts "Creating #{db}.#{table}"
|
24
|
+
sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
|
25
|
+
VerticaDb::Base.connection.execute sql
|
26
|
+
sql = VerticaUtils::VerticaLoader.create_table(:vertica_db => "bidw",
|
27
|
+
:vertica_table => table,
|
28
|
+
:vertica_schema => db,
|
29
|
+
:table => table,
|
30
|
+
:db => db)
|
31
|
+
sqls["#{table}"] = sql
|
32
|
+
VerticaDb::Base.connection.execute sql
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def prepare_options *args
|
37
|
+
options = args.extract_options!
|
38
|
+
options.reverse_merge!(
|
39
|
+
:host => "sfo-load-dw-01",
|
40
|
+
:user => "vertica",
|
41
|
+
:pass => "test",
|
42
|
+
:db => "bidw",
|
43
|
+
:schema => "test",
|
44
|
+
:table => "",
|
45
|
+
:file => "",
|
46
|
+
:delimiter => "\t",
|
47
|
+
:null_value => "NULL",
|
48
|
+
:enclosed => ""
|
49
|
+
)
|
50
|
+
return options
|
51
|
+
end
|
52
|
+
|
53
|
+
# VerticaUtils::VerticaLoader.load_to_vertica({:schema => "king", :table => "category_overview_data", :file => "tmp/vertica/category_overview_data.tsv", :null_value => "NULL"})
|
54
|
+
def load_to_vertica *args
|
55
|
+
list_of_nulls = ["0000-00-00"]
|
56
|
+
options = args.extract_options!
|
57
|
+
prepared_options = prepare_options options
|
58
|
+
Kernel.p prepared_options[:file]
|
59
|
+
if prepared_options[:file].blank?
|
60
|
+
raise "No input file"
|
61
|
+
end
|
62
|
+
|
63
|
+
begin
|
64
|
+
process_file(:file => prepared_options[:file], :list_of_nulls => list_of_nulls, :null_value => prepared_options[:null_value])
|
65
|
+
cmd = get_vsql_command(prepared_options)
|
66
|
+
Kernel.p cmd
|
67
|
+
system(cmd)
|
68
|
+
rescue Exception => e
|
69
|
+
raise e.message
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def get_vsql_command prepared_options
|
74
|
+
file_extension = prepared_options[:file].split('.').last
|
75
|
+
file_handler = ""
|
76
|
+
file_handler = "GZIP" if file_extension == "gz"
|
77
|
+
sql = "COPY #{prepared_options[:schema]}.#{prepared_options[:table]} FROM LOCAL \'#{prepared_options[:file]}\' #{file_handler} DELIMITER E\'#{prepared_options[:delimiter]}\' NULL as \'#{prepared_options[:null_value]}\' ENCLOSED BY \'#{prepared_options[:enclosed]}\' EXCEPTIONS 'tmp/vertica/load_exceptions.log';"
|
78
|
+
cmd = "/opt/vertica/bin/vsql -h #{prepared_options[:host]} -U #{prepared_options[:user]} -w #{prepared_options[:pass]} -d #{prepared_options[:db]} -c \"#{sql}\""
|
79
|
+
return cmd
|
80
|
+
end
|
81
|
+
|
82
|
+
def process_file *args
|
83
|
+
### replace the null values in the input file
|
84
|
+
options = args.extract_options!
|
85
|
+
options[:file].blank? ? return : file = options[:file]
|
86
|
+
options[:list_of_nulls].blank? ? list_of_nulls = [] : list_of_nulls = options[:list_of_nulls]
|
87
|
+
options[:null_value].blank? ? null_value = "NULL" : null_value = options[:null_value]
|
88
|
+
|
89
|
+
file_extension = file.split('.').last
|
90
|
+
case file_extension
|
91
|
+
when "tsv", "csv"
|
92
|
+
process_flat_file(file, list_of_nulls, null_value)
|
93
|
+
when "gz"
|
94
|
+
process_gzip_file(file, list_of_nulls, null_value)
|
95
|
+
else
|
96
|
+
raise "Un supported file extension"
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def replace_null(file, list_of_nulls, null_value = "NULL")
|
101
|
+
list_of_nulls.each do | value|
|
102
|
+
# special case for NULL MySQL datetime/date type but the column is defined NOT NULL
|
103
|
+
if value == '0000-00-00'
|
104
|
+
cmd1 = "sed -i 's/#{value}/1900-01-01/g' #{file}"
|
105
|
+
Kernel.p cmd1
|
106
|
+
system(cmd1)
|
107
|
+
else
|
108
|
+
cmd1 = "sed -i 's/#{value}/#{null_value}/g' #{file}"
|
109
|
+
Kernel.p cmd1
|
110
|
+
system(cmd1)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def process_flat_file file, list_of_nulls, null_value
|
116
|
+
# sed
|
117
|
+
replace_null(file, list_of_nulls, null_value)
|
118
|
+
end
|
119
|
+
|
120
|
+
def process_gzip_file file, list_of_nulls, null_value
|
121
|
+
# unzip
|
122
|
+
cmd = "gunzip -f #{file} -c > tmp/temp.txt"
|
123
|
+
system(cmd)
|
124
|
+
# sed
|
125
|
+
replace_null(file, list_of_nulls, null_value)
|
126
|
+
# zip
|
127
|
+
cmd2 = "gzip tmp/temp.txt -c > #{file}"
|
128
|
+
system(cmd2)
|
129
|
+
end
|
130
|
+
|
131
|
+
def get_table_columns * args
|
132
|
+
options = args.extract_options!
|
133
|
+
if options[:host].blank? or options[:user].blank? or options[:pass].blank? or options[:db].blank? or options[:schema].blank? or options[:table].blank?
|
134
|
+
raise "Unspecified host, user, pass, db, schema or table"
|
135
|
+
end
|
136
|
+
result = {}
|
137
|
+
count = 0
|
138
|
+
begin
|
139
|
+
cmd = "/opt/vertica/bin/vsql -h #{options[:host]} -U #{options[:user]} -w #{options[:pass]} -d #{options[:db]} -c '\\d #{options[:schema]}.#{options[:table]}';"
|
140
|
+
puts cmd
|
141
|
+
output = `#{cmd}`
|
142
|
+
lines = output.split("\n")
|
143
|
+
data = lines[3..lines.size-2]
|
144
|
+
data.each do |item|
|
145
|
+
count += 1
|
146
|
+
column_name = item.split("|")[2].lstrip!.rstrip!
|
147
|
+
column_type = item.split("|")[3].lstrip!.rstrip!
|
148
|
+
puts column_name + " " + column_type
|
149
|
+
result[count] = {column_name => column_type}
|
150
|
+
end
|
151
|
+
rescue Exception => e
|
152
|
+
raise e.message
|
153
|
+
end
|
154
|
+
return result
|
155
|
+
end
|
156
|
+
|
157
|
+
def ssh_connection options
|
158
|
+
ssh = Net::SSH.start(options[:ssh_host], options[:ssh_user], :password => options[:ssh_password])
|
159
|
+
end
|
160
|
+
|
161
|
+
def table_definition options
|
162
|
+
sql = "SELECT table_schema, table_name, column_name, is_nullable, data_type, column_type, column_key "
|
163
|
+
sql += "FROM INFORMATION_SCHEMA.COLUMNS where table_name = '#{options[:table]}' "
|
164
|
+
sql += "and table_schema = '#{options[:db]}';"
|
165
|
+
|
166
|
+
puts sql
|
167
|
+
|
168
|
+
desc = SourceDb.exec_sql(options[:db], sql)
|
169
|
+
|
170
|
+
return desc
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module VerticaUtils
|
2
|
+
class VerticaSql
|
3
|
+
|
4
|
+
def self.create_table_stmt options
|
5
|
+
|
6
|
+
sql = "CREATE TABLE IF NOT EXISTS #{options[:vertica_db]}."
|
7
|
+
sql += "#{options[:vertica_schema]}." if options[:vertica_schema]
|
8
|
+
sql += "#{options[:table]} ("
|
9
|
+
|
10
|
+
index = 1
|
11
|
+
primary_set = false
|
12
|
+
|
13
|
+
options[:columns].each do |column|
|
14
|
+
sql += "\"#{column['column_name']}\" "
|
15
|
+
|
16
|
+
sql += data_type(column['data_type'], column['column_type'])
|
17
|
+
sql += " "
|
18
|
+
|
19
|
+
if column['column_key'] == "PRI"
|
20
|
+
sql += key(column['column_key']) + " " unless primary_set # set only one primary key
|
21
|
+
primary_set = true
|
22
|
+
end
|
23
|
+
|
24
|
+
sql += nullable(column['is_nullable'])
|
25
|
+
sql += " "
|
26
|
+
|
27
|
+
if index < options[:columns].size
|
28
|
+
sql += ", "
|
29
|
+
else
|
30
|
+
sql += ");"
|
31
|
+
end
|
32
|
+
index += 1
|
33
|
+
end
|
34
|
+
|
35
|
+
puts sql
|
36
|
+
|
37
|
+
return sql
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.nullable is_nullable
|
41
|
+
if is_nullable == "YES"
|
42
|
+
return "NULL"
|
43
|
+
elsif is_nullable == "NO"
|
44
|
+
return "NOT NULL"
|
45
|
+
end
|
46
|
+
return ""
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.data_type type, col_type
|
50
|
+
type = VerticaTypes.convert type, col_type
|
51
|
+
result = " #{type} "
|
52
|
+
return result
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.key col_key
|
56
|
+
col_key = VerticaTypes.convert_key col_key
|
57
|
+
return "#{col_key} "
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
data/lib/myreplicator/version.rb
CHANGED
@@ -1,20 +1,13 @@
|
|
1
1
|
# SQLite version 3.x
|
2
|
-
#
|
2
|
+
# gem install sqlite3
|
3
3
|
#
|
4
|
-
#
|
5
|
-
#
|
6
|
-
vertica:
|
7
|
-
adapter: vertica
|
8
|
-
database: test
|
9
|
-
username: jferris
|
10
|
-
password: okl
|
11
|
-
host: ec2-23-22-175-247.compute-1.amazonaws.com
|
12
|
-
vsql: /Users/lnguyen/opt/vertica/bin/vsql
|
4
|
+
# Ensure the SQLite 3 gem is defined in your Gemfile
|
5
|
+
# gem 'sqlite3'
|
13
6
|
development:
|
14
7
|
adapter: mysql2
|
15
8
|
database: myreplicator
|
16
|
-
username:
|
17
|
-
password:
|
9
|
+
username: sas
|
10
|
+
password: sasan
|
18
11
|
host: 127.0.0.1
|
19
12
|
|
20
13
|
okl_dev:
|
@@ -23,6 +16,7 @@ okl_dev:
|
|
23
16
|
username: sas
|
24
17
|
password: sasan
|
25
18
|
host: 127.0.0.1
|
19
|
+
myreplicator: true
|
26
20
|
|
27
21
|
okl_test:
|
28
22
|
adapter: mysql2
|
@@ -30,6 +24,7 @@ okl_test:
|
|
30
24
|
username: xbox
|
31
25
|
password: okl
|
32
26
|
host: 54.243.176.8
|
27
|
+
myreplicator: true
|
33
28
|
|
34
29
|
myreplicator:
|
35
30
|
adapter: mysql2
|
@@ -38,6 +33,13 @@ myreplicator:
|
|
38
33
|
password: sasan
|
39
34
|
host: 127.0.0.1
|
40
35
|
|
36
|
+
vertica:
|
37
|
+
adapter: vertica
|
38
|
+
host: ec2-23-22-175-247.compute-1.amazonaws.com
|
39
|
+
username: jferris
|
40
|
+
password: okl
|
41
|
+
database: test
|
42
|
+
|
41
43
|
# Warning: The database defined as "test" will be erased and
|
42
44
|
# re-generated from your development database when you run "rake".
|
43
45
|
# Do not set this db to the same as development or production.
|
@@ -51,4 +53,4 @@ production:
|
|
51
53
|
adapter: sqlite3
|
52
54
|
database: db/production.sqlite3
|
53
55
|
pool: 5
|
54
|
-
timeout: 5000
|
56
|
+
timeout: 5000
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# SQLite version 3.x
|
2
|
+
# gem install sqlite3
|
3
|
+
#
|
4
|
+
# Ensure the SQLite 3 gem is defined in your Gemfile
|
5
|
+
# gem 'sqlite3'
|
6
|
+
development:
|
7
|
+
adapter: mysql2
|
8
|
+
database: myreplicator
|
9
|
+
username: sas
|
10
|
+
password: sasan
|
11
|
+
host: 127.0.0.1
|
12
|
+
|
13
|
+
okl_dev:
|
14
|
+
adapter: mysql2
|
15
|
+
database: okl_dev
|
16
|
+
username: sas
|
17
|
+
password: sasan
|
18
|
+
host: 127.0.0.1
|
19
|
+
|
20
|
+
okl_test:
|
21
|
+
adapter: mysql2
|
22
|
+
database: okl_test
|
23
|
+
username: xbox
|
24
|
+
password: okl
|
25
|
+
host: 54.243.176.8
|
26
|
+
|
27
|
+
myreplicator:
|
28
|
+
adapter: mysql2
|
29
|
+
database: myreplicator
|
30
|
+
username: sas
|
31
|
+
password: sasan
|
32
|
+
host: 127.0.0.1
|
33
|
+
|
34
|
+
vertica:
|
35
|
+
adapter: vertica
|
36
|
+
host: ec2-23-22-175-247.compute-1.amazonaws.com
|
37
|
+
username: jferris
|
38
|
+
password: okl
|
39
|
+
database: test
|
40
|
+
|
41
|
+
# Warning: The database defined as "test" will be erased and
|
42
|
+
# re-generated from your development database when you run "rake".
|
43
|
+
# Do not set this db to the same as development or production.
|
44
|
+
test:
|
45
|
+
adapter: sqlite3
|
46
|
+
database: db/test.sqlite3
|
47
|
+
pool: 5
|
48
|
+
timeout: 5000
|
49
|
+
|
50
|
+
production:
|
51
|
+
adapter: sqlite3
|
52
|
+
database: db/production.sqlite3
|
53
|
+
pool: 5
|
54
|
+
timeout: 5000
|
@@ -1,24 +1,28 @@
|
|
1
1
|
myreplicator:
|
2
2
|
tmp_path:
|
3
|
-
mysqldump:
|
4
|
-
mysqlimport:
|
3
|
+
mysqldump: mysqldump
|
4
|
+
mysqlimport: mysqlimport
|
5
5
|
mysql: mysql
|
6
6
|
outfile_location: /tmp/myreplicator
|
7
7
|
escape_by: '"'
|
8
8
|
terminate_by: '\t'
|
9
9
|
enclosed_by: '"'
|
10
10
|
lines_terminate_by: '\n'
|
11
|
-
login_redirect: /
|
12
11
|
|
13
|
-
okl_dev:
|
14
12
|
ssh_host: localhost
|
15
|
-
ssh_user:
|
16
|
-
ssh_password:
|
17
|
-
ssh_tmp_dir: /
|
13
|
+
ssh_user: guest
|
14
|
+
ssh_password: sasanguest
|
15
|
+
ssh_tmp_dir: /home/guest/tmp
|
18
16
|
|
17
|
+
okl_dev:
|
18
|
+
ssh_host: localhost
|
19
|
+
ssh_user: guest
|
20
|
+
ssh_password: sasanguest
|
21
|
+
ssh_tmp_dir: /home/guest/tmp
|
22
|
+
|
19
23
|
okl_test:
|
20
24
|
ssh_host: 54.243.176.8
|
21
25
|
ssh_user: ubuntu
|
22
26
|
ssh_db_host: 127.0.0.1
|
23
|
-
ssh_private_key:
|
24
|
-
ssh_tmp_dir: /
|
27
|
+
ssh_private_key: ~/.ssh/oklbi-team.pem
|
28
|
+
ssh_tmp_dir: /home/ubuntu/myreplicator_tmp
|
@@ -0,0 +1,23 @@
|
|
1
|
+
myreplicator:
|
2
|
+
tmp_path:
|
3
|
+
mysqldump: mysqldump
|
4
|
+
mysqlimport: mysqlimport
|
5
|
+
mysql: mysql
|
6
|
+
outfile_location: /tmp/myreplicator
|
7
|
+
escape_by: '"'
|
8
|
+
terminate_by: '\t'
|
9
|
+
enclosed_by: '"'
|
10
|
+
lines_terminate_by: '\n'
|
11
|
+
|
12
|
+
okl_dev:
|
13
|
+
ssh_host: localhost
|
14
|
+
ssh_user: guest
|
15
|
+
ssh_password: sasanguest
|
16
|
+
ssh_tmp_dir: /home/guest/tmp
|
17
|
+
|
18
|
+
okl_test:
|
19
|
+
ssh_host: 54.243.176.8
|
20
|
+
ssh_user: ubuntu
|
21
|
+
ssh_db_host: 127.0.0.1
|
22
|
+
ssh_private_key: ~/.ssh/oklbi-team.pem
|
23
|
+
ssh_tmp_dir: /home/ubuntu/myreplicator_tmp
|
Binary file
|
File without changes
|