myreplicator 1.0.6 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/app/assets/javascripts/myreplicator/select2.js +2685 -0
- data/app/assets/stylesheets/myreplicator/select2-spinner.gif +0 -0
- data/app/assets/stylesheets/myreplicator/select2.css +612 -0
- data/app/assets/stylesheets/myreplicator/select2.png +0 -0
- data/app/models/myreplicator/export.rb +25 -7
- data/app/models/myreplicator/vertica_export.rb +5 -0
- data/app/views/myreplicator/exports/_form.html.erb +48 -27
- data/app/views/myreplicator/exports/index.html.erb +4 -4
- data/db/migrate/20130213211927_create_myreplicator_vertica_exports.rb +22 -0
- data/db/migrate/20130213211927_create_myreplicator_vertica_exports.rb~ +8 -0
- data/lib/exporter/export_metadata.rb +7 -1
- data/lib/exporter/mysql_exporter.rb +103 -29
- data/lib/exporter/sql_commands.rb +8 -9
- data/lib/exporter.rb +1 -0
- data/lib/loader/loader.rb +32 -12
- data/lib/loader/vertica/source_db.rb +30 -0
- data/lib/loader/vertica/source_db.rb~ +30 -0
- data/lib/loader/vertica/types.rb +44 -0
- data/lib/loader/vertica/vertica_loader.rb +235 -0
- data/lib/loader/vertica/vertica_loader.rb~ +175 -0
- data/lib/loader/vertica/vertica_sql.rb +59 -0
- data/lib/loader/vertica/vertica_sql.rb~ +60 -0
- data/lib/loader/vertica.rb +1 -0
- data/lib/myreplicator/version.rb +1 -1
- data/lib/transporter/transporter.rb +11 -1
- data/test/dummy/app/models/vertica_db.rb +7 -0
- data/test/dummy/config/database.yml +7 -0
- data/test/dummy/config/database.yml~ +2 -2
- data/test/dummy/config/myreplicator.yml +5 -0
- data/test/dummy/config/myreplicator.yml~ +0 -2
- data/test/dummy/log/development.log +1664 -9510
- data/test/dummy/tmp/cache/assets/CD5/B90/sprockets%2Fc999d13a6a21113981c0d820e8043bdf +0 -0
- data/test/dummy/tmp/cache/assets/CD7/030/sprockets%2F9ba4859590582b8b72a650b2b00b6cd2 +0 -0
- data/test/dummy/tmp/cache/assets/CE5/670/sprockets%2Fe9e4122f1706626a21da6f8457f088ce +0 -0
- data/test/dummy/tmp/cache/assets/D06/5D0/sprockets%2F91850a20c0ddfa3d8814ca91870fb715 +0 -0
- data/test/dummy/tmp/cache/assets/D14/3A0/sprockets%2Fe59a60053fada52e8185281b4ee887a5 +0 -0
- data/test/dummy/tmp/cache/assets/D7C/E30/sprockets%2F0ba91e21bddfc7e1de102b22183e1e11 +0 -0
- data/test/dummy/tmp/cache/assets/D8B/B60/sprockets%2Faa32227c440a378ccd21218eefeb80bf +0 -0
- data/test/dummy/tmp/cache/assets/DA7/E50/sprockets%2F47bf4f2b4afeac775e6d572a83343fb8 +0 -0
- data/test/dummy/tmp/cache/assets/DA8/910/sprockets%2Fab5775c4a837bd4d97ac394d473cda9b +0 -0
- data/test/dummy/tmp/cache/assets/DAA/060/sprockets%2Facc0d22b9d28123cc1c84d0db630d0ba +0 -0
- data/test/dummy/tmp/cache/assets/DF8/5D0/sprockets%2Fb815ed34d61cfed96222daa3bfd1d84d +0 -0
- data/test/dummy/tmp/cache/assets/E1C/AC0/sprockets%2Faff544a3a34eb7dab7d46b0cb2cd7b70 +0 -0
- data/test/dummy/tmp/cache/assets/E2E/1F0/sprockets%2Fa24e3d7bc5ae4d40adf6f1b8fe94e7c3 +0 -0
- data/test/fixtures/myreplicator/vertica_exports.yml +11 -0
- data/test/unit/myreplicator/vertica_export_test.rb +9 -0
- metadata +43 -13
- data/test/dummy/tmp/myreplicator/okl_test_batchy_batches_1358547945.tsv.gz +0 -0
- data/test/dummy/tmp/myreplicator/okl_test_batchy_batches_1358547945.tsv.json +0 -1
@@ -0,0 +1,235 @@
|
|
1
|
+
module Myreplicator
|
2
|
+
class VerticaLoader
|
3
|
+
class << self
|
4
|
+
|
5
|
+
def create_table *args
|
6
|
+
options = args.extract_options!
|
7
|
+
columns = []
|
8
|
+
options[:mysql_schema].each(:as => :hash) do |row|
|
9
|
+
columns << row
|
10
|
+
end
|
11
|
+
options[:columns] = columns
|
12
|
+
|
13
|
+
sql = Myreplicator::VerticaSql.create_table_stmt options
|
14
|
+
puts sql
|
15
|
+
VerticaDb::Base.connection.execute sql
|
16
|
+
end
|
17
|
+
|
18
|
+
def destination_table_vertica options
|
19
|
+
sql = "select column_name, data_type From columns where
|
20
|
+
table_name = '#{options[:table]}' AND table_schema = '#{options[:destination_schema]}'"
|
21
|
+
puts sql
|
22
|
+
result = DB.exec_sql("vertica",sql)
|
23
|
+
return result
|
24
|
+
end
|
25
|
+
|
26
|
+
##
|
27
|
+
# Schema Change Algorithm
|
28
|
+
# Create temp table to load all data
|
29
|
+
# Load data
|
30
|
+
# Drop table
|
31
|
+
# Rename table
|
32
|
+
##
|
33
|
+
# rasing a concern: using the same schema or the tmp schema for the tmp table? Vertica doesn't lock the schema
|
34
|
+
def apply_schema_change options, temp_table
|
35
|
+
Myreplicator::VerticaLoader.create_table({:mysql_schema => options[:mysql_schema],
|
36
|
+
:vertica_db => options[:vertica_db],
|
37
|
+
:vertica_schema => options[:vertica_schema],
|
38
|
+
:table => temp_table,
|
39
|
+
:mysql_table => options[:table]
|
40
|
+
})
|
41
|
+
end
|
42
|
+
|
43
|
+
def full_load options, temp_table
|
44
|
+
export_id = options[:export_id]
|
45
|
+
new_options = prepare_options options
|
46
|
+
begin
|
47
|
+
a = Myreplicator::Export.find(export_id)
|
48
|
+
a.max_incremental_value = "0"
|
49
|
+
a.save!
|
50
|
+
a.export
|
51
|
+
b = Myreplicator::Transporter
|
52
|
+
b.perform
|
53
|
+
file = File.join(Myreplicator.app_root,"tmp", "myreplicator", "#{a.filename}")
|
54
|
+
`gunzip #{file}.gz`
|
55
|
+
new_options[:file] = file
|
56
|
+
new_options[:table] = temp_table
|
57
|
+
new_options[:schema] = options[:vertica_schema]
|
58
|
+
|
59
|
+
vertica_copy new_options
|
60
|
+
|
61
|
+
#drop
|
62
|
+
FileUtils.rm file
|
63
|
+
FileUtils.rm "#{file}.json"
|
64
|
+
sql = "DROP TABLE #{options[:vertica_db]}.#{options[:vertica_schema]}.#{options[:table]} CASCADE;"
|
65
|
+
VerticaDb::Base.connection.execute sql
|
66
|
+
#rename
|
67
|
+
sql = "ALTER TABLE #{options[:vertica_db]}.#{options[:vertica_schema]}.#{temp_table} RENAME TO #{options[:table]};"
|
68
|
+
VerticaDb::Base.connection.execute sql
|
69
|
+
rescue Exception => e
|
70
|
+
raise e.message
|
71
|
+
ensure
|
72
|
+
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
# def create_all_tables db
|
77
|
+
# tables = Loader::SourceDb.get_tables(db)
|
78
|
+
# sqls = {}
|
79
|
+
# tables.each do |table|
|
80
|
+
# puts "Creating #{db}.#{table}"
|
81
|
+
# sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
|
82
|
+
# VerticaDb::Base.connection.execute sql
|
83
|
+
# sql = Loader::VerticaLoader.create_table(:vertica_db => "bidw",
|
84
|
+
# :vertica_table => table,
|
85
|
+
# :vertica_schema => db,
|
86
|
+
# :table => table,
|
87
|
+
# :db => db)
|
88
|
+
# sqls["#{table}"] = sql
|
89
|
+
# VerticaDb::Base.connection.execute sql
|
90
|
+
# end
|
91
|
+
# end
|
92
|
+
|
93
|
+
def prepare_options *args
|
94
|
+
options = args.extract_options!
|
95
|
+
vertica_options =ActiveRecord::Base.configurations["vertica"]
|
96
|
+
|
97
|
+
result = options.clone
|
98
|
+
result.reverse_merge!(
|
99
|
+
:host => vertica_options["host"],
|
100
|
+
:user => vertica_options["username"],
|
101
|
+
:pass => vertica_options["password"],
|
102
|
+
:db => vertica_options["database"],
|
103
|
+
:schema => options[:destination_schema],
|
104
|
+
:table => options[:table_name],
|
105
|
+
:file => options[:filepath],
|
106
|
+
:delimiter => "\t",
|
107
|
+
:null_value => "NULL",
|
108
|
+
:enclosed => ""
|
109
|
+
)
|
110
|
+
if !vertica_options["vsql"].blank?
|
111
|
+
result.reverse_merge!(
|
112
|
+
:vsql => vertica_options["vsql"]
|
113
|
+
)
|
114
|
+
else
|
115
|
+
result.reverse_merge!(
|
116
|
+
:vsql => "/opt/vertica/bin/vsql"
|
117
|
+
)
|
118
|
+
end
|
119
|
+
|
120
|
+
return result
|
121
|
+
end
|
122
|
+
|
123
|
+
# Loader::VerticaLoader.load({:schema => "king", :table => "category_overview_data", :file => "tmp/vertica/category_overview_data.tsv", :null_value => "NULL"})
|
124
|
+
def load *args
|
125
|
+
options = args.extract_options!
|
126
|
+
#options = {:table => "app_csvs", :db => "public", :source_schema => "okl_dev"}
|
127
|
+
schema_check = Myreplicator::MysqlExporter.schema_changed?(:table => options[:table_name],
|
128
|
+
:destination_schema => options[:destination_schema],
|
129
|
+
:source_schema => options[:source_schema])
|
130
|
+
|
131
|
+
#create a temp table
|
132
|
+
temp_table = "temp_" + options[:table_name] + DateTime.now.strftime('%Y%m%d_%H%M%S').to_s
|
133
|
+
ops = {:mysql_schema => schema_check[:mysql_schema],
|
134
|
+
:vertica_db => options[:db],
|
135
|
+
:vertica_schema => options[:destination_schema],
|
136
|
+
:table => options[:table_name],
|
137
|
+
:export_id => options[:export_id]
|
138
|
+
}
|
139
|
+
begin
|
140
|
+
if schema_check[:new]
|
141
|
+
create_table(ops)
|
142
|
+
apply_schema_change(ops, temp_table)
|
143
|
+
#vertica_copy options
|
144
|
+
full_load(ops, temp_table)
|
145
|
+
elsif schema_check[:changed]
|
146
|
+
apply_schema_change(ops, temp_table)
|
147
|
+
full_load(ops, temp_table)
|
148
|
+
else
|
149
|
+
vertica_copy options
|
150
|
+
end
|
151
|
+
rescue Exception => e
|
152
|
+
raise e.message
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def vertica_copy * args
|
157
|
+
options = args.extract_options!
|
158
|
+
list_of_nulls = ["0000-00-00"]
|
159
|
+
prepared_options = prepare_options options
|
160
|
+
if prepared_options[:file].blank?
|
161
|
+
raise "No input file"
|
162
|
+
end
|
163
|
+
|
164
|
+
begin
|
165
|
+
process_file(:file => prepared_options[:file], :list_of_nulls => list_of_nulls, :null_value => prepared_options[:null_value])
|
166
|
+
cmd = get_vsql_command(prepared_options)
|
167
|
+
Kernel.p cmd
|
168
|
+
system(cmd)
|
169
|
+
rescue Exception => e
|
170
|
+
raise e.message
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
def get_vsql_command prepared_options
|
175
|
+
file_extension = prepared_options[:file].split('.').last
|
176
|
+
file_handler = ""
|
177
|
+
file_handler = "GZIP" if file_extension == "gz"
|
178
|
+
sql = "COPY #{prepared_options[:schema]}.#{prepared_options[:table]} FROM LOCAL \'#{prepared_options[:file]}\' #{file_handler} DELIMITER E\'#{prepared_options[:delimiter]}\' NULL as \'#{prepared_options[:null_value]}\' ENCLOSED BY \'#{prepared_options[:enclosed]}\' EXCEPTIONS 'load_exceptions.log';"
|
179
|
+
cmd = "#{prepared_options[:vsql]} -h #{prepared_options[:host]} -U #{prepared_options[:user]} -w #{prepared_options[:pass]} -d #{prepared_options[:db]} -c \"#{sql}\""
|
180
|
+
return cmd
|
181
|
+
end
|
182
|
+
|
183
|
+
def process_file *args
|
184
|
+
### replace the null values in the input file
|
185
|
+
options = args.extract_options!
|
186
|
+
options[:file].blank? ? return : file = options[:file]
|
187
|
+
options[:list_of_nulls].blank? ? list_of_nulls = [] : list_of_nulls = options[:list_of_nulls]
|
188
|
+
options[:null_value].blank? ? null_value = "NULL" : null_value = options[:null_value]
|
189
|
+
|
190
|
+
file_extension = file.split('.').last
|
191
|
+
case file_extension
|
192
|
+
when "tsv", "csv"
|
193
|
+
process_flat_file(file, list_of_nulls, null_value)
|
194
|
+
when "gz"
|
195
|
+
process_gzip_file(file, list_of_nulls, null_value)
|
196
|
+
else
|
197
|
+
raise "Un supported file extension"
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
def replace_null(file, list_of_nulls, null_value = "NULL")
|
202
|
+
list_of_nulls.each do | value|
|
203
|
+
# special case for NULL MySQL datetime/date type but the column is defined NOT NULL
|
204
|
+
if value == '0000-00-00'
|
205
|
+
cmd1 = "sed -i 's/#{value}/1900-01-01/g' #{file}"
|
206
|
+
Kernel.p cmd1
|
207
|
+
system(cmd1)
|
208
|
+
else
|
209
|
+
cmd1 = "sed -i 's/#{value}/#{null_value}/g' #{file}"
|
210
|
+
Kernel.p cmd1
|
211
|
+
system(cmd1)
|
212
|
+
end
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
def process_flat_file file, list_of_nulls, null_value
|
217
|
+
# sed
|
218
|
+
replace_null(file, list_of_nulls, null_value)
|
219
|
+
end
|
220
|
+
|
221
|
+
def process_gzip_file file, list_of_nulls, null_value
|
222
|
+
# unzip
|
223
|
+
cmd = "gunzip -f #{file} -c > tmp/temp.txt"
|
224
|
+
system(cmd)
|
225
|
+
# sed
|
226
|
+
replace_null(file, list_of_nulls, null_value)
|
227
|
+
# zip
|
228
|
+
cmd2 = "gzip tmp/temp.txt -c > #{file}"
|
229
|
+
system(cmd2)
|
230
|
+
end
|
231
|
+
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
@@ -0,0 +1,175 @@
|
|
1
|
+
require 'vertica_utils/types'
|
2
|
+
require 'vertica_utils/source_db'
|
3
|
+
require 'vertica_utils/vertica_sql'
|
4
|
+
|
5
|
+
module VerticaUtils
|
6
|
+
class VerticaLoader
|
7
|
+
class << self
|
8
|
+
def create_table *args
|
9
|
+
options = args.extract_options!
|
10
|
+
columns = []
|
11
|
+
table_definition(options).each(:as => :hash) do |row|
|
12
|
+
columns << row
|
13
|
+
end
|
14
|
+
options[:columns] = columns
|
15
|
+
|
16
|
+
sql = VerticaUtils::VerticaSql.create_table_stmt options
|
17
|
+
end
|
18
|
+
|
19
|
+
def create_all_tables db
|
20
|
+
tables = VerticaUtils::SourceDb.get_tables(db)
|
21
|
+
sqls = {}
|
22
|
+
tables.each do |table|
|
23
|
+
puts "Creating #{db}.#{table}"
|
24
|
+
sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
|
25
|
+
VerticaDb::Base.connection.execute sql
|
26
|
+
sql = VerticaUtils::VerticaLoader.create_table(:vertica_db => "bidw",
|
27
|
+
:vertica_table => table,
|
28
|
+
:vertica_schema => db,
|
29
|
+
:table => table,
|
30
|
+
:db => db)
|
31
|
+
sqls["#{table}"] = sql
|
32
|
+
VerticaDb::Base.connection.execute sql
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def prepare_options *args
|
37
|
+
options = args.extract_options!
|
38
|
+
options.reverse_merge!(
|
39
|
+
:host => "sfo-load-dw-01",
|
40
|
+
:user => "vertica",
|
41
|
+
:pass => "test",
|
42
|
+
:db => "bidw",
|
43
|
+
:schema => "test",
|
44
|
+
:table => "",
|
45
|
+
:file => "",
|
46
|
+
:delimiter => "\t",
|
47
|
+
:null_value => "NULL",
|
48
|
+
:enclosed => ""
|
49
|
+
)
|
50
|
+
return options
|
51
|
+
end
|
52
|
+
|
53
|
+
# VerticaUtils::VerticaLoader.load_to_vertica({:schema => "king", :table => "category_overview_data", :file => "tmp/vertica/category_overview_data.tsv", :null_value => "NULL"})
|
54
|
+
def load_to_vertica *args
|
55
|
+
list_of_nulls = ["0000-00-00"]
|
56
|
+
options = args.extract_options!
|
57
|
+
prepared_options = prepare_options options
|
58
|
+
Kernel.p prepared_options[:file]
|
59
|
+
if prepared_options[:file].blank?
|
60
|
+
raise "No input file"
|
61
|
+
end
|
62
|
+
|
63
|
+
begin
|
64
|
+
process_file(:file => prepared_options[:file], :list_of_nulls => list_of_nulls, :null_value => prepared_options[:null_value])
|
65
|
+
cmd = get_vsql_command(prepared_options)
|
66
|
+
Kernel.p cmd
|
67
|
+
system(cmd)
|
68
|
+
rescue Exception => e
|
69
|
+
raise e.message
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def get_vsql_command prepared_options
|
74
|
+
file_extension = prepared_options[:file].split('.').last
|
75
|
+
file_handler = ""
|
76
|
+
file_handler = "GZIP" if file_extension == "gz"
|
77
|
+
sql = "COPY #{prepared_options[:schema]}.#{prepared_options[:table]} FROM LOCAL \'#{prepared_options[:file]}\' #{file_handler} DELIMITER E\'#{prepared_options[:delimiter]}\' NULL as \'#{prepared_options[:null_value]}\' ENCLOSED BY \'#{prepared_options[:enclosed]}\' EXCEPTIONS 'tmp/vertica/load_exceptions.log';"
|
78
|
+
cmd = "/opt/vertica/bin/vsql -h #{prepared_options[:host]} -U #{prepared_options[:user]} -w #{prepared_options[:pass]} -d #{prepared_options[:db]} -c \"#{sql}\""
|
79
|
+
return cmd
|
80
|
+
end
|
81
|
+
|
82
|
+
def process_file *args
|
83
|
+
### replace the null values in the input file
|
84
|
+
options = args.extract_options!
|
85
|
+
options[:file].blank? ? return : file = options[:file]
|
86
|
+
options[:list_of_nulls].blank? ? list_of_nulls = [] : list_of_nulls = options[:list_of_nulls]
|
87
|
+
options[:null_value].blank? ? null_value = "NULL" : null_value = options[:null_value]
|
88
|
+
|
89
|
+
file_extension = file.split('.').last
|
90
|
+
case file_extension
|
91
|
+
when "tsv", "csv"
|
92
|
+
process_flat_file(file, list_of_nulls, null_value)
|
93
|
+
when "gz"
|
94
|
+
process_gzip_file(file, list_of_nulls, null_value)
|
95
|
+
else
|
96
|
+
raise "Un supported file extension"
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def replace_null(file, list_of_nulls, null_value = "NULL")
|
101
|
+
list_of_nulls.each do | value|
|
102
|
+
# special case for NULL MySQL datetime/date type but the column is defined NOT NULL
|
103
|
+
if value == '0000-00-00'
|
104
|
+
cmd1 = "sed -i 's/#{value}/1900-01-01/g' #{file}"
|
105
|
+
Kernel.p cmd1
|
106
|
+
system(cmd1)
|
107
|
+
else
|
108
|
+
cmd1 = "sed -i 's/#{value}/#{null_value}/g' #{file}"
|
109
|
+
Kernel.p cmd1
|
110
|
+
system(cmd1)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def process_flat_file file, list_of_nulls, null_value
|
116
|
+
# sed
|
117
|
+
replace_null(file, list_of_nulls, null_value)
|
118
|
+
end
|
119
|
+
|
120
|
+
def process_gzip_file file, list_of_nulls, null_value
|
121
|
+
# unzip
|
122
|
+
cmd = "gunzip -f #{file} -c > tmp/temp.txt"
|
123
|
+
system(cmd)
|
124
|
+
# sed
|
125
|
+
replace_null(file, list_of_nulls, null_value)
|
126
|
+
# zip
|
127
|
+
cmd2 = "gzip tmp/temp.txt -c > #{file}"
|
128
|
+
system(cmd2)
|
129
|
+
end
|
130
|
+
|
131
|
+
def get_table_columns * args
|
132
|
+
options = args.extract_options!
|
133
|
+
if options[:host].blank? or options[:user].blank? or options[:pass].blank? or options[:db].blank? or options[:schema].blank? or options[:table].blank?
|
134
|
+
raise "Unspecified host, user, pass, db, schema or table"
|
135
|
+
end
|
136
|
+
result = {}
|
137
|
+
count = 0
|
138
|
+
begin
|
139
|
+
cmd = "/opt/vertica/bin/vsql -h #{options[:host]} -U #{options[:user]} -w #{options[:pass]} -d #{options[:db]} -c '\\d #{options[:schema]}.#{options[:table]}';"
|
140
|
+
puts cmd
|
141
|
+
output = `#{cmd}`
|
142
|
+
lines = output.split("\n")
|
143
|
+
data = lines[3..lines.size-2]
|
144
|
+
data.each do |item|
|
145
|
+
count += 1
|
146
|
+
column_name = item.split("|")[2].lstrip!.rstrip!
|
147
|
+
column_type = item.split("|")[3].lstrip!.rstrip!
|
148
|
+
puts column_name + " " + column_type
|
149
|
+
result[count] = {column_name => column_type}
|
150
|
+
end
|
151
|
+
rescue Exception => e
|
152
|
+
raise e.message
|
153
|
+
end
|
154
|
+
return result
|
155
|
+
end
|
156
|
+
|
157
|
+
def ssh_connection options
|
158
|
+
ssh = Net::SSH.start(options[:ssh_host], options[:ssh_user], :password => options[:ssh_password])
|
159
|
+
end
|
160
|
+
|
161
|
+
def table_definition options
|
162
|
+
sql = "SELECT table_schema, table_name, column_name, is_nullable, data_type, column_type, column_key "
|
163
|
+
sql += "FROM INFORMATION_SCHEMA.COLUMNS where table_name = '#{options[:table]}' "
|
164
|
+
sql += "and table_schema = '#{options[:db]}';"
|
165
|
+
|
166
|
+
puts sql
|
167
|
+
|
168
|
+
desc = SourceDb.exec_sql(options[:db], sql)
|
169
|
+
|
170
|
+
return desc
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module Myreplicator
|
2
|
+
class VerticaSql
|
3
|
+
|
4
|
+
def self.create_table_stmt options
|
5
|
+
sql = "CREATE TABLE IF NOT EXISTS #{options[:vertica_db]}."
|
6
|
+
sql += "#{options[:vertica_schema]}." if options[:vertica_schema]
|
7
|
+
sql += "#{options[:table]} ("
|
8
|
+
|
9
|
+
index = 1
|
10
|
+
primary_set = false
|
11
|
+
|
12
|
+
options[:columns].each do |column|
|
13
|
+
sql += "\"#{column['column_name']}\" "
|
14
|
+
|
15
|
+
sql += data_type(column['data_type'], column['column_type'])
|
16
|
+
sql += " "
|
17
|
+
|
18
|
+
if column['column_key'] == "PRI"
|
19
|
+
sql += key(column['column_key']) + " " unless primary_set # set only one primary key
|
20
|
+
primary_set = true
|
21
|
+
end
|
22
|
+
|
23
|
+
sql += nullable(column['is_nullable'])
|
24
|
+
sql += " "
|
25
|
+
|
26
|
+
if index < options[:columns].size
|
27
|
+
sql += ", "
|
28
|
+
else
|
29
|
+
sql += ");"
|
30
|
+
end
|
31
|
+
index += 1
|
32
|
+
end
|
33
|
+
|
34
|
+
puts sql
|
35
|
+
|
36
|
+
return sql
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.nullable is_nullable
|
40
|
+
if is_nullable == "YES"
|
41
|
+
return "NULL"
|
42
|
+
elsif is_nullable == "NO"
|
43
|
+
return "NOT NULL"
|
44
|
+
end
|
45
|
+
return ""
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.data_type type, col_type
|
49
|
+
type = VerticaTypes.convert type, col_type
|
50
|
+
result = " #{type} "
|
51
|
+
return result
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.key col_key
|
55
|
+
col_key = VerticaTypes.convert_key col_key
|
56
|
+
return "#{col_key} "
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module VerticaUtils
|
2
|
+
class VerticaSql
|
3
|
+
|
4
|
+
def self.create_table_stmt options
|
5
|
+
|
6
|
+
sql = "CREATE TABLE IF NOT EXISTS #{options[:vertica_db]}."
|
7
|
+
sql += "#{options[:vertica_schema]}." if options[:vertica_schema]
|
8
|
+
sql += "#{options[:table]} ("
|
9
|
+
|
10
|
+
index = 1
|
11
|
+
primary_set = false
|
12
|
+
|
13
|
+
options[:columns].each do |column|
|
14
|
+
sql += "\"#{column['column_name']}\" "
|
15
|
+
|
16
|
+
sql += data_type(column['data_type'], column['column_type'])
|
17
|
+
sql += " "
|
18
|
+
|
19
|
+
if column['column_key'] == "PRI"
|
20
|
+
sql += key(column['column_key']) + " " unless primary_set # set only one primary key
|
21
|
+
primary_set = true
|
22
|
+
end
|
23
|
+
|
24
|
+
sql += nullable(column['is_nullable'])
|
25
|
+
sql += " "
|
26
|
+
|
27
|
+
if index < options[:columns].size
|
28
|
+
sql += ", "
|
29
|
+
else
|
30
|
+
sql += ");"
|
31
|
+
end
|
32
|
+
index += 1
|
33
|
+
end
|
34
|
+
|
35
|
+
puts sql
|
36
|
+
|
37
|
+
return sql
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.nullable is_nullable
|
41
|
+
if is_nullable == "YES"
|
42
|
+
return "NULL"
|
43
|
+
elsif is_nullable == "NO"
|
44
|
+
return "NOT NULL"
|
45
|
+
end
|
46
|
+
return ""
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.data_type type, col_type
|
50
|
+
type = VerticaTypes.convert type, col_type
|
51
|
+
result = " #{type} "
|
52
|
+
return result
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.key col_key
|
56
|
+
col_key = VerticaTypes.convert_key col_key
|
57
|
+
return "#{col_key} "
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
Dir["#{File.expand_path(File.dirname(__FILE__))}/vertica/*.rb"].each { | f | require File.expand_path(f) }
|
data/lib/myreplicator/version.rb
CHANGED
@@ -100,8 +100,13 @@ module Myreplicator
|
|
100
100
|
Log.run(:job_type => "transporter", :name => "export_file",
|
101
101
|
:file => dump_file, :export_id => export.id) do |log|
|
102
102
|
puts "Downloading #{dump_file}"
|
103
|
-
|
103
|
+
local_dump_file = File.join(tmp_dir, dump_file.split("/").last)
|
104
|
+
sftp.download!(dump_file, local_dump_file)
|
104
105
|
Transporter.remove!(export, json_file, dump_file)
|
106
|
+
# store back up as well
|
107
|
+
unless metadata.store_in.blank?
|
108
|
+
Transporter.backup_files(metadata.backup_path, json_local_path, local_dump_file)
|
109
|
+
end
|
105
110
|
end
|
106
111
|
end #if
|
107
112
|
puts "#{Thread.current.to_s}___Exiting download..."
|
@@ -109,6 +114,11 @@ module Myreplicator
|
|
109
114
|
}
|
110
115
|
end
|
111
116
|
|
117
|
+
def self.backup_files location, metadata_path, dump_path
|
118
|
+
FileUtils.cp(metadata_path, location)
|
119
|
+
FileUtils.cp(dump_path, location)
|
120
|
+
end
|
121
|
+
|
112
122
|
##
|
113
123
|
# Returns true if the file should be deleted
|
114
124
|
##
|
@@ -31,6 +31,13 @@ myreplicator:
|
|
31
31
|
password: sasan
|
32
32
|
host: 127.0.0.1
|
33
33
|
|
34
|
+
vertica:
|
35
|
+
adapter: vertica
|
36
|
+
host: ec2-23-22-175-247.compute-1.amazonaws.com
|
37
|
+
username: jferris
|
38
|
+
password: okl
|
39
|
+
database: test
|
40
|
+
|
34
41
|
# Warning: The database defined as "test" will be erased and
|
35
42
|
# re-generated from your development database when you run "rake".
|
36
43
|
# Do not set this db to the same as development or production.
|