myreplicator 1.0.6 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/app/assets/javascripts/myreplicator/select2.js +2685 -0
  2. data/app/assets/stylesheets/myreplicator/select2-spinner.gif +0 -0
  3. data/app/assets/stylesheets/myreplicator/select2.css +612 -0
  4. data/app/assets/stylesheets/myreplicator/select2.png +0 -0
  5. data/app/models/myreplicator/export.rb +25 -7
  6. data/app/models/myreplicator/vertica_export.rb +5 -0
  7. data/app/views/myreplicator/exports/_form.html.erb +48 -27
  8. data/app/views/myreplicator/exports/index.html.erb +4 -4
  9. data/db/migrate/20130213211927_create_myreplicator_vertica_exports.rb +22 -0
  10. data/db/migrate/20130213211927_create_myreplicator_vertica_exports.rb~ +8 -0
  11. data/lib/exporter/export_metadata.rb +7 -1
  12. data/lib/exporter/mysql_exporter.rb +103 -29
  13. data/lib/exporter/sql_commands.rb +8 -9
  14. data/lib/exporter.rb +1 -0
  15. data/lib/loader/loader.rb +32 -12
  16. data/lib/loader/vertica/source_db.rb +30 -0
  17. data/lib/loader/vertica/source_db.rb~ +30 -0
  18. data/lib/loader/vertica/types.rb +44 -0
  19. data/lib/loader/vertica/vertica_loader.rb +235 -0
  20. data/lib/loader/vertica/vertica_loader.rb~ +175 -0
  21. data/lib/loader/vertica/vertica_sql.rb +59 -0
  22. data/lib/loader/vertica/vertica_sql.rb~ +60 -0
  23. data/lib/loader/vertica.rb +1 -0
  24. data/lib/myreplicator/version.rb +1 -1
  25. data/lib/transporter/transporter.rb +11 -1
  26. data/test/dummy/app/models/vertica_db.rb +7 -0
  27. data/test/dummy/config/database.yml +7 -0
  28. data/test/dummy/config/database.yml~ +2 -2
  29. data/test/dummy/config/myreplicator.yml +5 -0
  30. data/test/dummy/config/myreplicator.yml~ +0 -2
  31. data/test/dummy/log/development.log +1664 -9510
  32. data/test/dummy/tmp/cache/assets/CD5/B90/sprockets%2Fc999d13a6a21113981c0d820e8043bdf +0 -0
  33. data/test/dummy/tmp/cache/assets/CD7/030/sprockets%2F9ba4859590582b8b72a650b2b00b6cd2 +0 -0
  34. data/test/dummy/tmp/cache/assets/CE5/670/sprockets%2Fe9e4122f1706626a21da6f8457f088ce +0 -0
  35. data/test/dummy/tmp/cache/assets/D06/5D0/sprockets%2F91850a20c0ddfa3d8814ca91870fb715 +0 -0
  36. data/test/dummy/tmp/cache/assets/D14/3A0/sprockets%2Fe59a60053fada52e8185281b4ee887a5 +0 -0
  37. data/test/dummy/tmp/cache/assets/D7C/E30/sprockets%2F0ba91e21bddfc7e1de102b22183e1e11 +0 -0
  38. data/test/dummy/tmp/cache/assets/D8B/B60/sprockets%2Faa32227c440a378ccd21218eefeb80bf +0 -0
  39. data/test/dummy/tmp/cache/assets/DA7/E50/sprockets%2F47bf4f2b4afeac775e6d572a83343fb8 +0 -0
  40. data/test/dummy/tmp/cache/assets/DA8/910/sprockets%2Fab5775c4a837bd4d97ac394d473cda9b +0 -0
  41. data/test/dummy/tmp/cache/assets/DAA/060/sprockets%2Facc0d22b9d28123cc1c84d0db630d0ba +0 -0
  42. data/test/dummy/tmp/cache/assets/DF8/5D0/sprockets%2Fb815ed34d61cfed96222daa3bfd1d84d +0 -0
  43. data/test/dummy/tmp/cache/assets/E1C/AC0/sprockets%2Faff544a3a34eb7dab7d46b0cb2cd7b70 +0 -0
  44. data/test/dummy/tmp/cache/assets/E2E/1F0/sprockets%2Fa24e3d7bc5ae4d40adf6f1b8fe94e7c3 +0 -0
  45. data/test/fixtures/myreplicator/vertica_exports.yml +11 -0
  46. data/test/unit/myreplicator/vertica_export_test.rb +9 -0
  47. metadata +43 -13
  48. data/test/dummy/tmp/myreplicator/okl_test_batchy_batches_1358547945.tsv.gz +0 -0
  49. data/test/dummy/tmp/myreplicator/okl_test_batchy_batches_1358547945.tsv.json +0 -1
@@ -0,0 +1,235 @@
1
+ module Myreplicator
2
+ class VerticaLoader
3
+ class << self
4
+
5
+ def create_table *args
6
+ options = args.extract_options!
7
+ columns = []
8
+ options[:mysql_schema].each(:as => :hash) do |row|
9
+ columns << row
10
+ end
11
+ options[:columns] = columns
12
+
13
+ sql = Myreplicator::VerticaSql.create_table_stmt options
14
+ puts sql
15
+ VerticaDb::Base.connection.execute sql
16
+ end
17
+
18
+ def destination_table_vertica options
19
+ sql = "select column_name, data_type From columns where
20
+ table_name = '#{options[:table]}' AND table_schema = '#{options[:destination_schema]}'"
21
+ puts sql
22
+ result = DB.exec_sql("vertica",sql)
23
+ return result
24
+ end
25
+
26
+ ##
27
+ # Schema Change Algorithm
28
+ # Create temp table to load all data
29
+ # Load data
30
+ # Drop table
31
+ # Rename table
32
+ ##
33
+ # rasing a concern: using the same schema or the tmp schema for the tmp table? Vertica doesn't lock the schema
34
+ def apply_schema_change options, temp_table
35
+ Myreplicator::VerticaLoader.create_table({:mysql_schema => options[:mysql_schema],
36
+ :vertica_db => options[:vertica_db],
37
+ :vertica_schema => options[:vertica_schema],
38
+ :table => temp_table,
39
+ :mysql_table => options[:table]
40
+ })
41
+ end
42
+
43
+ def full_load options, temp_table
44
+ export_id = options[:export_id]
45
+ new_options = prepare_options options
46
+ begin
47
+ a = Myreplicator::Export.find(export_id)
48
+ a.max_incremental_value = "0"
49
+ a.save!
50
+ a.export
51
+ b = Myreplicator::Transporter
52
+ b.perform
53
+ file = File.join(Myreplicator.app_root,"tmp", "myreplicator", "#{a.filename}")
54
+ `gunzip #{file}.gz`
55
+ new_options[:file] = file
56
+ new_options[:table] = temp_table
57
+ new_options[:schema] = options[:vertica_schema]
58
+
59
+ vertica_copy new_options
60
+
61
+ #drop
62
+ FileUtils.rm file
63
+ FileUtils.rm "#{file}.json"
64
+ sql = "DROP TABLE #{options[:vertica_db]}.#{options[:vertica_schema]}.#{options[:table]} CASCADE;"
65
+ VerticaDb::Base.connection.execute sql
66
+ #rename
67
+ sql = "ALTER TABLE #{options[:vertica_db]}.#{options[:vertica_schema]}.#{temp_table} RENAME TO #{options[:table]};"
68
+ VerticaDb::Base.connection.execute sql
69
+ rescue Exception => e
70
+ raise e.message
71
+ ensure
72
+
73
+ end
74
+ end
75
+
76
+ # def create_all_tables db
77
+ # tables = Loader::SourceDb.get_tables(db)
78
+ # sqls = {}
79
+ # tables.each do |table|
80
+ # puts "Creating #{db}.#{table}"
81
+ # sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
82
+ # VerticaDb::Base.connection.execute sql
83
+ # sql = Loader::VerticaLoader.create_table(:vertica_db => "bidw",
84
+ # :vertica_table => table,
85
+ # :vertica_schema => db,
86
+ # :table => table,
87
+ # :db => db)
88
+ # sqls["#{table}"] = sql
89
+ # VerticaDb::Base.connection.execute sql
90
+ # end
91
+ # end
92
+
93
+ def prepare_options *args
94
+ options = args.extract_options!
95
+ vertica_options =ActiveRecord::Base.configurations["vertica"]
96
+
97
+ result = options.clone
98
+ result.reverse_merge!(
99
+ :host => vertica_options["host"],
100
+ :user => vertica_options["username"],
101
+ :pass => vertica_options["password"],
102
+ :db => vertica_options["database"],
103
+ :schema => options[:destination_schema],
104
+ :table => options[:table_name],
105
+ :file => options[:filepath],
106
+ :delimiter => "\t",
107
+ :null_value => "NULL",
108
+ :enclosed => ""
109
+ )
110
+ if !vertica_options["vsql"].blank?
111
+ result.reverse_merge!(
112
+ :vsql => vertica_options["vsql"]
113
+ )
114
+ else
115
+ result.reverse_merge!(
116
+ :vsql => "/opt/vertica/bin/vsql"
117
+ )
118
+ end
119
+
120
+ return result
121
+ end
122
+
123
+ # Loader::VerticaLoader.load({:schema => "king", :table => "category_overview_data", :file => "tmp/vertica/category_overview_data.tsv", :null_value => "NULL"})
124
+ def load *args
125
+ options = args.extract_options!
126
+ #options = {:table => "app_csvs", :db => "public", :source_schema => "okl_dev"}
127
+ schema_check = Myreplicator::MysqlExporter.schema_changed?(:table => options[:table_name],
128
+ :destination_schema => options[:destination_schema],
129
+ :source_schema => options[:source_schema])
130
+
131
+ #create a temp table
132
+ temp_table = "temp_" + options[:table_name] + DateTime.now.strftime('%Y%m%d_%H%M%S').to_s
133
+ ops = {:mysql_schema => schema_check[:mysql_schema],
134
+ :vertica_db => options[:db],
135
+ :vertica_schema => options[:destination_schema],
136
+ :table => options[:table_name],
137
+ :export_id => options[:export_id]
138
+ }
139
+ begin
140
+ if schema_check[:new]
141
+ create_table(ops)
142
+ apply_schema_change(ops, temp_table)
143
+ #vertica_copy options
144
+ full_load(ops, temp_table)
145
+ elsif schema_check[:changed]
146
+ apply_schema_change(ops, temp_table)
147
+ full_load(ops, temp_table)
148
+ else
149
+ vertica_copy options
150
+ end
151
+ rescue Exception => e
152
+ raise e.message
153
+ end
154
+ end
155
+
156
+ def vertica_copy * args
157
+ options = args.extract_options!
158
+ list_of_nulls = ["0000-00-00"]
159
+ prepared_options = prepare_options options
160
+ if prepared_options[:file].blank?
161
+ raise "No input file"
162
+ end
163
+
164
+ begin
165
+ process_file(:file => prepared_options[:file], :list_of_nulls => list_of_nulls, :null_value => prepared_options[:null_value])
166
+ cmd = get_vsql_command(prepared_options)
167
+ Kernel.p cmd
168
+ system(cmd)
169
+ rescue Exception => e
170
+ raise e.message
171
+ end
172
+ end
173
+
174
+ def get_vsql_command prepared_options
175
+ file_extension = prepared_options[:file].split('.').last
176
+ file_handler = ""
177
+ file_handler = "GZIP" if file_extension == "gz"
178
+ sql = "COPY #{prepared_options[:schema]}.#{prepared_options[:table]} FROM LOCAL \'#{prepared_options[:file]}\' #{file_handler} DELIMITER E\'#{prepared_options[:delimiter]}\' NULL as \'#{prepared_options[:null_value]}\' ENCLOSED BY \'#{prepared_options[:enclosed]}\' EXCEPTIONS 'load_exceptions.log';"
179
+ cmd = "#{prepared_options[:vsql]} -h #{prepared_options[:host]} -U #{prepared_options[:user]} -w #{prepared_options[:pass]} -d #{prepared_options[:db]} -c \"#{sql}\""
180
+ return cmd
181
+ end
182
+
183
+ def process_file *args
184
+ ### replace the null values in the input file
185
+ options = args.extract_options!
186
+ options[:file].blank? ? return : file = options[:file]
187
+ options[:list_of_nulls].blank? ? list_of_nulls = [] : list_of_nulls = options[:list_of_nulls]
188
+ options[:null_value].blank? ? null_value = "NULL" : null_value = options[:null_value]
189
+
190
+ file_extension = file.split('.').last
191
+ case file_extension
192
+ when "tsv", "csv"
193
+ process_flat_file(file, list_of_nulls, null_value)
194
+ when "gz"
195
+ process_gzip_file(file, list_of_nulls, null_value)
196
+ else
197
+ raise "Un supported file extension"
198
+ end
199
+ end
200
+
201
+ def replace_null(file, list_of_nulls, null_value = "NULL")
202
+ list_of_nulls.each do | value|
203
+ # special case for NULL MySQL datetime/date type but the column is defined NOT NULL
204
+ if value == '0000-00-00'
205
+ cmd1 = "sed -i 's/#{value}/1900-01-01/g' #{file}"
206
+ Kernel.p cmd1
207
+ system(cmd1)
208
+ else
209
+ cmd1 = "sed -i 's/#{value}/#{null_value}/g' #{file}"
210
+ Kernel.p cmd1
211
+ system(cmd1)
212
+ end
213
+ end
214
+ end
215
+
216
+ def process_flat_file file, list_of_nulls, null_value
217
+ # sed
218
+ replace_null(file, list_of_nulls, null_value)
219
+ end
220
+
221
+ def process_gzip_file file, list_of_nulls, null_value
222
+ # unzip
223
+ cmd = "gunzip -f #{file} -c > tmp/temp.txt"
224
+ system(cmd)
225
+ # sed
226
+ replace_null(file, list_of_nulls, null_value)
227
+ # zip
228
+ cmd2 = "gzip tmp/temp.txt -c > #{file}"
229
+ system(cmd2)
230
+ end
231
+
232
+ end
233
+ end
234
+ end
235
+
@@ -0,0 +1,175 @@
1
+ require 'vertica_utils/types'
2
+ require 'vertica_utils/source_db'
3
+ require 'vertica_utils/vertica_sql'
4
+
5
+ module VerticaUtils
6
+ class VerticaLoader
7
+ class << self
8
+ def create_table *args
9
+ options = args.extract_options!
10
+ columns = []
11
+ table_definition(options).each(:as => :hash) do |row|
12
+ columns << row
13
+ end
14
+ options[:columns] = columns
15
+
16
+ sql = VerticaUtils::VerticaSql.create_table_stmt options
17
+ end
18
+
19
+ def create_all_tables db
20
+ tables = VerticaUtils::SourceDb.get_tables(db)
21
+ sqls = {}
22
+ tables.each do |table|
23
+ puts "Creating #{db}.#{table}"
24
+ sql = "DROP TABLE IF EXISTS #{db}.#{table} CASCADE;"
25
+ VerticaDb::Base.connection.execute sql
26
+ sql = VerticaUtils::VerticaLoader.create_table(:vertica_db => "bidw",
27
+ :vertica_table => table,
28
+ :vertica_schema => db,
29
+ :table => table,
30
+ :db => db)
31
+ sqls["#{table}"] = sql
32
+ VerticaDb::Base.connection.execute sql
33
+ end
34
+ end
35
+
36
+ def prepare_options *args
37
+ options = args.extract_options!
38
+ options.reverse_merge!(
39
+ :host => "sfo-load-dw-01",
40
+ :user => "vertica",
41
+ :pass => "test",
42
+ :db => "bidw",
43
+ :schema => "test",
44
+ :table => "",
45
+ :file => "",
46
+ :delimiter => "\t",
47
+ :null_value => "NULL",
48
+ :enclosed => ""
49
+ )
50
+ return options
51
+ end
52
+
53
+ # VerticaUtils::VerticaLoader.load_to_vertica({:schema => "king", :table => "category_overview_data", :file => "tmp/vertica/category_overview_data.tsv", :null_value => "NULL"})
54
+ def load_to_vertica *args
55
+ list_of_nulls = ["0000-00-00"]
56
+ options = args.extract_options!
57
+ prepared_options = prepare_options options
58
+ Kernel.p prepared_options[:file]
59
+ if prepared_options[:file].blank?
60
+ raise "No input file"
61
+ end
62
+
63
+ begin
64
+ process_file(:file => prepared_options[:file], :list_of_nulls => list_of_nulls, :null_value => prepared_options[:null_value])
65
+ cmd = get_vsql_command(prepared_options)
66
+ Kernel.p cmd
67
+ system(cmd)
68
+ rescue Exception => e
69
+ raise e.message
70
+ end
71
+ end
72
+
73
+ def get_vsql_command prepared_options
74
+ file_extension = prepared_options[:file].split('.').last
75
+ file_handler = ""
76
+ file_handler = "GZIP" if file_extension == "gz"
77
+ sql = "COPY #{prepared_options[:schema]}.#{prepared_options[:table]} FROM LOCAL \'#{prepared_options[:file]}\' #{file_handler} DELIMITER E\'#{prepared_options[:delimiter]}\' NULL as \'#{prepared_options[:null_value]}\' ENCLOSED BY \'#{prepared_options[:enclosed]}\' EXCEPTIONS 'tmp/vertica/load_exceptions.log';"
78
+ cmd = "/opt/vertica/bin/vsql -h #{prepared_options[:host]} -U #{prepared_options[:user]} -w #{prepared_options[:pass]} -d #{prepared_options[:db]} -c \"#{sql}\""
79
+ return cmd
80
+ end
81
+
82
+ def process_file *args
83
+ ### replace the null values in the input file
84
+ options = args.extract_options!
85
+ options[:file].blank? ? return : file = options[:file]
86
+ options[:list_of_nulls].blank? ? list_of_nulls = [] : list_of_nulls = options[:list_of_nulls]
87
+ options[:null_value].blank? ? null_value = "NULL" : null_value = options[:null_value]
88
+
89
+ file_extension = file.split('.').last
90
+ case file_extension
91
+ when "tsv", "csv"
92
+ process_flat_file(file, list_of_nulls, null_value)
93
+ when "gz"
94
+ process_gzip_file(file, list_of_nulls, null_value)
95
+ else
96
+ raise "Un supported file extension"
97
+ end
98
+ end
99
+
100
+ def replace_null(file, list_of_nulls, null_value = "NULL")
101
+ list_of_nulls.each do | value|
102
+ # special case for NULL MySQL datetime/date type but the column is defined NOT NULL
103
+ if value == '0000-00-00'
104
+ cmd1 = "sed -i 's/#{value}/1900-01-01/g' #{file}"
105
+ Kernel.p cmd1
106
+ system(cmd1)
107
+ else
108
+ cmd1 = "sed -i 's/#{value}/#{null_value}/g' #{file}"
109
+ Kernel.p cmd1
110
+ system(cmd1)
111
+ end
112
+ end
113
+ end
114
+
115
+ def process_flat_file file, list_of_nulls, null_value
116
+ # sed
117
+ replace_null(file, list_of_nulls, null_value)
118
+ end
119
+
120
+ def process_gzip_file file, list_of_nulls, null_value
121
+ # unzip
122
+ cmd = "gunzip -f #{file} -c > tmp/temp.txt"
123
+ system(cmd)
124
+ # sed
125
+ replace_null(file, list_of_nulls, null_value)
126
+ # zip
127
+ cmd2 = "gzip tmp/temp.txt -c > #{file}"
128
+ system(cmd2)
129
+ end
130
+
131
+ def get_table_columns * args
132
+ options = args.extract_options!
133
+ if options[:host].blank? or options[:user].blank? or options[:pass].blank? or options[:db].blank? or options[:schema].blank? or options[:table].blank?
134
+ raise "Unspecified host, user, pass, db, schema or table"
135
+ end
136
+ result = {}
137
+ count = 0
138
+ begin
139
+ cmd = "/opt/vertica/bin/vsql -h #{options[:host]} -U #{options[:user]} -w #{options[:pass]} -d #{options[:db]} -c '\\d #{options[:schema]}.#{options[:table]}';"
140
+ puts cmd
141
+ output = `#{cmd}`
142
+ lines = output.split("\n")
143
+ data = lines[3..lines.size-2]
144
+ data.each do |item|
145
+ count += 1
146
+ column_name = item.split("|")[2].lstrip!.rstrip!
147
+ column_type = item.split("|")[3].lstrip!.rstrip!
148
+ puts column_name + " " + column_type
149
+ result[count] = {column_name => column_type}
150
+ end
151
+ rescue Exception => e
152
+ raise e.message
153
+ end
154
+ return result
155
+ end
156
+
157
+ def ssh_connection options
158
+ ssh = Net::SSH.start(options[:ssh_host], options[:ssh_user], :password => options[:ssh_password])
159
+ end
160
+
161
+ def table_definition options
162
+ sql = "SELECT table_schema, table_name, column_name, is_nullable, data_type, column_type, column_key "
163
+ sql += "FROM INFORMATION_SCHEMA.COLUMNS where table_name = '#{options[:table]}' "
164
+ sql += "and table_schema = '#{options[:db]}';"
165
+
166
+ puts sql
167
+
168
+ desc = SourceDb.exec_sql(options[:db], sql)
169
+
170
+ return desc
171
+ end
172
+ end
173
+ end
174
+ end
175
+
@@ -0,0 +1,59 @@
1
+ module Myreplicator
2
+ class VerticaSql
3
+
4
+ def self.create_table_stmt options
5
+ sql = "CREATE TABLE IF NOT EXISTS #{options[:vertica_db]}."
6
+ sql += "#{options[:vertica_schema]}." if options[:vertica_schema]
7
+ sql += "#{options[:table]} ("
8
+
9
+ index = 1
10
+ primary_set = false
11
+
12
+ options[:columns].each do |column|
13
+ sql += "\"#{column['column_name']}\" "
14
+
15
+ sql += data_type(column['data_type'], column['column_type'])
16
+ sql += " "
17
+
18
+ if column['column_key'] == "PRI"
19
+ sql += key(column['column_key']) + " " unless primary_set # set only one primary key
20
+ primary_set = true
21
+ end
22
+
23
+ sql += nullable(column['is_nullable'])
24
+ sql += " "
25
+
26
+ if index < options[:columns].size
27
+ sql += ", "
28
+ else
29
+ sql += ");"
30
+ end
31
+ index += 1
32
+ end
33
+
34
+ puts sql
35
+
36
+ return sql
37
+ end
38
+
39
+ def self.nullable is_nullable
40
+ if is_nullable == "YES"
41
+ return "NULL"
42
+ elsif is_nullable == "NO"
43
+ return "NOT NULL"
44
+ end
45
+ return ""
46
+ end
47
+
48
+ def self.data_type type, col_type
49
+ type = VerticaTypes.convert type, col_type
50
+ result = " #{type} "
51
+ return result
52
+ end
53
+
54
+ def self.key col_key
55
+ col_key = VerticaTypes.convert_key col_key
56
+ return "#{col_key} "
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,60 @@
1
+ module VerticaUtils
2
+ class VerticaSql
3
+
4
+ def self.create_table_stmt options
5
+
6
+ sql = "CREATE TABLE IF NOT EXISTS #{options[:vertica_db]}."
7
+ sql += "#{options[:vertica_schema]}." if options[:vertica_schema]
8
+ sql += "#{options[:table]} ("
9
+
10
+ index = 1
11
+ primary_set = false
12
+
13
+ options[:columns].each do |column|
14
+ sql += "\"#{column['column_name']}\" "
15
+
16
+ sql += data_type(column['data_type'], column['column_type'])
17
+ sql += " "
18
+
19
+ if column['column_key'] == "PRI"
20
+ sql += key(column['column_key']) + " " unless primary_set # set only one primary key
21
+ primary_set = true
22
+ end
23
+
24
+ sql += nullable(column['is_nullable'])
25
+ sql += " "
26
+
27
+ if index < options[:columns].size
28
+ sql += ", "
29
+ else
30
+ sql += ");"
31
+ end
32
+ index += 1
33
+ end
34
+
35
+ puts sql
36
+
37
+ return sql
38
+ end
39
+
40
+ def self.nullable is_nullable
41
+ if is_nullable == "YES"
42
+ return "NULL"
43
+ elsif is_nullable == "NO"
44
+ return "NOT NULL"
45
+ end
46
+ return ""
47
+ end
48
+
49
+ def self.data_type type, col_type
50
+ type = VerticaTypes.convert type, col_type
51
+ result = " #{type} "
52
+ return result
53
+ end
54
+
55
+ def self.key col_key
56
+ col_key = VerticaTypes.convert_key col_key
57
+ return "#{col_key} "
58
+ end
59
+ end
60
+ end
@@ -0,0 +1 @@
1
+ Dir["#{File.expand_path(File.dirname(__FILE__))}/vertica/*.rb"].each { | f | require File.expand_path(f) }
@@ -1,3 +1,3 @@
1
1
  module Myreplicator
2
- VERSION = "1.0.6"
2
+ VERSION = "1.1.0"
3
3
  end
@@ -100,8 +100,13 @@ module Myreplicator
100
100
  Log.run(:job_type => "transporter", :name => "export_file",
101
101
  :file => dump_file, :export_id => export.id) do |log|
102
102
  puts "Downloading #{dump_file}"
103
- sftp.download!(dump_file, File.join(tmp_dir, dump_file.split("/").last))
103
+ local_dump_file = File.join(tmp_dir, dump_file.split("/").last)
104
+ sftp.download!(dump_file, local_dump_file)
104
105
  Transporter.remove!(export, json_file, dump_file)
106
+ # store back up as well
107
+ unless metadata.store_in.blank?
108
+ Transporter.backup_files(metadata.backup_path, json_local_path, local_dump_file)
109
+ end
105
110
  end
106
111
  end #if
107
112
  puts "#{Thread.current.to_s}___Exiting download..."
@@ -109,6 +114,11 @@ module Myreplicator
109
114
  }
110
115
  end
111
116
 
117
+ def self.backup_files location, metadata_path, dump_path
118
+ FileUtils.cp(metadata_path, location)
119
+ FileUtils.cp(dump_path, location)
120
+ end
121
+
112
122
  ##
113
123
  # Returns true if the file should be deleted
114
124
  ##
@@ -0,0 +1,7 @@
1
+ module VerticaDb
2
+ class Base < ActiveRecord::Base
3
+ establish_connection(ActiveRecord::Base.configurations["vertica"])
4
+ end
5
+ end
6
+
7
+ Dir['vertica_db/*.rb'].each { | f | require File.expand_path(f) }
@@ -31,6 +31,13 @@ myreplicator:
31
31
  password: sasan
32
32
  host: 127.0.0.1
33
33
 
34
+ vertica:
35
+ adapter: vertica
36
+ host: ec2-23-22-175-247.compute-1.amazonaws.com
37
+ username: jferris
38
+ password: okl
39
+ database: test
40
+
34
41
  # Warning: The database defined as "test" will be erased and
35
42
  # re-generated from your development database when you run "rake".
36
43
  # Do not set this db to the same as development or production.
@@ -24,9 +24,9 @@ okl_test:
24
24
  password: okl
25
25
  host: 54.243.176.8
26
26
 
27
- destination_db:
27
+ myreplicator:
28
28
  adapter: mysql2
29
- database: raw_sources
29
+ database: myreplicator
30
30
  username: sas
31
31
  password: sasan
32
32
  host: 127.0.0.1
@@ -9,6 +9,11 @@ myreplicator:
9
9
  enclosed_by: '"'
10
10
  lines_terminate_by: '\n'
11
11
 
12
+ ssh_host: localhost
13
+ ssh_user: guest
14
+ ssh_password: sasanguest
15
+ ssh_tmp_dir: /home/guest/tmp
16
+
12
17
  okl_dev:
13
18
  ssh_host: localhost
14
19
  ssh_user: guest
@@ -8,8 +8,6 @@ myreplicator:
8
8
  terminate_by: '\t'
9
9
  enclosed_by: '"'
10
10
  lines_terminate_by: '\n'
11
- auth_required: yes
12
- login_redirect: /
13
11
 
14
12
  okl_dev:
15
13
  ssh_host: localhost