myreplicator 0.0.16 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/app/assets/stylesheets/myreplicator/application.css +53 -10
- data/app/controllers/myreplicator/exports_controller.rb +17 -1
- data/app/controllers/myreplicator/home_controller.rb +4 -1
- data/app/helpers/myreplicator/application_helper.rb +20 -0
- data/app/models/myreplicator/export.rb +108 -23
- data/app/models/myreplicator/log.rb +90 -0
- data/app/views/myreplicator/exports/_form.html.erb +33 -9
- data/app/views/myreplicator/home/_home_menu.erb +6 -2
- data/app/views/myreplicator/home/errors.html.erb +15 -5
- data/app/views/myreplicator/home/index.html.erb +20 -43
- data/db/migrate/20121025191622_create_myreplicator_exports.rb +0 -6
- data/db/migrate/20121212003652_create_myreplicator_logs.rb +21 -0
- data/lib/exporter/export_metadata.rb +8 -1
- data/lib/exporter/mysql_exporter.rb +32 -0
- data/lib/exporter/sql_commands.rb +85 -4
- data/lib/loader/loader.rb +48 -7
- data/lib/myreplicator.rb +16 -4
- data/lib/myreplicator/version.rb +1 -1
- data/lib/transporter/parallelizer.rb +8 -0
- data/lib/transporter/transporter.rb +75 -11
- data/test/dummy/config/myreplicator.yml +0 -2
- data/test/dummy/config/myreplicator.yml~ +2 -0
- data/test/dummy/db/migrate/{20121108204327_create_myreplicator_exports.myreplicator.rb~ → 20121213003552_create_myreplicator_exports.myreplicator.rb} +3 -1
- data/test/dummy/db/migrate/20121213003553_create_myreplicator_logs.myreplicator.rb +22 -0
- data/test/dummy/db/schema.rb +24 -7
- data/test/dummy/log/development.log +1083 -0
- data/test/dummy/tmp/cache/assets/CD5/B90/sprockets%2Fc999d13a6a21113981c0d820e8043bdf +0 -0
- data/test/dummy/tmp/cache/assets/D8B/B60/sprockets%2Faa32227c440a378ccd21218eefeb80bf +0 -0
- data/test/dummy/tmp/cache/assets/DF8/5D0/sprockets%2Fb815ed34d61cfed96222daa3bfd1d84d +0 -0
- data/test/dummy/tmp/myreplicator/okl_test_batchy_batches_1355432256.tsv.gz +0 -0
- data/test/unit/myreplicator/log_test.rb +9 -0
- metadata +14 -8
- data/test/dummy/db/migrate/20121115194022_create_myreplicator_exports.myreplicator.rb +0 -36
@@ -129,7 +129,7 @@ html { overflow-y:scroll;overflow-x:auto; }
|
|
129
129
|
padding-left:20px;
|
130
130
|
margin-top:-10px;
|
131
131
|
}
|
132
|
-
.home-menu span {
|
132
|
+
.home-menu > span {
|
133
133
|
color:#474747;
|
134
134
|
display:block;
|
135
135
|
float:left;
|
@@ -150,6 +150,7 @@ html { overflow-y:scroll;overflow-x:auto; }
|
|
150
150
|
letter-spacing:1px;
|
151
151
|
margin:8px 15px 8px 0;
|
152
152
|
padding:3px 12px;
|
153
|
+
position:relative;
|
153
154
|
text-decoration:none;
|
154
155
|
-webkit-transition: all 0.25s ease-in-out;
|
155
156
|
-moz-transition: all 0.25s ease-in-out;
|
@@ -178,7 +179,31 @@ html { overflow-y:scroll;overflow-x:auto; }
|
|
178
179
|
text-shadow: 0 1px #f9f9f9;
|
179
180
|
}
|
180
181
|
.home-menu a:active {background:#e0e0e0;}
|
181
|
-
|
182
|
+
.home-menu a span {
|
183
|
+
background:#990000;
|
184
|
+
box-shadow:0 0 3px rgba(0,0,0,0.4);
|
185
|
+
border-radius:999px;
|
186
|
+
color:#fff;
|
187
|
+
font-size:11px;
|
188
|
+
font-weight:normal;
|
189
|
+
height:13px;
|
190
|
+
display:block;
|
191
|
+
opacity:0;
|
192
|
+
padding:0 0 3px 1px;
|
193
|
+
position:absolute;
|
194
|
+
top:-7px;
|
195
|
+
right:-5px;
|
196
|
+
text-align:center;
|
197
|
+
text-shadow:1px 1px #333;
|
198
|
+
width:13px;
|
199
|
+
-webkit-transition: all 0.25s ease-in-out;
|
200
|
+
-moz-transition: all 0.25s ease-in-out;
|
201
|
+
-o-transition: all 0.25s ease-in-out;
|
202
|
+
transition: all 0.25s ease-in-out;
|
203
|
+
}
|
204
|
+
.home-menu a.overview span {background:#1F6015;}
|
205
|
+
.home-menu a:hover span,
|
206
|
+
.home-menu a.on span {opacity:1;}
|
182
207
|
#sub-header a.active{
|
183
208
|
background:#666;
|
184
209
|
background-image: -moz-linear-gradient(top, #555, #777);
|
@@ -203,6 +228,14 @@ h2 {
|
|
203
228
|
margin:20px;
|
204
229
|
}
|
205
230
|
|
231
|
+
h6 {
|
232
|
+
color:#474747;
|
233
|
+
font-size:14px;
|
234
|
+
font-weight:normal;
|
235
|
+
text-shadow:0 1px #fff;
|
236
|
+
margin:10px 5px;
|
237
|
+
}
|
238
|
+
|
206
239
|
form {margin:5px 20px;}
|
207
240
|
form div.form-section {
|
208
241
|
width:400px;
|
@@ -324,15 +357,9 @@ div.flash {
|
|
324
357
|
z-index:110;
|
325
358
|
}
|
326
359
|
|
327
|
-
div.table-wrapper {
|
328
|
-
margin:10px 20px;
|
329
|
-
}
|
360
|
+
div.table-wrapper { margin:10px 20px; }
|
330
361
|
|
331
|
-
table.data-grid {
|
332
|
-
font-size:12px;
|
333
|
-
margin:10px 0;
|
334
|
-
width:100%;
|
335
|
-
}
|
362
|
+
table.data-grid { font-size:12px; margin:10px 0; width:100%; }
|
336
363
|
table.data-grid thead tr,
|
337
364
|
#states-list thead tr{
|
338
365
|
background-color:#ddd;
|
@@ -542,3 +569,19 @@ p.hint {color:#474747;font-size:12px;line-height:20px;margin:8px 5px;width:300px
|
|
542
569
|
margin:0 0 10px;
|
543
570
|
text-transform:capitalize;
|
544
571
|
}
|
572
|
+
|
573
|
+
table.overview {
|
574
|
+
width:100%;
|
575
|
+
}
|
576
|
+
|
577
|
+
table.overview td {
|
578
|
+
border-bottom:1px solid #eee;
|
579
|
+
font-size:12px;
|
580
|
+
padding:5px 8px 5px 5px;
|
581
|
+
text-shadow: 0 1px #fff;
|
582
|
+
vertical-align:middle;
|
583
|
+
}
|
584
|
+
table.overview tr:nth-child(even) {background:#fbfbfb;}
|
585
|
+
table.overview tr:last-child td {border-bottom:0px;}
|
586
|
+
table.overview td span.name {color:#474747;display:block;font-weight:bold; font-size:13px;padding-bottom:5px;}
|
587
|
+
table.overview td span.file {color:#999;display:block;font-style:italic;}
|
@@ -31,7 +31,8 @@ module Myreplicator
|
|
31
31
|
# GET /exports/new.json
|
32
32
|
def new
|
33
33
|
@export = Export.new
|
34
|
-
|
34
|
+
@dbs = get_dbs
|
35
|
+
@tables = db_metadata
|
35
36
|
respond_to do |format|
|
36
37
|
format.html # new.html.erb
|
37
38
|
format.json { render json: @export }
|
@@ -41,6 +42,8 @@ module Myreplicator
|
|
41
42
|
# GET /exports/1/edit
|
42
43
|
def edit
|
43
44
|
@export = Export.find(params[:id])
|
45
|
+
@dbs = get_dbs
|
46
|
+
@tables = db_metadata
|
44
47
|
@edit = true
|
45
48
|
end
|
46
49
|
|
@@ -48,6 +51,7 @@ module Myreplicator
|
|
48
51
|
# POST /exports.json
|
49
52
|
def create
|
50
53
|
@export = Export.new(params[:export])
|
54
|
+
@dbs = get_dbs
|
51
55
|
|
52
56
|
respond_to do |format|
|
53
57
|
if @export.save
|
@@ -64,6 +68,7 @@ module Myreplicator
|
|
64
68
|
# PUT /exports/1.json
|
65
69
|
def update
|
66
70
|
@export = Export.find(params[:id])
|
71
|
+
@dbs = get_dbs
|
67
72
|
|
68
73
|
respond_to do |format|
|
69
74
|
if @export.update_attributes(params[:export])
|
@@ -101,6 +106,17 @@ module Myreplicator
|
|
101
106
|
def sort_direction
|
102
107
|
%w[asc desc].include?(params[:direction]) ? params[:direction] : "asc"
|
103
108
|
end
|
109
|
+
|
110
|
+
def db_metadata
|
111
|
+
@db_metadata ||= Myreplicator::Export.available_tables
|
112
|
+
end
|
104
113
|
|
114
|
+
def get_dbs
|
115
|
+
return db_metadata.keys
|
116
|
+
end
|
117
|
+
|
118
|
+
def get_tables(db)
|
119
|
+
return db_metadata[db]
|
120
|
+
end
|
105
121
|
end
|
106
122
|
end
|
@@ -7,6 +7,8 @@ module Myreplicator
|
|
7
7
|
@tab = 'home'
|
8
8
|
@option = 'overview'
|
9
9
|
@exports = Export.order('state DESC')
|
10
|
+
@logs = Log.where(:state => 'running').order("started_at DESC")
|
11
|
+
@now = Time.zone.now
|
10
12
|
respond_to do |format|
|
11
13
|
format.html # index.html.erb
|
12
14
|
format.json { render json: @exports }
|
@@ -16,7 +18,8 @@ module Myreplicator
|
|
16
18
|
def errors
|
17
19
|
@tab = 'home'
|
18
20
|
@option = 'errors'
|
19
|
-
@exports = Export.where("error is not null").order('source_schema ASC')
|
21
|
+
@exports = Export.where("error is not null").order('source_schema ASC')
|
22
|
+
@logs = Log.where(:state => 'error').order("started_at DESC")
|
20
23
|
end
|
21
24
|
|
22
25
|
end
|
@@ -8,5 +8,25 @@ module Myreplicator
|
|
8
8
|
link_to content_tag(:span, title), {:sort => column, :direction => direction}, {:class => css_class}
|
9
9
|
end
|
10
10
|
|
11
|
+
def chronos(secs)
|
12
|
+
[[60, :seconds], [60, :minutes], [24, :hours], [1000, :days]].map{ |count, name|
|
13
|
+
if secs > 0
|
14
|
+
secs, n = secs.divmod(count)
|
15
|
+
"#{n.to_i} #{name}"
|
16
|
+
end
|
17
|
+
}.compact.reverse.join(', ')
|
18
|
+
end
|
19
|
+
|
20
|
+
def err_count
|
21
|
+
total = Log.where(:state => 'error').count
|
22
|
+
return total
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
def run_count
|
27
|
+
total = Log.where(:state => 'running').count
|
28
|
+
return total
|
29
|
+
end
|
30
|
+
|
11
31
|
end
|
12
32
|
end
|
@@ -28,8 +28,22 @@ module Myreplicator
|
|
28
28
|
)
|
29
29
|
|
30
30
|
attr_reader :filename
|
31
|
-
|
32
|
-
|
31
|
+
|
32
|
+
@queue = :myreplicator_export # Provided for Resque
|
33
|
+
|
34
|
+
##
|
35
|
+
# Perfoms the export job, Provided for Resque
|
36
|
+
##
|
37
|
+
def self.perform(export_id, *args)
|
38
|
+
options = args.extract_options!
|
39
|
+
export_obj = Export.find(export_id)
|
40
|
+
export_obj.export_table
|
41
|
+
end
|
42
|
+
|
43
|
+
##
|
44
|
+
# Runs the export process using the required Exporter library
|
45
|
+
##
|
46
|
+
def export_table
|
33
47
|
exporter = MysqlExporter.new
|
34
48
|
exporter.export_table self
|
35
49
|
end
|
@@ -70,34 +84,90 @@ module Myreplicator
|
|
70
84
|
puts "Connecting SFTP..."
|
71
85
|
return connection_factory(:sftp)
|
72
86
|
end
|
73
|
-
|
87
|
+
|
88
|
+
##
|
89
|
+
# Connects to the server via ssh/sftp
|
90
|
+
##
|
74
91
|
def connection_factory type
|
92
|
+
config = Myreplicator.configs[self.source_schema]
|
93
|
+
|
75
94
|
case type
|
76
95
|
when :ssh
|
77
|
-
if
|
78
|
-
return Net::SSH.start(
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
elsif(Myreplicator.configs[self.source_schema].has_key? "ssh_private_key")
|
83
|
-
return Net::SSH.start(Myreplicator.configs[self.source_schema]["ssh_host"],
|
84
|
-
Myreplicator.configs[self.source_schema]["ssh_user"],
|
85
|
-
:keys => [Myreplicator.configs[self.source_schema]["ssh_private_key"]])
|
96
|
+
if config.has_key? "ssh_password"
|
97
|
+
return Net::SSH.start(config["ssh_host"], config["ssh_user"], :password => config["ssh_password"])
|
98
|
+
|
99
|
+
elsif(config.has_key? "ssh_private_key")
|
100
|
+
return Net::SSH.start(config["ssh_host"], config["ssh_user"], :keys => [config["ssh_private_key"]])
|
86
101
|
end
|
87
102
|
when :sftp
|
88
|
-
if
|
89
|
-
return Net::SFTP.start(
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
elsif(Myreplicator.configs[self.source_schema].has_key? "ssh_private_key")
|
94
|
-
return Net::SFTP.start(Myreplicator.configs[self.source_schema]["ssh_host"],
|
95
|
-
Myreplicator.configs[self.source_schema]["ssh_user"],
|
96
|
-
:keys => [Myreplicator.configs[self.source_schema]["ssh_private_key"]])
|
103
|
+
if config.has_key? "ssh_password"
|
104
|
+
return Net::SFTP.start(config["ssh_host"], config["ssh_user"], :password => config["ssh_password"])
|
105
|
+
|
106
|
+
elsif(config.has_key? "ssh_private_key")
|
107
|
+
return Net::SFTP.start(config["ssh_host"], config["ssh_user"], :keys => [config["ssh_private_key"]])
|
97
108
|
end
|
98
109
|
end
|
99
110
|
end
|
100
111
|
|
112
|
+
##
|
113
|
+
# Returns a hash of {DB_NAME => [TableName1,...], DB => ...}
|
114
|
+
##
|
115
|
+
def self.available_tables
|
116
|
+
metadata = {}
|
117
|
+
available_dbs.each do |db|
|
118
|
+
tables = SourceDb.get_tables(db)
|
119
|
+
metadata[db] = tables
|
120
|
+
end
|
121
|
+
return metadata
|
122
|
+
end
|
123
|
+
|
124
|
+
##
|
125
|
+
# List of all avaiable databases from database.yml file
|
126
|
+
# All Export/Load jobs can use these databases
|
127
|
+
##
|
128
|
+
def self.available_dbs
|
129
|
+
dbs = ActiveRecord::Base.configurations.keys
|
130
|
+
dbs.delete("development")
|
131
|
+
dbs.delete("test")
|
132
|
+
return dbs
|
133
|
+
end
|
134
|
+
|
135
|
+
##
|
136
|
+
# NOTE: Provided for Resque use
|
137
|
+
# Schedules all the exports in resque
|
138
|
+
# Requires Resque Scheduler
|
139
|
+
##
|
140
|
+
def self.schedule_in_resque
|
141
|
+
exports = Export.find(:all)
|
142
|
+
exports.each do |export|
|
143
|
+
if export.active
|
144
|
+
export.schedule
|
145
|
+
else
|
146
|
+
Resque.remove_schedule(export.schedule_name)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
Resque.reload_schedule! # Reload all schedules in Resque
|
150
|
+
end
|
151
|
+
|
152
|
+
##
|
153
|
+
# Name used for the job in Resque
|
154
|
+
##
|
155
|
+
def schedule_name
|
156
|
+
name = "#{source_schema}_#{destination_schema}_#{table_name}"
|
157
|
+
end
|
158
|
+
|
159
|
+
##
|
160
|
+
# Schedules the export job in Resque
|
161
|
+
##
|
162
|
+
def schedule
|
163
|
+
Resque.set_schedule(schedule_name, {
|
164
|
+
:cron => cron,
|
165
|
+
:class => "Myreplicator::Export",
|
166
|
+
:queue => "myreplicator_export",
|
167
|
+
:args => id
|
168
|
+
})
|
169
|
+
end
|
170
|
+
|
101
171
|
##
|
102
172
|
# Inner Class that connects to the source database
|
103
173
|
# Handles connecting to multiple databases
|
@@ -106,9 +176,24 @@ module Myreplicator
|
|
106
176
|
class SourceDb < ActiveRecord::Base
|
107
177
|
|
108
178
|
def self.connect db
|
109
|
-
@@connected ||= true
|
110
179
|
establish_connection(ActiveRecord::Base.configurations[db])
|
111
|
-
|
180
|
+
end
|
181
|
+
|
182
|
+
##
|
183
|
+
# Returns tables as an Array
|
184
|
+
# releases the connection
|
185
|
+
##
|
186
|
+
def self.get_tables(db)
|
187
|
+
tables = []
|
188
|
+
begin
|
189
|
+
self.connect(db)
|
190
|
+
tables = self.connection.tables
|
191
|
+
self.connection_pool.release_connection
|
192
|
+
rescue Mysql2::Error => e
|
193
|
+
puts "Connection to #{db} Failed!"
|
194
|
+
puts e.message
|
195
|
+
end
|
196
|
+
return tables
|
112
197
|
end
|
113
198
|
|
114
199
|
def self.exec_sql source_db,sql
|
@@ -0,0 +1,90 @@
|
|
1
|
+
module Myreplicator
|
2
|
+
class Log < ActiveRecord::Base
|
3
|
+
attr_accessible(:pid,
|
4
|
+
:job_type,
|
5
|
+
:name,
|
6
|
+
:file,
|
7
|
+
:state,
|
8
|
+
:thread_state,
|
9
|
+
:hostname,
|
10
|
+
:export_id,
|
11
|
+
:error,
|
12
|
+
:backtrace,
|
13
|
+
:guid,
|
14
|
+
:started_at,
|
15
|
+
:finished_at)
|
16
|
+
|
17
|
+
##
|
18
|
+
# Creates a log object
|
19
|
+
# Stores the state and related information about the job
|
20
|
+
# File's names are supposed to be unique
|
21
|
+
##
|
22
|
+
def self.run *args
|
23
|
+
options = args.extract_options!
|
24
|
+
options.reverse_merge!(:started_at => Time.now,
|
25
|
+
:pid => Process.pid,
|
26
|
+
:hostname => Socket.gethostname,
|
27
|
+
:guid => SecureRandom.hex(5),
|
28
|
+
:thread_state => Thread.current.status,
|
29
|
+
:state => "new")
|
30
|
+
|
31
|
+
log = Log.create options
|
32
|
+
|
33
|
+
unless log.running?
|
34
|
+
begin
|
35
|
+
log.state = "running"
|
36
|
+
log.save!
|
37
|
+
|
38
|
+
yield log
|
39
|
+
|
40
|
+
log.state = "completed"
|
41
|
+
rescue Exception => e
|
42
|
+
log.state = "error"
|
43
|
+
log.error = e.message
|
44
|
+
log.backtrace = e.backtrace
|
45
|
+
|
46
|
+
ensure
|
47
|
+
log.finished_at = Time.now
|
48
|
+
log.thread_state = Thread.current.status
|
49
|
+
log.save!
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
##
|
56
|
+
# Kills the job if running
|
57
|
+
# Using PID
|
58
|
+
##
|
59
|
+
def kill
|
60
|
+
begin
|
61
|
+
Process.kill('TERM', pid)
|
62
|
+
rescue Errno::ESRCH
|
63
|
+
puts "pid #{pid} does not exist!"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
##
|
68
|
+
# Checks to see if the PID of the log is active or not
|
69
|
+
##
|
70
|
+
def running?
|
71
|
+
logs = Log.where(:file => file, :job_type => job_type, :state => "running")
|
72
|
+
|
73
|
+
if logs.count > 0
|
74
|
+
logs.each do |log|
|
75
|
+
begin
|
76
|
+
Process.getpgid(log.pid)
|
77
|
+
puts "still running #{filepath}"
|
78
|
+
return true
|
79
|
+
rescue Errno::ESRCH
|
80
|
+
log.state = "error"
|
81
|
+
log.save!
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
return false
|
87
|
+
end
|
88
|
+
|
89
|
+
end
|
90
|
+
end
|