myreplicator 1.0.3 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. data/app/assets/javascripts/myreplicator/application.js +1 -0
  2. data/app/assets/javascripts/myreplicator/cronwtf.js +15 -4
  3. data/app/assets/stylesheets/myreplicator/application.css +64 -9
  4. data/app/assets/stylesheets/myreplicator/slider_handle.png +0 -0
  5. data/app/models/myreplicator/log.rb +44 -4
  6. data/app/views/myreplicator/exports/_form.html.erb +124 -32
  7. data/lib/loader/loader.rb +33 -12
  8. data/lib/myreplicator/version.rb +1 -1
  9. data/lib/transporter/transporter.rb +3 -0
  10. data/test/dummy/config/application.rb +4 -0
  11. data/test/dummy/log/development.log +853 -0
  12. data/test/dummy/tmp/cache/assets/C7F/970/sprockets%2F03f29761343991c6cd347bbd2184d560 +0 -0
  13. data/test/dummy/tmp/cache/assets/CC6/7C0/sprockets%2F881234d3e954859c46bf49bb21f72d79 +0 -0
  14. data/test/dummy/tmp/cache/assets/CCC/A20/sprockets%2F6f3c2f4ae1f4d0219804a257410ce405 +0 -0
  15. data/test/dummy/tmp/cache/assets/CD5/B90/sprockets%2Fc999d13a6a21113981c0d820e8043bdf +0 -0
  16. data/test/dummy/tmp/cache/assets/CD7/030/sprockets%2F9ba4859590582b8b72a650b2b00b6cd2 +0 -0
  17. data/test/dummy/tmp/cache/assets/CE5/670/sprockets%2Fe9e4122f1706626a21da6f8457f088ce +0 -0
  18. data/test/dummy/tmp/cache/assets/D21/D40/sprockets%2Fe5a6e691816fa904f414ea89a6f0586f +0 -0
  19. data/test/dummy/tmp/cache/assets/D69/6F0/sprockets%2F94fff7f55bc4c300b25f3f9361ac1a52 +0 -0
  20. data/test/dummy/tmp/cache/assets/D8B/B60/sprockets%2Faa32227c440a378ccd21218eefeb80bf +0 -0
  21. data/test/dummy/tmp/cache/assets/DA8/910/sprockets%2Fab5775c4a837bd4d97ac394d473cda9b +0 -0
  22. data/test/dummy/tmp/cache/assets/DF8/5D0/sprockets%2Fb815ed34d61cfed96222daa3bfd1d84d +0 -0
  23. data/test/dummy/tmp/cache/assets/E35/4F0/sprockets%2F96b1cdf8db6a1c8eb8abcce05958ae74 +0 -0
  24. metadata +12 -11
  25. data/test/dummy/tmp/myreplicator/okl_test_batchy_batches_1358276518.tsv.gz +0 -0
  26. data/test/dummy/tmp/myreplicator/okl_test_batchy_batches_1358276518.tsv.json +0 -1
  27. data/test/dummy/tmp/myreplicator/okl_test_batchy_batches_1358278078.tsv.gz +0 -0
  28. data/test/dummy/tmp/myreplicator/okl_test_batchy_batches_1358278078.tsv.json +0 -1
@@ -11,6 +11,7 @@
11
11
  // GO AFTER THE REQUIRES BELOW.
12
12
  //
13
13
  //= require jquery
14
+ //=require jquery-ui
14
15
  //= require jquery_ujs
15
16
  //= require_tree .
16
17
  $(function(){
@@ -127,10 +127,21 @@ var CronWTF = {
127
127
 
128
128
  var CronUI = {
129
129
  translate: function(){
130
- var min = $("#min").val();
131
- min_v = min == null ? '*' : this.process('#min',min) ;
132
- var hr = $("#hour").val()
133
- hr_v = hr == null ? '*' : this.process('#hour',hr) ;
130
+ if($("#cron-slider-trigger-min").attr("checked") != "checked"){
131
+ var min = $("#min").val();
132
+ min_v = min == null ? '*' : this.process('#min',min) ;
133
+ }else{
134
+ //grab from display as the lag between slider val assignment and the translation causes errors
135
+ var min = "*/"+$("#cron-n-min").text().replace(':',"")
136
+ min_v = min;
137
+ }
138
+ if($("#cron-slider-trigger-hr").attr("checked") != "checked"){
139
+ var hr = $("#hour").val()
140
+ hr_v = hr == null ? '*' : this.process('#hour',hr) ;
141
+ }else{
142
+ var hr = "*/"+$("#cron-n-hr").text().replace(':',"")
143
+ hr_v = hr;
144
+ }
134
145
  var d = $("#day").val();
135
146
  d_v = d == null ? '*' : this.process('#day',d) ;
136
147
  var mon = $("#month").val();
@@ -9,6 +9,7 @@
9
9
  * compiled file, but it's generally better to create a new file per style scope.
10
10
  *
11
11
  *= require_self
12
+
12
13
  *= require_tree .
13
14
  */
14
15
  @font-face{
@@ -241,6 +242,7 @@ form div.form-section {
241
242
  width:400px;
242
243
  float:left;
243
244
  }
245
+ form div.form-section.right {width:450px;}
244
246
 
245
247
  form label {
246
248
  color:#474747;
@@ -261,10 +263,8 @@ form input {
261
263
  }
262
264
 
263
265
  form span.cron-label {
264
- clear:left;
265
266
  color:#777;
266
- display:block;
267
- float:left;
267
+ display:inline-block;
268
268
  font-size:12px;
269
269
  font-weight:bold;
270
270
  line-height:30px;
@@ -416,7 +416,7 @@ table.data-grid td span.status { display:block; height:16px; width:16px;}
416
416
  table.data-grid td span.status.active {background:url(status.png) 50% 50% no-repeat;}
417
417
  table.data-grid td span.status.inactive {background:url(status-busy.png) 50% 50% no-repeat;}
418
418
 
419
- table.data-grid td a.action {color:#474747; display:block;font-weight:normal;float:left; line-height:16px; margin-right:20px; padding-left:20px; text-decoration:none;}
419
+ table.data-grid td a.action {color:#474747; display:block;font-weight:normal; line-height:16px; margin-bottom:5px; padding-left:20px; text-decoration:none;}
420
420
  table.data-grid td a:hover {text-decoration:underline;}
421
421
  table.data-grid td a.view {background:url(clipboard-list.png) 0 50% no-repeat;}
422
422
  table.data-grid td a.edit {background:url(gear.png) 0 50% no-repeat;}
@@ -570,10 +570,7 @@ p.hint {color:#474747;font-size:12px;line-height:20px;margin:8px 5px;width:300px
570
570
  text-transform:capitalize;
571
571
  }
572
572
 
573
- table.overview {
574
- width:100%;
575
- }
576
-
573
+ table.overview { width:100%;}
577
574
  table.overview td {
578
575
  border-bottom:1px solid #eee;
579
576
  font-size:12px;
@@ -598,4 +595,62 @@ a.kill:before {
598
595
  padding-right:3px;
599
596
  font-size:14px;
600
597
  }
601
- a.kill:hover:before {color:#990000}
598
+ a.kill:hover:before {color:#990000}
599
+
600
+ .cron-option {height:50px;}
601
+ .cron-option span.cron-label {vertical-align:top;}
602
+ .cron-option .cron-select {
603
+ display:inline-block;
604
+ margin-right:10px;
605
+ -webkit-transition: all 0.5s ease-in-out;
606
+ -moz-transition: all 0.5s ease-in-out;
607
+ -o-transition: all 0.5s ease-in-out;
608
+ transition: all 0.5s ease-in-out;
609
+ width:270px;
610
+ }
611
+ .cron-option .cron-select.trans {overflow:hidden;}
612
+ .cron-option .cron-select.hide {margin:0px;width:0;}
613
+ .cron-option label.n { color:#777; display:inline-block; font-size:12px; font-weight:normal; vertical-align:top;}
614
+ .cron-option label em {display:inline-block;min-width:18px;}
615
+ .cron-option label input[type="checkbox"] {width:12px;margin-right:5px;}
616
+ .cron-option .cron-slider {
617
+ display:inline-block;
618
+ overflow:hidden;
619
+ -webkit-transition: all 0.5s ease-in-out;
620
+ -moz-transition: all 0.5s ease-in-out;
621
+ -o-transition: all 0.5s ease-in-out;
622
+ transition: all 0.5s ease-in-out;
623
+ vertical-align:top;
624
+ width:0px;
625
+ }
626
+ .cron-option .cron-slider.show {padding:5px 5px 5px 17px;width:250px;}
627
+ .ui-slider {
628
+ border-radius:999px;
629
+ box-shadow:inset 0 0 3px rgba(0,0,0,0.3);
630
+ width:200px;
631
+ position: relative;
632
+ text-align: left;
633
+ top:3px;
634
+ }
635
+ .ui-slider .ui-slider-handle {
636
+ background:url('slider_handle.png') 0 0 no-repeat;
637
+ position: absolute;
638
+ z-index: 2; width: 17px; height: 17px; cursor: default; }
639
+ .ui-slider .ui-slider-range { position: absolute; z-index: 1; height:15px; display: block; border: 0; }
640
+ .ui-slider-horizontal { height: 13px; }
641
+ .ui-slider-horizontal .ui-slider-handle { top: -2px; margin-left:-8px; }
642
+ .ui-slider-horizontal .ui-slider-range {
643
+ filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#333', endColorstr='#666');
644
+ background:-moz-linear-gradient(top, #333, #666);
645
+ background:-o-linear-gradient(top, #333, #666);
646
+ background:-webkit-gradient(linear, left top, left bottom, from(#333), to(#666));
647
+ border-radius:999px;
648
+ -moz-box-shadow: inset 0 0 3px rgba(0,0,0,0.8);
649
+ -webkit-box-shadow: inset 0 0 3px rgba(0,0,0,0.8);
650
+ box-shadow: inset 0 0 3px rgba(0,0,0,0.8);
651
+ top: 0;
652
+ height: 100%;
653
+
654
+ }
655
+ .ui-slider-horizontal .ui-slider-range-min { left: 0; }
656
+ .ui-slider-horizontal .ui-slider-range-max { right: 0; }
@@ -57,14 +57,14 @@ module Myreplicator
57
57
  # Using PID
58
58
  ##
59
59
  def kill
60
+ return false unless hostname == Socket.gethostname
60
61
  begin
61
62
  Process.kill('TERM', pid)
62
63
  self.state = "killed"
63
64
  self.save!
64
65
  rescue Errno::ESRCH
65
66
  puts "pid #{pid} does not exist!"
66
- self.state = "dead-pid"
67
- self.save!
67
+ mark_dead
68
68
  end
69
69
  end
70
70
 
@@ -85,14 +85,54 @@ module Myreplicator
85
85
  puts "still running #{filepath}"
86
86
  return true
87
87
  rescue Errno::ESRCH
88
- log.state = "error"
89
- log.save!
88
+ log.mark_dead
89
+ end
90
+ end
91
+ end
92
+
93
+ return false
94
+ end
95
+
96
+ ##
97
+ # Clear all logs marked running that are not running
98
+ ##
99
+ def self.clear_deads
100
+ logs = Log.where(:state => "running")
101
+
102
+ if logs.count > 0
103
+ logs.each do |log|
104
+ begin
105
+ Process.getpgid(log.pid) if hostname == Socket.gethostname
106
+ rescue Errno::ESRCH
107
+ log.mark_dead
90
108
  end
91
109
  end
92
110
  end
93
111
 
112
+ end
113
+
114
+ ##
115
+ # Gets a jobtype, file and export_id
116
+ # returns true if the job is completed
117
+ ##
118
+ def self.completed? *args
119
+ options = args.extract_options!
120
+ log = Log.where(:export_id => options[:export_id],
121
+ :file => options[:export_id],
122
+ :job_type => options[:job_type]).last
123
+ if log.nil?
124
+ return true
125
+ else
126
+ return true if log.state != "running"
127
+ end
128
+
94
129
  return false
95
130
  end
96
131
 
132
+ def mark_dead
133
+ self.state = "dead"
134
+ self.save!
135
+ end
136
+
97
137
  end
98
138
  end
@@ -48,39 +48,72 @@ export_type = ["incremental","fulldump"]
48
48
  <option value="false" <% if @export.active == false %>SELECTED<% end %>>false</option>
49
49
  </select>
50
50
  </div>
51
- <div class="form-section">
51
+ <div class="form-section right">
52
52
  <%= f.hidden_field :cron %>
53
53
  <label>Cron</label>
54
- <span class="cron-label">minutes:</span><select id="min" class="cron" multiple data-placeholder="Minutes">
55
- <option value="*">*</option>
56
- <% (0..59).each do |n| %>
57
- <option value="<%=n %>"><%= n%></option>
58
- <% end %>
59
- </select><br/>
60
- <span class="cron-label">hours:</span><select id="hour" class="cron" multiple data-placeholder="Hours">
61
- <option value="*">*</option>
62
- <% (0..23).each do |n| %>
63
- <option value="<%=n %>"><%= n%></option>
64
- <% end %>
65
- </select><br/>
66
- <span class="cron-label">days:</span><select id="day" class="cron" multiple data-placeholder="Days">
67
- <option value="*">*</option>
68
- <% (1..31).each do |n| %>
69
- <option value="<%=n %>"><%= n%></option>
70
- <% end %>
71
- </select><br/>
72
- <span class="cron-label">months:</span><select id="month" class="cron" multiple data-placeholder="Months">
73
- <option value="*">*</option>
74
- <% (1..12).each do |n| %>
75
- <option value="<%= n - 1%>"><%= Date::MONTHNAMES[n]%></option>
76
- <% end %>
77
- </select><br/>
78
- <span class="cron-label">weekday:</span><select id="dow" class="cron" multiple data-placeholder="Day of the Week">
79
- <option value="*">*</option>
80
- <% (0..6).each do |n| %>
81
- <option value="<%=n %>"><%= Date::DAYNAMES[n]%></option>
82
- <% end %>
83
- </select>
54
+ <div class="cron-option">
55
+ <span class="cron-label">minutes:</span>
56
+ <div class="cron-select">
57
+ <select id="min" class="cron" multiple data-placeholder="Minutes">
58
+ <option value="*">*</option>
59
+ <% (0..59).each do |n| %>
60
+ <option value="<%=n %>"><%= n%></option>
61
+ <% end %>
62
+ </select>
63
+ </div>
64
+ <label class="n"><input type="checkbox" data-slider="cron-min" data-chosen="min" id="cron-slider-trigger-min"/>every <em id="cron-n-min">n:</em></label>
65
+ <div class="cron-slider">
66
+ <div id="cron-min"></div>
67
+ </div>
68
+ </div>
69
+ <div class="cron-option">
70
+ <span class="cron-label">hours:</span>
71
+ <div class="cron-select">
72
+ <select id="hour" class="cron" multiple data-placeholder="Hours">
73
+ <option value="*">*</option>
74
+ <% (0..23).each do |n| %>
75
+ <option value="<%=n %>"><%= n%></option>
76
+ <% end %>
77
+ </select>
78
+ </div>
79
+ <label class="n"><input type="checkbox" data-slider="cron-hr" data-chosen="hour" id="cron-slider-trigger-hr"/>every <em id="cron-n-hr">n:</em></label>
80
+ <div class="cron-slider">
81
+ <div id="cron-hr"></div>
82
+ </div>
83
+ </div>
84
+ <div class="cron-option">
85
+ <span class="cron-label">days:</span>
86
+ <div class="cron-select">
87
+ <select id="day" class="cron" multiple data-placeholder="Days">
88
+ <option value="*">*</option>
89
+ <% (1..31).each do |n| %>
90
+ <option value="<%=n %>"><%= n%></option>
91
+ <% end %>
92
+ </select>
93
+ </div>
94
+ </div>
95
+ <div class="cron-option">
96
+ <span class="cron-label">months:</span>
97
+ <div class="cron-select">
98
+ <select id="month" class="cron" multiple data-placeholder="Months">
99
+ <option value="*">*</option>
100
+ <% (1..12).each do |n| %>
101
+ <option value="<%= n - 1%>"><%= Date::MONTHNAMES[n]%></option>
102
+ <% end %>
103
+ </select>
104
+ </div>
105
+ </div>
106
+ <div class="cron-option">
107
+ <span class="cron-label">weekday:</span>
108
+ <div class="cron-select">
109
+ <select id="dow" class="cron" multiple data-placeholder="Day of the Week">
110
+ <option value="*">*</option>
111
+ <% (0..6).each do |n| %>
112
+ <option value="<%=n %>"><%= Date::DAYNAMES[n]%></option>
113
+ <% end %>
114
+ </select>
115
+ </div>
116
+ </div>
84
117
  <label style="clear:left">Cron synatx</label>
85
118
  <div id="cron-val"></div>
86
119
  <label>Cron translation</label>
@@ -112,6 +145,38 @@ $(function(){
112
145
  exportSchemaSelect($(this).val())
113
146
  });
114
147
  exportSchemaSelect($("#export_source_schema").val());
148
+ $("#cron-min").slider({
149
+ min: 1,
150
+ max: 59,
151
+ range: "min",
152
+ slide: function(event,ui){
153
+ $("#cron-n-min").text(ui.value+":")
154
+ CronUI.translate();
155
+ }
156
+ });
157
+ $("#cron-hr").slider({
158
+ min: 1,
159
+ max: 23,
160
+ range: "min",
161
+ slide: function(event,ui){
162
+ $("#cron-n-hr").text(ui.value+":")
163
+ CronUI.translate();
164
+ }
165
+ });
166
+ $("label.n").find("input").click(function(e){
167
+ var box = $(this)
168
+ var val = $("#"+box.data("slider")).slider("value");
169
+ if(box.attr("checked") == "checked"){
170
+ box.parent("label").prev("div").addClass("trans").addClass("hide").end().next("div").addClass("show").end().find("em").text(val+":");
171
+ $("#"+box.data("chosen")).val("").trigger("liszt:updated");
172
+ }else{
173
+ box.parent("label").prev("div").removeClass("hide").end().next("div").removeClass("show").end().find('em').text("n:");
174
+ var d = box.parent("label").prev("div")
175
+ window.setTimeout(function(){d.removeClass('trans')},500)
176
+ }
177
+ CronUI.translate();
178
+ })
179
+
115
180
  <%- if @edit -%>
116
181
  editInit();
117
182
  <%- end -%>
@@ -123,7 +188,34 @@ function editInit(){
123
188
  var cron_vals = cron.split(" ");
124
189
  for(i=0;i<cron_vals.length; i++){
125
190
  //console.log(displays[i]+" val: "+cron_vals[i])
126
- $(displays[i]).val(cron_vals[i]).trigger("liszt:updated");
191
+ //console.log(cron_vals[i].indexOf(","))
192
+ if(i < 2){
193
+ if(cron_vals[i].toString().charAt(1) == '/'){
194
+ if(displays[i] == "#min"){
195
+ var val = cron_vals[i].toString().split('/')[1];
196
+ $("#cron-slider-trigger-min").attr("checked","checked").parent("label").prev("div").addClass("trans").addClass("hide").end().next("div").addClass("show").end().find("em").text(val+":");
197
+ $("#cron-min").slider("value",val)
198
+ }else{
199
+ var val = cron_vals[i].toString().split('/')[1];
200
+ $("#cron-slider-trigger-hr").attr("checked","checked").parent("label").prev("div").addClass("trans").addClass("hide").end().next("div").addClass("show").end().find("em").text(val+":");
201
+ $("#cron-hr").slider("value",val)
202
+ }
203
+ }else{
204
+ if(cron_vals[i].indexOf(",") != -1){
205
+ var vals = cron_vals[i].split(',')
206
+ $(displays[i]).val(vals).trigger("liszt:updated");
207
+ }else{
208
+ $(displays[i]).val(cron_vals[i]).trigger("liszt:updated");
209
+ }
210
+ }
211
+ }else{
212
+ if(cron_vals[i].indexOf(",") != -1){
213
+ var vals = cron_vals[i].split(',')
214
+ $(displays[i]).val(vals).trigger("liszt:updated");
215
+ }else{
216
+ $(displays[i]).val(cron_vals[i]).trigger("liszt:updated");
217
+ }
218
+ }
127
219
  }
128
220
  CronUI.translate();
129
221
  $("#export_table_name").val("<%= @export.table_name %>").trigger("liszt:updated");
data/lib/loader/loader.rb CHANGED
@@ -15,8 +15,11 @@ module Myreplicator
15
15
 
16
16
  ##
17
17
  # Main method provided for resque
18
+ # Reconnection provided for resque workers
18
19
  ##
19
20
  def self.perform
21
+ ActiveRecord::Base.verify_active_connections!
22
+ ActiveRecord::Base.connection.reconnect!
20
23
  load # Kick off the load process
21
24
  end
22
25
 
@@ -75,9 +78,12 @@ module Myreplicator
75
78
  :name => "#{metadata.export_type}_import",
76
79
  :file => metadata.filename,
77
80
  :export_id => metadata.export_id) do |log|
78
-
79
- Loader.initial_load metadata
80
- Loader.cleanup metadata
81
+
82
+ if Loader.transfer_completed? metadata
83
+ Loader.initial_load metadata
84
+ Loader.cleanup metadata
85
+ end
86
+
81
87
  end
82
88
  }
83
89
  end
@@ -100,9 +106,12 @@ module Myreplicator
100
106
  :name => "incremental_import",
101
107
  :file => metadata.filename,
102
108
  :export_id => metadata.export_id) do |log|
103
-
104
- Loader.incremental_load metadata
105
- Loader.cleanup metadata
109
+
110
+ if Loader.transfer_completed? metadata
111
+ Loader.incremental_load metadata
112
+ Loader.cleanup metadata
113
+ end
114
+
106
115
  end
107
116
  end # group
108
117
  }
@@ -145,18 +154,17 @@ module Myreplicator
145
154
  exp = Export.find(metadata.export_id)
146
155
  Loader.unzip(metadata.filename)
147
156
  metadata.zipped = false
148
-
157
+
149
158
  cmd = ImportSql.initial_load(:db => exp.destination_schema,
150
159
  :filepath => metadata.destination_filepath(tmp_dir))
151
160
  puts cmd
152
-
161
+
153
162
  result = `#{cmd}` # execute
154
163
  unless result.nil?
155
164
  if result.size > 0
156
165
  raise Exceptions::LoaderError.new("Initial Load #{metadata.filename} Failed!\n#{result}")
157
166
  end
158
167
  end
159
-
160
168
  end
161
169
 
162
170
  ##
@@ -170,14 +178,14 @@ module Myreplicator
170
178
 
171
179
  options = {:table_name => exp.table_name, :db => exp.destination_schema,
172
180
  :filepath => metadata.destination_filepath(tmp_dir)}
173
-
181
+
174
182
  if metadata.export_type == "incremental_outfile"
175
183
  options[:fields_terminated_by] = ";~;"
176
184
  options[:lines_terminated_by] = "\\n"
177
185
  end
178
-
186
+
179
187
  cmd = ImportSql.load_data_infile(options)
180
-
188
+
181
189
  puts cmd
182
190
 
183
191
  result = `#{cmd}` # execute
@@ -189,6 +197,19 @@ module Myreplicator
189
197
  end
190
198
  end
191
199
 
200
+ ##
201
+ # Returns true if the transfer of the file
202
+ # being loaded is completed
203
+ ##
204
+ def self.transfer_completed? metadata
205
+ if Log.completed?(:export_id => metadata.export_id,
206
+ :file => metadata.destination_filepath(tmp_dir),
207
+ :job_type => "transporter")
208
+ return true
209
+ end
210
+ return false
211
+ end
212
+
192
213
  ##
193
214
  # Deletes the metadata file and extract
194
215
  ##