myreplicator 0.1.1 → 1.0.0

data/app/controllers/myreplicator/exports_controller.rb

@@ -45,14 +45,17 @@ module Myreplicator
       @dbs = get_dbs
       @tables = db_metadata
       @edit = true
-      @export.schedule # schedule in resque
+
+      Myreplicator::Export.schedule_in_resque # schedule in resque
+
     end
   
     # POST /exports
     # POST /exports.json
     def create
       @export = Export.new(params[:export])
-      @export.schedule # schedule in resque
+
+      Myreplicator::Export.schedule_in_resque # schedule in resque
 
       @dbs = get_dbs
 
@@ -71,7 +74,8 @@ module Myreplicator
     # PUT /exports/1.json
     def update
       @export = Export.find(params[:id])
-      @export.schedule # schedule in resque
+      Myreplicator::Export.schedule_in_resque # schedule in resque
+
       @dbs = get_dbs
   
       respond_to do |format|

data/app/models/myreplicator/export.rb

@@ -51,6 +51,21 @@ module Myreplicator
       end
     end
 
+    def export_type?
+      if state == "new"
+        return :new
+      elsif incremental_export?
+        return :incremental
+      end
+    end
+
+    def incremental_export?
+      if export_type == "incremental"
+        return true
+      end
+      return false
+    end
+
     def filename
       @file_name ||= "#{source_schema}_#{table_name}_#{Time.now.to_i}.tsv"
     end
@@ -172,6 +187,20 @@ module Myreplicator
                           })
     end
 
+    ##
+    # Throws ExportIgnored if the job is still running
+    # Checks the state of the job using PID and state
+    ##
+    def is_running?
+      return false if state != "exporting"
+      begin
+        Process.getpgid(exporter_pid)
+        raise Exceptions::ExportIgnored.new("Ignored")
+      rescue Errno::ESRCH
+        return false
+      end
+    end
+
     ##
     # Inner Class that connects to the source database 
     # Handles connecting to multiple databases

data/app/views/myreplicator/exports/_form.html.erb

@@ -24,6 +24,8 @@ export_type = ["incremental","fulldump"]
 <%= f.select :table_name, [] %>
 <label>Incremental Column</label>
 <%= f.text_field :incremental_column %>
+<label>Incremental Column Type</label>
+<%= f.text_field :incremental_column_type %>
 <label>Maximum Incremental Value</label>
 <%= f.text_field :max_incremental_value %>
 <label>S3 Path</label>

data/app/views/myreplicator/exports/index.html.erb

@@ -12,6 +12,7 @@
        <th><%= sortable "destination_schema" %></th>
        <th><%= sortable "table_name" %></th>
        <th><%= sortable "incremental_column" %></th>
+       <th><%= sortable "incremental_column_type" %></th>
        <th><%= sortable "max_incremental_value" %></th>
        <th><%= sortable "export_to", "Export Desitination" %></th>
        <th><%= sortable "export_type" %></th> 
@@ -28,6 +29,7 @@
         <td><%= export.destination_schema %></td>
         <td><%= export.table_name %></td>
         <td><%= export.incremental_column %></td>
+        <td><%= export.incremental_column_type %></td>
         <td><%= export.max_incremental_value %></td>
         <td><%= export.export_to %></td>
         <td><%= export.export_type %></td> 
@@ -53,4 +55,4 @@ $(function(){
   $("span.status").tipTip();
   $("span.cron").tipTip();
 })
-</script>
+</script>

data/lib/exporter/mysql_exporter.rb

@@ -19,36 +19,21 @@ module Myreplicator
                             :filepath => filepath,
                             :incremental_col => @export_obj.incremental_column) do |metadata|
 
-        metadata.on_failure do |m|
-          update_export(:state => "failed", :export_finished_at => Time.now, :error => metadata.error)
-        end
-
         prepare metadata
 
-        if @export_obj.state == "new"
+        case @export_obj.export_type?
+        when :new
+          on_failure_state_trans(metadata, "new") # If failed, go back to new
+          on_export_success(metadata)
           initial_export metadata
-          metadata.on_success do |m|
-            metadata.state = "export_completed"
-            wrapup metadata
-          end
-
-        elsif !is_running?
-          # local max value for incremental export
-
-          max_value = incremental_export(metadata)
-          #max_value = incremental_export_into_outfile(metadata)
 
-          metadata.incremental_val = max_value # store max val in metadata
-
-          # Call back that updates the maximum value of incremental col
-          metadata.on_success do |m|
-            metadata.state = "export_completed"
-            wrapup metadata
-            @export_obj.update_max_val(max_value) #  update max value if export was successful
-          end
+        when :incremental
+          on_failure_state_trans(metadata, "failed") # Set state trans on failure
+          on_export_success(metadata)
+          incremental_export metadata
         end
         
-      end
+      end # metadata
     end
     
     ##
@@ -58,21 +43,7 @@ module Myreplicator
       ssh = @export_obj.ssh_to_source
       metadata.ssh = ssh
     end
-    
-    ##
-    # Throws ExportIgnored if the job is still running
-    # Checks the state of the job using PID and state
-    ##
-    def is_running?
-      return false if @export_obj.state != "exporting"
-      begin
-        Process.getpgid(@export_obj.exporter_pid)
-        raise Exceptions::ExportIgnored.new("Ignored")
-      rescue Errno::ESRCH
-        return false
-      end
-    end
-
+      
     def update_export *args
       options = args.extract_options!
       @export_obj.update_attributes! options
@@ -91,21 +62,29 @@ module Myreplicator
     ##
 
     def initial_export metadata
-      flags = ["create-options", "single-transaction"]       
-      cmd = SqlCommands.mysqldump(:db => @export_obj.source_schema,
-                                  :flags => flags,
-                                  :filepath => filepath,
-                                  :table_name => @export_obj.table_name)     
-      
       metadata.export_type = "initial"
+      max_value = @export_obj.max_value if @export_obj.incremental_export?
+      cmd = initial_mysqldump_cmd
 
-      update_export(:state => "exporting", :export_started_at => Time.now, :exporter_pid => Process.pid)
+      exporting_state_trans # mark exporting
 
       puts "Exporting..."
       result = execute_export(cmd, metadata)
+      
       check_result(result, 0)
+
+      @export_obj.update_max_val(max_value) if @export_obj.incremental_export?
     end
 
+    def initial_mysqldump_cmd
+      flags = ["create-options", "single-transaction"]       
+      cmd = SqlCommands.mysqldump(:db => @export_obj.source_schema,
+                                  :flags => flags,
+                                  :filepath => filepath,
+                                  :table_name => @export_obj.table_name)     
+      return cmd
+    end
+    
     ##
     # Exports table incrementally, using the incremental column specified
     # If column is not specified, it will export the entire table 
@@ -113,29 +92,35 @@ module Myreplicator
     ##
 
     def incremental_export metadata
-      max_value = @export_obj.max_value
-      @export_obj.update_max_val if @export_obj.max_incremental_value.blank?   
+      unless @export_obj.is_running?
+        max_value = @export_obj.max_value
+        metadata.export_type = "incremental"
+        @export_obj.update_max_val if @export_obj.max_incremental_value.blank?   
+        
+        cmd = incremental_export_cmd 
+        exporting_state_trans # mark exporting
+        puts "Exporting..."
+        result = execute_export(cmd, metadata)
+        check_result(result, 0)
+        metadata.incremental_val = max_value # store max val in metadata
+        @export_obj.update_max_val(max_value) # update max value if export was successful
+      end
+      return false
+    end
 
+    def incremental_export_cmd
       sql = SqlCommands.export_sql(:db => @export_obj.source_schema,
                                    :table => @export_obj.table_name,
                                    :incremental_col => @export_obj.incremental_column,
                                    :incremental_col_type => @export_obj.incremental_column_type,
                                    :incremental_val => @export_obj.max_incremental_value)      
-
+      
       cmd = SqlCommands.mysql_export(:db => @export_obj.source_schema,
                                      :filepath => filepath,
                                      :sql => sql)
-      
-      metadata.export_type = "incremental"
-      update_export(:state => "exporting", :export_started_at => Time.now, :exporter_pid => Process.pid)
-      puts "Exporting..."
-      result = execute_export(cmd, metadata)
-      check_result(result, 0)
-
-      return max_value
+      return cmd
     end
 
-
     ##
     # Exports table incrementally, similar to incremental_export method
     # Dumps file in tmp directory specified in myreplicator.yml
@@ -145,6 +130,8 @@ module Myreplicator
 
     def incremental_export_into_outfile metadata
       max_value = @export_obj.max_value
+      metadata.export_type = "incremental_outfile"
+
       @export_obj.update_max_val if @export_obj.max_incremental_value.blank?   
 
       cmd = SqlCommands.mysql_export_outfile(:db => @export_obj.source_schema,
@@ -153,27 +140,14 @@ module Myreplicator
                                              :incremental_col => @export_obj.incremental_column,
                                              :incremental_col_type => @export_obj.incremental_column_type,
                                              :incremental_val => @export_obj.max_incremental_value)      
-
-      metadata.export_type = "incremental_outfile"
-      update_export(:state => "exporting", :export_started_at => Time.now, :exporter_pid => Process.pid)
+      exporting_state_trans
       puts "Exporting..."
       result = execute_export(cmd, metadata)
       check_result(result, 0)
-
+      max_value = incremental_export_into_outfile(metadata)
       return max_value
     end
 
-    ##
-    # Completes an export process
-    # Zips files, updates states etc
-    ##
-    def wrapup metadata
-      puts "Zipping..."
-      zipfile(metadata)
-      update_export(:state => "export_completed", :export_finished_at => Time.now)
-      puts "Done.."
-    end
-
     ##
     # Checks the returned resut from SSH CMD
     # Size specifies if there should be any returned results or not
@@ -203,7 +177,9 @@ module Myreplicator
     ##
     def zipfile metadata
       cmd = "cd #{Myreplicator.configs[@export_obj.source_schema]["ssh_tmp_dir"]}; gzip #{@export_obj.filename}"
-      
+
+      puts cmd
+
       zip_result = metadata.ssh.exec!(cmd)
 
       unless zip_result.nil?        
@@ -214,6 +190,30 @@ module Myreplicator
 
       return zip_result
     end
+
+    def on_failure_state_trans metadata, state
+      metadata.on_failure do |m|
+        update_export(:state => state, 
+                      :export_finished_at => Time.now, 
+                      :error => metadata.error)
+      end
+    end
+
+    def exporting_state_trans
+      update_export(:state => "exporting", 
+                    :export_started_at => Time.now, 
+                    :exporter_pid => Process.pid)
+    end
+
+    def on_export_success metadata
+      metadata.on_success do |m|
+        update_export(:state => "export_completed", 
+                      :export_finished_at => Time.now, 
+                      :error => metadata.error)
+        metadata.state = "export_completed"
+        zipfile(metadata)
+      end
+    end
     
   end
 end

data/lib/loader/loader.rb

@@ -24,41 +24,38 @@ module Myreplicator
     # Kicks off all initial loads first and then all incrementals
     # Looks at metadata files stored locally
     # Note: Initials are loaded sequentially
-    # If there is a 
     ##
     def self.load
       initials = []
       incrementals = []
-      metadata = Loader.metadata_files
+      all_files = Loader.metadata_files
 
-      metadata.each do |m|
+      all_files.each do |m|
         if m.export_type == "initial"
           Kernel.p m
           initials << m # Add initial to the list
-          metadata.delete(m) # Delete obj from mixed list
+          all_files.delete(m) # Delete obj from mixed list
 
-          metadata.each do |md|
+          all_files.each do |md|
             if m.equals(md) && md.export_type == "incremental"
               initials << md # incremental should happen after the initial load
-              metadata.delete(md) # remove from current list of files
+              all_files.delete(md) # remove from current list of files
             end
           end
         end
       end
       
-      incrementals = metadata # Remaining are all incrementals
+      incrementals = all_files # Remaining are all incrementals
       
       initial_procs = Loader.initial_loads initials
-      Kernel.p initial_procs
       parallel_load initial_procs
 
       incremental_procs = Loader.incremental_loads incrementals
-      Kernel.p incremental_procs
       parallel_load incremental_procs
     end
     
     def self.parallel_load procs
-      p = Parallelizer.new(:klass => "Myreplicator::Transporter")
+      p = Parallelizer.new(:klass => "Myreplicator::Loader")
       procs.each do |proc|
         p.queue << {:params => [], :block => proc}
       end
@@ -127,7 +124,9 @@ module Myreplicator
 
       incrementals.each do |metadata|
         group = [metadata]
+        incrementals.delete(metadata)
 
+        # look for same loads
         incrementals.each do |md| 
           if metadata.equals(md)
             group << md
@@ -135,10 +134,8 @@ module Myreplicator
           end
         end
         
-        incrementals.delete(metadata)
         groups << group
       end
-
       return groups
     end
     

data/lib/myreplicator/version.rb

@@ -1,3 +1,3 @@
 module Myreplicator
-  VERSION = "0.1.1"
+  VERSION = "1.0.0"
 end

data/lib/transporter/transporter.rb

@@ -91,20 +91,42 @@ module Myreplicator
           json_local_path = File.join(tmp_dir,filename)
           puts "Downloading #{json_file}"
           sftp.download!(json_file, json_local_path)
-          dump_file = Transporter.get_dump_path(json_local_path)
-
-          Log.run(:job_type => "transporter", :name => "export_file",
-                  :file => dump_file, :export_id => export.id) do |log|
-            puts "Downloading #{dump_file}"
-            sftp.download!(dump_file, File.join(tmp_dir, dump_file.split("/").last))
-            # clear files
-            ssh.exec!("rm #{json_file}")
-            ssh.exec!("rm #{dump_file}")
-          end
+          metadata = Transporter.metadata_obj(json_local_path)
+          dump_file = metadata.export_path
+
+          if metadata.state == "export_completed"
+            Log.run(:job_type => "transporter", :name => "export_file",
+                    :file => dump_file, :export_id => export.id) do |log|
+              puts "Downloading #{dump_file}"
+              sftp.download!(dump_file, File.join(tmp_dir, dump_file.split("/").last))
+              Transporter.remove!(ssh, json_file, dump_file)
+            end
+          elsif Transporter.junk_file?(metadata)
+            Transporter.remove!(ssh, json_file, dump_file)
+          end #if
+
         end
       }
     end
 
+    ##
+    # Returns true if the file should be deleted
+    ##
+    def self.junk_file? metadata
+      case metadata.state
+      when "failed"
+        return true
+      when "ignored"
+        return true
+      end
+      return false
+    end
+
+    def self.remove! ssh, json_file, dump_file
+      ssh.exec!("rm #{json_file}")
+      ssh.exec!("rm #{dump_file}")           
+    end
+
     ##
     # Gets all files ready to be exported from server
     ##
@@ -118,11 +140,16 @@ module Myreplicator
       return []
     end
 
+    def self.metadata_obj json_path
+      metadata = ExportMetadata.new(:metadata_path => json_path)
+      return metadata
+    end
+
     ##
     # Reads metadata file for the export path
     ##
-    def self.get_dump_path json_path
-      metadata = ExportMetadata.new(:metadata_path => json_path)
+    def self.get_dump_path json_path, metadata = nil
+      metadata = Transporter.metadata_obj(json_path) if metadata.nil?
       return metadata.export_path
     end
 

data/test/dummy/log/development.log

@@ -9006,3 +9006,25 @@ Connecting to database specified by database.yml
 Connecting to database specified by database.yml
 Connecting to database specified by database.yml
 Connecting to database specified by database.yml
+Connecting to database specified by database.yml
+Connecting to database specified by database.yml
+DEPRECATION WARNING: Database connections will not be closed automatically, please close your
+database connection at the end of the thread by calling `close` on your
+connection.  For example: ActiveRecord::Base.connection.close
+. (called from mon_synchronize at /home/sasan/.rvm/rubies/ruby-1.9.3-p125/lib/ruby/1.9.1/monitor.rb:211)
+DEPRECATION WARNING: Database connections will not be closed automatically, please close your
+database connection at the end of the thread by calling `close` on your
+connection.  For example: ActiveRecord::Base.connection.close
+. (called from mon_synchronize at /home/sasan/.rvm/rubies/ruby-1.9.3-p125/lib/ruby/1.9.1/monitor.rb:211)
+DEPRECATION WARNING: Database connections will not be closed automatically, please close your
+database connection at the end of the thread by calling `close` on your
+connection.  For example: ActiveRecord::Base.connection.close
+. (called from mon_synchronize at /home/sasan/.rvm/rubies/ruby-1.9.3-p125/lib/ruby/1.9.1/monitor.rb:211)
+DEPRECATION WARNING: Database connections will not be closed automatically, please close your
+database connection at the end of the thread by calling `close` on your
+connection.  For example: ActiveRecord::Base.connection.close
+. (called from mon_synchronize at /home/sasan/.rvm/rubies/ruby-1.9.3-p125/lib/ruby/1.9.1/monitor.rb:211)
+Connecting to database specified by database.yml
+Connecting to database specified by database.yml
+Connecting to database specified by database.yml
+Connecting to database specified by database.yml

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: myreplicator
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 1.0.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-11 00:00:00.000000000 Z
+date: 2013-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -244,12 +244,6 @@ files:
 - test/dummy/app/assets/javascripts/application.js
 - test/dummy/app/assets/stylesheets/application.css
 - test/dummy/Rakefile
-- test/dummy/tmp/myreplicator/okl_test_batchy_batches_1354061910.tsv.gz
-- test/dummy/tmp/myreplicator/okl_test_batchy_batches_1357260317.tsv.json
-- test/dummy/tmp/myreplicator/okl_test_batchy_batches_1354061910.tsv.json
-- test/dummy/tmp/myreplicator/okl_test_batchy_batches_1357260317.tsv.gz
-- test/dummy/tmp/myreplicator/okl_test_batchy_batches_1357689827.tsv.gz
-- test/dummy/tmp/myreplicator/okl_test_batchy_batches_1357689827.tsv.json
 - test/dummy/tmp/cache/assets/DF8/5D0/sprockets%2Fb815ed34d61cfed96222daa3bfd1d84d
 - test/dummy/tmp/cache/assets/D3F/A00/sprockets%2F7a803404e1f60b8d672d763cb9ba8af5
 - test/dummy/tmp/cache/assets/C2E/D00/sprockets%2F667019818351638709494c01bddb5f68
@@ -314,7 +308,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4093979052196047912
+      hash: -2431171312280889434
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -323,7 +317,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4093979052196047912
+      hash: -2431171312280889434
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -376,12 +370,6 @@ test_files:
 - test/dummy/app/assets/javascripts/application.js
 - test/dummy/app/assets/stylesheets/application.css
 - test/dummy/Rakefile
-- test/dummy/tmp/myreplicator/okl_test_batchy_batches_1354061910.tsv.gz
-- test/dummy/tmp/myreplicator/okl_test_batchy_batches_1357260317.tsv.json
-- test/dummy/tmp/myreplicator/okl_test_batchy_batches_1354061910.tsv.json
-- test/dummy/tmp/myreplicator/okl_test_batchy_batches_1357260317.tsv.gz
-- test/dummy/tmp/myreplicator/okl_test_batchy_batches_1357689827.tsv.gz
-- test/dummy/tmp/myreplicator/okl_test_batchy_batches_1357689827.tsv.json
 - test/dummy/tmp/cache/assets/DF8/5D0/sprockets%2Fb815ed34d61cfed96222daa3bfd1d84d
 - test/dummy/tmp/cache/assets/D3F/A00/sprockets%2F7a803404e1f60b8d672d763cb9ba8af5
 - test/dummy/tmp/cache/assets/C2E/D00/sprockets%2F667019818351638709494c01bddb5f68

data/test/dummy/tmp/myreplicator/okl_test_batchy_batches_1354061910.tsv.json

@@ -1 +0,0 @@
-{"export_time":"2012-11-27T16:18:30-08:00","table":"batchy_batches","database":"okl_test","state":"export_completed","incremental_col":null,"incremental_val":null,"export_id":1,"filepath":"/home/ubuntu/myreplicator_tmp/okl_test_batchy_batches_1354061910.tsv","zipped":true,"export_type":"incremental"}

data/test/dummy/tmp/myreplicator/okl_test_batchy_batches_1357260317.tsv.json

@@ -1 +0,0 @@
-{"export_time":"2013-01-03T16:45:17-08:00","table":"batchy_batches","database":"okl_test","state":"export_completed","incremental_col":"started_at","incremental_val":null,"export_id":2,"filepath":"/home/ubuntu/myreplicator_tmp/okl_test_batchy_batches_1357260317.tsv","zipped":true,"export_type":"initial"}

data/test/dummy/tmp/myreplicator/okl_test_batchy_batches_1357689827.tsv.json

@@ -1 +0,0 @@
-{"export_time":"2013-01-08T16:03:47-08:00","table":"batchy_batches","database":"okl_test","state":"export_completed","incremental_col":"updated_at","incremental_val":"2012-11-13T10:25:18Z","export_id":1,"filepath":"/home/ubuntu/myreplicator_tmp/okl_test_batchy_batches_1357689827.tsv","zipped":true,"export_type":"incremental"}