rsynconrails 0.1.0

data/lib/cli_funcs_rsync.rb

@@ -0,0 +1,351 @@
+require 'system_config'
+require 'cli_funcs'
+
+class Rsync < CliFuncs
+  attr_accessor :flags_all, :cmd_run, :source, :destination
+  attr_reader :uptodate, :deleted, :modified, :created, :excluded, :ignored, :duplicates, :base_dir, :data_dir, :output, :transfer_stats
+
+  def initialize
+    super
+    @utility = CliUtils.new("rsync").utility_path
+    @uptodate = Array.new
+    @deleted = Array.new
+    @modified = Array.new
+    @created = Array.new
+    @excluded = Hash.new
+    @ignored = Array.new
+    @duplicates = Array.new
+    @transfer_stats = Hash.new
+    @source = Array.new
+    @destination = String
+    @output_filter_junk = Regexp
+    @output_filter_excluded = Regexp
+    @output_filter_warn_err = Regexp
+    @output_filter_stats = Regexp
+    @output_filter_duplicates = Regexp
+    @output_filter_created = Regexp
+    set_flags_base
+    set_output_filters
+  end
+
+  def rsync
+    run_and_capture(cmd_run)
+  end
+
+  def set_output_filters
+    set_output_filter_junk
+    set_output_filter_excluded
+    set_output_filter_warn_err
+    set_output_filter_stats
+    set_output_filter_duplicates
+    set_output_filter_created
+  end
+
+  def output_process
+    @output.each do |line|
+      if line.match @output_filter_junk
+        next
+      elsif line.match @output_filter_duplicates
+        puts "#{line.chomp} DUPLICATE!" if DEBUG
+        @duplicates.push(line)
+        next
+      elsif line.match @output_filter_excluded
+        puts "#{line.chomp} EXCLUDED!" if DEBUG
+        @excluded[$3] = $4
+        next
+      elsif line.match @output_filter_warn_err
+        # Capture warnings / errors here
+        next
+      elsif line.match @output_filter_stats
+        # Set hash of stats
+        @transfer_stats[$1] = $2
+        @transfer_stats[$3] = $4
+        @transfer_stats[$5] = $6
+        @transfer_stats[$7] = $8
+        @transfer_stats[$9] = $10
+        @transfer_stats[$11] = $12
+        @transfer_stats[$13] = $14
+        @transfer_stats[$15] = $16
+        @transfer_stats[$17] = $18
+        @transfer_stats[$19] = $20
+        @transfer_stats[$21] = $22
+        next
+      elsif line.match @output_filter_created
+        #@created.push(line)
+        next
+      else
+        # catch all, this is the main content
+        process_itemized(line)
+      end
+    end
+    puts @transfer_stats.inspect if DEBUG
+  end
+
+  def process_itemized(line)
+    # Break apart the line by spaces (e.g. ".f          9/file9")
+    attrs,item = line.split(/\s+/, 2)
+    # Break apart itemized attrs on each character 0 = . 1 = f 2 = nil, 3 = nil ...
+    attrs_p = attrs.split("")
+    # Begin check's for file/directory disposition according to rsync
+    # If element 0 contains a '.', it means no update has occurred, but may have attribute changes
+    if(attrs_p[0] == ".")
+       # This first check ignores directories which have had their timestamp changed
+       if(
+          attrs_p[1] == "d" and 
+          attrs_p[2] == "." and
+          attrs_p[3] == "." and
+          attrs_p[4] == "t" and
+          attrs_p[5] == "." and
+          attrs_p[6] == "." and
+          attrs_p[7] == "." and
+          attrs_p[8] == "." and
+          attrs_p[9] == "." and
+          attrs_p[10] == "."
+       )
+          puts "#{line.chomp} IGNORED!" if DEBUG
+          @ignored.push(line)
+          #return
+       # checks if nothing has changed with this item
+       elsif(
+          attrs_p[1] =~ /f|d|L|D|S/ and 
+          attrs_p[2] == nil and
+          attrs_p[3] == nil and
+          attrs_p[4] == nil and
+          attrs_p[5] == nil and
+          attrs_p[6] == nil and
+          attrs_p[7] == nil and
+          attrs_p[8] == nil and
+          attrs_p[9] == nil and
+          attrs_p[10] == nil
+       )
+          puts "#{line.chomp} UPTODATE!" if DEBUG
+          @uptodate.push(line)
+       # something must have changed, like an attribute (e.g. ownership or mode)
+       else
+          puts "#{line.chomp} MODIFIED OWNERSHIP OR MODE!" if DEBUG
+          @modified.push(line) 
+       end
+    elsif(attrs_p[0] =~ /\*|<|>|c|h/)
+      # checks if item is being deleted
+      if(
+         attrs_p[1] == "d" and 
+         attrs_p[2] == "e" and
+         attrs_p[3] == "l" and
+         attrs_p[4] == "e" and
+         attrs_p[5] == "t" and
+         attrs_p[6] == "i" and
+         attrs_p[7] == "n" and
+         attrs_p[8] == "g" and
+         attrs_p[9] == nil and
+         attrs_p[10] == nil
+      )
+         puts "#{line.chomp} DELETED!" if DEBUG
+         @deleted.push(line) 
+
+      # checks if item is being created (i.e. new file/dir)
+      elsif(
+        attrs_p[1] =~ /f|d/ and 
+        attrs_p[2] == "+" and
+        attrs_p[3] == "+" and
+        attrs_p[4] == "+" and
+        attrs_p[5] == "+" and
+        attrs_p[6] == "+" and
+        attrs_p[7] == "+" and
+        attrs_p[8] == "+" and
+        attrs_p[9] == "+" and
+        attrs_p[10] == "+"
+      )
+        puts "#{line.chomp} CREATED!" if DEBUG
+        @created.push(line)
+
+      # everthing else is considered a modification
+      else
+        puts "#{line.chomp} MODIFIED CATCH ALL 1!" if DEBUG
+        @modified.push(line) 
+      end
+    else
+      puts "#{line.chomp} MODIFIED CATCH ALL 2!" if DEBUG
+      @modified.push(line) 
+    end
+  end
+
+  def set_output_filter_warn_err
+    filter = Array.new
+
+    # These are warnings and errors kicked out by rsync during it's run
+    filter.push("WARNING: .* failed verification -- update discarded \(will try again\)\.")
+    filter.push("IO error encountered -- skipping file deletion")
+    filter.push("file has vanished: .*")
+    filter.push("rsync (error|warning): .*")
+    filter.push("cannot delete non-empty directory: .*")
+
+    @output_filter_warn_err = /#{filter.join("|")}/
+  end
+  
+  def set_output_filter_created
+    filter = Array.new
+
+    # These should be rare, for some reason it doesn't show up in the itemized list
+    # and rsync doesn't count it as a 'created file', it seems like the only time 
+    # this happens is if the destination root directory doesn't already exist.
+    filter.push("^created directory .*")
+
+    @output_filter_created = /#{filter[0]}/
+  end
+
+  def set_output_filter_stats
+    filter = Array.new
+
+    # These are for rsync stats which are output after the transfer is complete
+    filter.push("(Number of files): (\\d+)")
+    filter.push("(Number of files transferred): (\\d+)")
+    filter.push("(Total file size): (\\d+) bytes")
+    filter.push("(Total transferred file size): (\\d+) bytes")
+    filter.push("(Literal data): (\\d+) bytes")
+    filter.push("(Matched data): (\\d+) bytes")
+    filter.push("(File list size): (\\d+)")
+    filter.push("(File list generation time): (.*) seconds")
+    filter.push("(File list transfer time): (.*) seconds")
+    filter.push("(Total bytes sent): (\\d+)")
+    filter.push("(Total bytes received): (\\d+)")
+
+    @output_filter_stats = /#{filter.join("|")}/
+  end
+
+  def set_output_filter_excluded
+    # These are files/directories which have been excluded by a pattern we passed to rsync
+    @output_filter_excluded = /^\[generator\] (excluding|protecting) (file|directory) (.*) because of pattern (.*)$/
+  end
+    
+  def set_output_filter_duplicates
+    filter = Array.new
+
+    # These are file/directories which are the same in the sources list
+    filter.push("^removing duplicate name .* from file list .*")
+
+    @output_filter_duplicates = /#{filter[0]}/
+  end
+
+  def set_output_filter_junk
+    filter = Array.new
+
+    # blank line
+    filter.push("^$")
+    # ignore line
+    filter.push("^sending incremental file list")
+    # ignore line
+    filter.push("^building file list ...")
+    # ignore line
+    filter.push("^expand file_list\s\w+")
+    # ignore line
+    filter.push("^rsync: expand\s\w+")
+    # ignore line
+    filter.push("^opening connection\s\w+")
+    # ignore line - should probably capture this info
+    filter.push("^total")
+    # ignore line - should probably capture this info
+    filter.push("^wrote")
+    # ignore line - should probably capture this info
+    filter.push("^sent")
+    # ignore line
+    filter.push("^done")
+    # ignore line
+    filter.push("^excluding")
+    # ignore line
+    filter.push("^hiding")
+    # ignore line
+    filter.push("^delta( |-)transmission (dis|en)abled")
+    # ignore line
+    filter.push("^deleting in \./")
+    # ignore line
+    filter.push("^opening connection using: ssh")
+    # ignore line
+    filter.push("^rsync\\[\\d+\\] \\(sender\\) heap statistics:")
+    # ignore line
+    filter.push("^rsync\\[\\d+\\] \\(server receiver\\) heap statistics:")
+    # ignore line
+    filter.push("^rsync\\[\\d+\\] \\(server generator\\) heap statistics:")
+    # ignore line
+    filter.push("^  arena:")
+    # ignore line
+    filter.push("^  ordblks:")
+    # ignore line
+    filter.push("^  smblks:")
+    # ignore line
+    filter.push("^  hblks:")
+    # ignore line
+    filter.push("^  hblkhd:")
+    # ignore line
+    filter.push("^  allmem:")
+    # ignore line
+    filter.push("^  usmblks:")
+    # ignore line
+    filter.push("^  fsmblks:")
+    # ignore line
+    filter.push("^  uordblks:")
+    # ignore line
+    filter.push("^  fordblks:")
+    # ignore line
+    filter.push("^  keepcost:")
+
+    @output_filter_junk = /#{filter.join("|")}/
+  end
+
+  def cmd_run
+    u = CliUtils.new(@utility)
+    puts [u.utility_path, flags_run, @source, @destination].flatten.inspect if DEBUG
+    [u.utility_path, flags_run, @source, @destination].flatten
+  end
+  
+  def flag_delete
+    flag_add("--delete")
+  end
+
+  def flag_compress
+    flag_add("-z")
+  end
+
+  def flag_dryrun 
+    flag_add("-n")
+  end
+
+  def flag_verbose
+    flag_add("-vv")
+  end
+
+  def flag_archive
+    flag_add("-a")
+  end
+
+  def flag_itemized
+    flag_add("-i")
+  end
+
+  def flag_checksum
+    flag_add("-c")
+  end
+  
+  def flag_stats
+    flag_add("--stats")
+  end
+
+  def flag_bwlimit(kbps)
+    flag_add("--bwlimit=#{kbps}")
+  end
+
+  def flag_rsync_path(path)
+    flag_add("--rsync-path=#{path}")
+  end
+
+  def flag_exclude(pattern)
+    flag_add("--exclude=#{pattern[1..-1]}")
+  end
+
+  def set_flags_base
+    flag_archive
+    flag_verbose
+    flag_itemized
+    flag_delete
+    flag_stats
+  end
+end

data/lib/cli_utils.rb

@@ -0,0 +1,70 @@
+require 'open3'
+
+class CliUtils
+  attr_accessor :utility, :utility_path
+
+  def initialize(utility)
+    @utility = utility
+    @utility_path
+    @version
+    valid?
+  end
+
+  def valid?
+    find_utility
+    get_version
+    supp_util_versions
+  end
+
+  def find_utility
+    cmd = "which #{@utility}"
+    ph = IO.popen(cmd)
+    output = ph.readlines()
+    ph.close
+    if($? != 0)
+      puts "Cannot find required utility #{@utility} in path."
+      false
+    else
+      @utility_path = output[0].chomp
+    end
+  end
+
+  def get_version
+    version = Array.new
+    if(@utility_path == nil)
+      find_utility
+    end
+    cmd = "#{@utility_path} --version"
+    Open3.popen2e(cmd) { |i,oe,t|
+      oe.each do |line|
+        version.push(line)
+      end
+    }
+
+    @version = version.to_s
+  end
+
+  def supp_util_versions
+    case @utility
+    when "rsync"
+      supp_version = 3
+      version_pattern.match(@version)
+      if($1.to_s < supp_version.to_s)
+        puts "Unsupported version of 'rsync', use version #{supp_version} or higher."
+        false 
+      else
+        true
+      end
+    end
+  end
+
+  def version_pattern
+    case @utility
+    when "rsync"
+      Regexp.new(/rsync  version ([\d\.]+)  protocol version (\d+)/)
+    else
+      puts "Not a supported utility: #{utility}"
+      exit 1
+    end
+  end
+end

data/lib/host.rb

@@ -0,0 +1,194 @@
+require 'system_config'
+require 'find'
+
+class Host
+  attr_accessor :host, :domain, :host_yml, :autocreate
+  attr_reader :hosts_dir, :host_dir, :found_host, :host_yml_values
+
+  def initialize(hostname="")
+    @autocreate = false
+    @hostname = hostname
+    @host = ""
+    @domain = ""
+    @host_dir = ""
+    @host_yml = ""
+    @found_host = ""
+    @host_yml_values = Hash.new
+    
+    @hosts_dir = SYSTEM_CONFIG['hosts_dir']
+    prepare_hosts_dir
+    set_hostname_domain
+  end
+
+  def set_hostname_domain
+    if @hostname.include? "."
+      @host = @hostname.split(".").shift
+      @domain = @hostname.split(".")[1..-1].join(".")
+    else
+      @host = @hostname
+      @domain = ""
+    end
+  end
+
+  def find_host
+    begin
+        Find.find(@hosts_dir) do |path|
+          # These are the 3 things which define a 'host' in the system, 2 if there's no domain
+          if path =~ /#{@domain}/ and path =~ /#{@host}$/ and File.directory? "#{path}/overrides"
+            @host_dir = path  
+            @host_yml = "#{@host_dir}/host.yml"
+            # Since this is a path with more than just the domain and hostname in it,
+            # we need to test if this is a hostname without a domain in front. If so
+            # we need to set @found_host differently. We test to see if the full path
+            # minus the hostname is equal to the '@hosts_dir' because someone might
+            # name create a host without it having a domain component.
+            if "/#{path.split('/')[1..-2].join('/')}" == @hosts_dir
+              # If we get here, it means this path did not have a domain component
+              @found_host = "#{path.split('/')[-1]}"
+            else
+              # This path did have a domain component
+              @found_host = "#{path.split('/')[-2]}/#{path.split('/')[-1]}"
+            end
+            # This should set @host_yml_values
+            unless host_valid?
+              return false
+            end
+          else
+            prepare_host
+          end
+        end
+    rescue Exception => e
+      puts "Tried to find host: #{@hostname} in #{@hosts_dir} during Host.find_host, received exception #{e}"
+      exit 1
+    end
+  end
+
+  def prepare_host
+    if @autocreate
+      @host_dir = "#{@hosts_dir}/#{@domain}/#{@host}"
+      @host_yml = "#{@host_dir}/host.yml"
+      # This should set @host_yml_values
+      unless host_valid? 
+        return false 
+      end
+    end
+  end
+
+  def host_valid?
+    if host_dir?
+      if host_yml?
+        return true
+      else
+        puts "Error: @host_yml: #{@host_yml} isn't valid!"
+        return false
+      end
+    else
+      puts "Error: @host_dir: #{@host_dir} does not exist!"
+      return false
+    end
+  end
+
+  def host_yml?
+    unless File.exists? @host_yml then
+      prepare_host_yml
+      yml_valid?
+    else
+      yml_valid?
+    end
+  end
+
+  def prepare_host_yml
+    begin
+      f = open(@host_yml, 'w+')
+      f.puts "config:"
+      f.puts "  package_base: test_dist"
+      f.puts "  release_tag: current"
+      f.puts "  rsync_path:"
+      f.puts "  ssh_port:"
+      f.puts "  session_mode:"
+      f.puts "include:"
+      f.puts "  # - section/packagename"
+      f.puts "exclude:"
+      f.puts "  # - /somefile"
+      f.puts "exclude_backup:"
+      f.puts "  # - /somefile"
+      f.puts "execute:"
+      f.puts "  # - some arbitrary command"
+      f.close
+    rescue Exception => e
+      puts "Tried to open #{@host_yml} during 'Host.prepare_host_yml', received exception: #{e}"
+      false
+    end
+  end
+
+  def yml_valid?
+    begin
+      @host_yml_values = YAML.load_file(@host_yml)
+      
+      valid_keys = ['config','include','exclude','exclude_backup','execute']
+      config_valid_subkeys = ['package_base','release_tag','rsync_path','ssh_port','session_mode']
+      
+      # Begin check for basic elements
+      @host_yml_values.each_pair do |key,val|
+        if key == 'config' and val.class == Hash
+          val.each_pair do |skey,sval|
+            unless config_valid_subkeys.include? skey
+              puts "Invalid sub-option for 'config:' => #{skey} detected in #{@host_yml}.  Exiting..."
+              return false
+            end
+          end
+        else
+          unless valid_keys.include? key
+            puts "Invalid option => #{key} detected in #{@host_yml}.  Exiting..."
+            return false
+          end
+        end
+      end
+    rescue Exception => e
+      puts "Tried to load #{@host_yml} during 'Host.yml_valid?', received exception: #{e}"
+    end
+  end
+
+  def host_dir?
+    if File.directory? @host_dir
+      true
+    else
+      prepare_host_dir
+    end
+  end
+  
+  def hosts_dir?
+    if File.directory? @hosts_dir
+      true
+    else
+      prepare_hosts_dir
+    end
+  end
+
+  def prompt_host_dir
+    # This isn't used right now
+    print "Configuration directory doesn't exist for #{@host}, create? (N/y): "
+    input = gets.chomp
+    if input =~ /[yY]/ then true else exit end
+  end
+
+  def prepare_host_dir
+    begin
+      FileUtils.mkdir_p "#{@host_dir}/overrides"
+      true
+    rescue Exception => e
+      puts "Tried to create #{@host_dir} during 'Host.prepare_host_dir', received exception: #{e}"
+      false
+    end
+  end
+  
+  def prepare_hosts_dir
+    begin
+      FileUtils.mkdir_p @hosts_dir
+      true
+    rescue Exception => e
+      puts "Tried to create #{@hosts_dir} during 'Host.prepare_hosts_dir', received exception: #{e}"
+      false
+    end
+  end
+end

data/lib/package.rb

@@ -0,0 +1,66 @@
+require 'system_config'
+require 'find'
+
+class Package
+  attr_accessor :package, :package_yml, :package_base, :package_release_tag, :package_dir
+  attr_reader :packages_dir
+
+  def initialize(package="")
+    @package = package
+    @package_base = ""
+    @package_release_tag = ""
+    @package_yml = ""
+    @package_dir = ""
+    
+    @packages_dir = SYSTEM_CONFIG['packages_dir']
+  end
+
+  def set_properties(obj)
+    @package_base = obj.host_yml_values['config']['package_base']
+    @package_release_tag = obj.host_yml_values['config']['release_tag']
+    @package_dir = "#{@packages_dir}/#{@package_base}/#{package}/#{@package_release_tag}"
+    @package_yml = "#{@package_dir}/package.yml"
+  end
+
+  def package_valid?
+    if package_dir?
+      if package_yml?
+        return true
+      else
+        puts "Error: @package_yml: #{@package_yml} isn't valid!"
+        return false
+      end
+    else
+      puts "Error: @package_dir: #{@package_dir} does not exist!"
+      return false
+    end
+  end
+
+  def package_yml?
+    File.exists? @package_yml
+    yml_valid?
+  end
+
+  def yml_valid?
+    begin
+      package_yml = YAML.load_file(@package_yml)
+      
+      valid_keys = ['rank','include','exclude','exclude_backup','execute']
+      
+      # Begin check for basic elements
+      package_yml.each_pair do |key,val|
+        unless valid_keys.include? key
+          puts "Invalid option => #{key} detected in #{@package_yml}.  Exiting..."
+          return false
+        end
+      end
+    rescue Exception => e
+      puts "Tried to load #{@package_yml} during 'Package.yml_valid?', received exception: #{e}"
+    end
+  end
+
+  def package_dir?
+    # Package isn't valid unless it has a 'files' directory under it
+    File.directory? "#{@package_dir}/files"
+  end
+end

data/lib/ssh_funcs.rb

@@ -0,0 +1,29 @@
+require 'system_config'
+require 'net/ssh'
+
+class SSHFuncs
+  attr_reader :output
+  attr_accessor :hostname, :remote_cmd
+
+  def initialize(hostname)
+    @hostname = hostname
+    @output
+    @run_cmd
+    @ssh
+    ssh_channel
+  end
+
+  def ssh_channel
+    begin
+      @ssh = Net::SSH.start(@hostname, 'root')
+    rescue Exception => e
+      puts "Couldn't establish SSH session with #{@hostname}, received exception: #{e}"
+    end
+  end
+
+  def run_cmd(cmd)
+    # capture all stderr and stdout output from a remote process
+    @output = @ssh.exec!(cmd)
+    puts "SSH CMD OUTPUT: #{@output}" if DEBUG
+  end
+end