ksync 0.5.0 → 0.5.1

data/README

@@ -1,31 +1,49 @@
+=KSync for Ruby/Jruby
+==Description
 This gem is a simple folder synchroniser between a changing source folder and a
 'backup' destination folder (presumed to be unchanged). It uses a list of files hash which is a
 hash containing the filepath as the key and file information [FileSize, FileMTime] as value.
 
 In order to compare the source and destination folders:
-  a. it always creates a list of files hash for the source
-  b. it uses an existing list of files hash for the destination or creates one if non existing
-  c. It compares the 2 lists of files hash and copies/deletes from destination accordingly.
-  d. It compares files based on size and then on last modification date. It may optionally use
-     hash calculation (inactive by default because it is slow)
+* It always creates a list of files hash for the source
+* It uses an existing list of files hash for the destination or creates one if it does not exist
+* It compares the 2 lists of files hash and copies/deletes from destination accordingly.
+* It compares files based on size and then on last modification date. It may optionally uses hash calculation (inactive by default because it is slow)
 
-NOTE: If any files in the destination folder have changed and the files hash exists in the
-destination and the -u option (use hash) is not specified, then these changes will NOT be detected.
-This is a fast backup solution and presumes that the destination never changes.
+The gem comes with a frontend, 'ksync' which can be used directly to invoke the main class.
 
-The gem comes with a frontend, 'ksync' which can be used directly to invoke the object
+==Options
+Following options may be used:
+  # -d or --dry_run         : Do a dry run (nothing will be changed) (default : do the real copy)
+  # -v or --verbosity=value : The level of verbosity (1..3) (default = 0 : very silent)
+  # -u or --use_hash        : Use hash based calculation of files differences (default : dont use hash)
+  # -f or --force_dest_hash : Force recalculation of files hash in destination folder (default : use existing files hash).
+  #                           Use this option to still take into account changes done on destination
+  # -h or --help            : Display help message
+==Examples
+To backup folder c:/dev to c:/dev_backup with default options:
 
-Examples of usage:
+  ksync c:/dev c:/dev_backup
+
+The above, but inside rour ruby code:
+
+  require 'ksync'
+  KSync::Base.new({:src => 'c:/dev', :dst => 'c:/dev_backup'}).do_sync
 
-To backup folder c:/dev to c:/dev_backup with default options :
-> ksync c:/dev c:/dev_backup
-The above, but inside rour ruby code :
->  require 'ksync'
->  KSync.new({:src => 'c:/dev', :dst => 'c:/dev_backup'}).do_sync
 the method do_sync will return false if there were no changes, true otherwise
 
 To backup folder c:/dev to c:/dev_backup forcing hash calculation (hash calculation will only be used if the files have
 the same size and modification date):
-> ksync --use_hash c:/dev c:/dev_backup
+  ksync --use_hash c:/dev c:/dev_backup
 or
-> ksync -u c:/dev c:/dev_backup
+  ksync -u c:/dev c:/dev_backup
+
+==Tests
+To run the tests from the gem folder:
+  rake test
+
+==Notes
+If any files in the destination folder have changed then there is no guarantee that the changes will be detected.
+This is a fast backup solution and presumes that the destination never changes. However, in order to bypass this
+apparent limitation, the option -f or --force_dest_hash may be used. In this case, the destination folder files hash
+will be recreated and so will reflect the changes on the destination. The syncing then will take place normally.

data/bin/ksync

@@ -17,7 +17,8 @@ o.banner = "Usage: #{$0} [options] source_folder destination_folder"
 o.on('-d', '--dry_run', 'dry run (default : do the real copy - no dry run)') { |s| options[:real_copy] = false }
 o.on('-v', '--verbosity=value', 'The level of verbosity (1..3) (default = 0 : very silent)') { |s| options[:verbose] = s.to_i }
 o.on('-u', '--use_hash', 'use hash calculation (default : dont use hash)') { |s| options[:use_hash] = true }
-o.on('-h', '--help') { message(o) }
+o.on('-f', '--force_dest_hash', 'force recalculation of files hash in destination folder') { |s| options[:force_dest_hash] = true }
+o.on('-h', '--help', 'Display this message') { message(o) }
 
 o.parse!
 
@@ -26,5 +27,5 @@ if !ARGV[0] || !ARGV[1]
 else
     options[:src] =  ARGV[0]
     options[:dst] =  ARGV[1]
-    KSync.new(options).do_sync
+    KSync::Base.new(options).do_sync
 end

data/lib/ksync.rb

@@ -1,159 +1,218 @@
 require 'fileutils'
 require 'digest'
 require 'find'
+require 'yaml'
 
-class KSync
-
-  SSIZE_I=0
-  MTIME_I=1
-
-  attr_accessor :all_h, :ll
-  def initialize(init_opt)
-    @all_h={}
-    @changes = false
-    init_opt[:real_copy] = init_opt[:real_copy] == nil ? true : init_opt[:real_copy]
-    init_opt[:use_hash] = init_opt[:use_hash] == nil ? false : init_opt[:use_hash]
-    init_opt[:verbose] = init_opt[:verbose] == nil ? 0 : init_opt[:verbose]
-    return nil if !init_opt[:src] | !init_opt[:dst]
-    @options = init_opt
-    @ll = [init_opt[:src], init_opt[:dst]]
-    create_hash    
-  end
+# this is the main KSync module
+module KSync
+  # this is the only class in the KSync module, needs to be instantiated
+  class Base
 
-  def do_sync    
-    src=0    
-    dst=1    
-    # browse source hash to detect new and changed files
-    return false unless File.exists?(ll.first)
-    @all_h[ll[src]].each do |k,v|
-      src_fp = File.join(@ll[src],k)
-      dst_fp = File.join(@ll[dst],k)
-      if !@all_h[@ll[dst]][k]
-        puts "N: #{k}" if @options[:verbose] > 0
-        do_copy(src_fp, dst_fp, v) if @options[:real_copy]
-        @changes = true
-      else
-        if !File.directory?(File.join(@ll[src],k))
-          if !same?(src_fp, dst_fp, v, all_h[@ll[dst]][k])
-            puts "C: #{k}" if @options[:verbose] > 0
-            do_copy(src_fp, dst_fp, v) if @options[:real_copy]
-            @changes = true
+    # index for size of file
+    SSIZE_I=0
+    # index for modification date of file
+    MTIME_I=1
+
+    # the name of the file containing the hash for the destination folder
+    KSYNC_FILES_HASH = '.ksync_files_hash'
+
+    # the version of the ksync gem
+    VERSION = '0.5.1'
+
+    attr_accessor :all_h, :ll
+    # initializes the class
+    def initialize(init_opt)
+      @all_h={}
+      @changes = false
+      init_opt[:real_copy] = init_opt[:real_copy] == nil ? true : init_opt[:real_copy]
+      init_opt[:use_hash] = init_opt[:use_hash] == nil ? false : init_opt[:use_hash]
+      init_opt[:verbose] = init_opt[:verbose] == nil ? 0 : init_opt[:verbose]
+      init_opt[:force_dest_hash] = init_opt[:force_dest_hash] == nil ? false : init_opt[:force_dest_hash]
+
+      return nil if !init_opt[:src] | !init_opt[:dst]
+      @options = init_opt
+      @ll = [init_opt[:src], init_opt[:dst]]
+      create_hash
+    end
+
+    # this method does the actual syncing : frontend
+    def do_sync
+      # browse source hash to detect new and changed files
+      return false unless File.exists?(ll.first)
+      all_h[ll.first].each do |k,v|
+        src_fp = File.join(ll.first,k)
+        dst_fp = File.join(ll.last,k)
+        if !all_h[ll.last][k]
+          copy_del_actions('New', k, src_fp, dst_fp)
+        else
+          if !File.directory?(File.join(ll.first,k))
+            copy_del_actions('Changed',k, src_fp, dst_fp) if !same?(src_fp, dst_fp, v, all_h[ll.last][k])
           end
-        end   
-      end   
-    end  
-    # browse destination hash to detect obsolete files
-    @all_h[ll[dst]].each do |k,v|
-      if !all_h[ll[src]][k] 
-        dst_fp = File.join(ll[dst],k)
-        puts "D: #{dst_fp}" if @options[:verbose] > 0
-        do_delete dst_fp if @options[:real_copy]
-        @changes = true
-      end  
-    end  
-    #puts "there were changes" if @changes
-    #puts "nothing changed" if !@changes
-    return @changes
-  end  
-  
-  def get_hash(in_file)    
-    if !File.exists?(in_file)
-      puts "ERR: inexistent file : #{in_file}"
-      return nil
-    end  
-    sha1 = Digest::SHA1.new
-    begin
-      File.open(in_file) do |file|
-        buffer = ''
-        # Read the file 512 bytes at a time
-        while not file.eof
-          file.read(512, buffer)
-          sha1.update(buffer)
         end
       end
-    rescue 
-      puts "ERR: while calculating hash for : #{in_file}: (#{$!})"
-      return nil
-    end  
-    return sha1.to_s
-  end 
-
-  def do_copy(src_fp, dst_fp, src)
-    begin       
-      if File.directory?(src_fp)
-        file_op = "mkdir"
-        FileUtils.mkdir_p dst_fp 
-      else
-        file_op = "cp"
-        FileUtils.copy_file src_fp, dst_fp        
+      # browse destination hash to detect obsolete files
+      all_h[ll.last].each do |k,v|
+        #next if File.join(ll.last,KSYNC_FILES_HASH) == f  # ignore files list hash
+
+        if !all_h[ll.first][k]
+          dst_fp = File.join(ll.last,k)
+          copy_del_actions('Deleted', k, nil, dst_fp )
+        end
       end
-    rescue 
-      puts "ERR: while copying (#{file_op}) [#{src_fp} to #{dst_fp}] : (#{$!})"
-    end    
-  end
+      #puts "there were changes" if @changes
+      #puts "nothing changed" if !@changes
+      save_files_hash(File.join(ll.last, KSYNC_FILES_HASH)) if @changes and @options[:real_copy] == true
+      return @changes
+    end
 
-  def do_delete (dst_fp)
-    begin 
-      if File.exists?(dst_fp) 
-        if File.directory?(dst_fp)
-          file_op = 'remove_dir'
-          FileUtils.remove_dir(dst_fp)
+    # saves the files hash to disk
+    def save_files_hash(file)
+      File.open(file, 'w'){|f| f.write all_h[ll.last].to_yaml }
+    end
+
+    #loads the files hash from disk
+    def load_files_hash(file)
+      return YAML::load_file(file)
+    end
+
+    # calculate a hash of a file
+    def get_hash(in_file)
+      if !File.exists?(in_file)
+        puts "ERR: inexistent file : #{in_file}"
+        return nil
+      end
+      sha1 = Digest::SHA1.new
+      begin
+        File.open(in_file) do |file|
+          buffer = ''
+          # Read the file 512 bytes at a time
+          while not file.eof
+            file.read(512, buffer)
+            sha1.update(buffer)
+          end
+        end
+      rescue
+        puts "ERR: while calculating hash for : #{in_file}: (#{$!})"
+        return nil
+      end
+      return sha1.to_s
+    end
+
+    # does the actual copy between source and destination
+    def do_copy(src_fp, dst_fp)
+      begin
+        if File.directory?(src_fp)
+          file_op = "mkdir"
+          FileUtils.mkdir_p dst_fp
         else
-          file_op = 'rm'
-          FileUtils.rm dst_fp
-        end          
-      else
-        puts "ERR: inexistent file/folder : #{dst_fp}"
-      end      
-    rescue 
-      puts "ERR: (#{file_op}) for #{dst_fp} : (#{$!})"
+          file_op = "cp"
+          FileUtils.copy_file src_fp, dst_fp
+        end
+      rescue
+        puts "ERR: while copying (#{file_op}) [#{src_fp} to #{dst_fp}] : (#{$!})"
+      end
     end
-  end
-  
-  # if hash => also check hash if sizes are same
-  def same?(src_fp, dst_fp, src, dst)
-    size_ok = false
-    date_ok = false   
-    if src[SSIZE_I] == dst[SSIZE_I] # sizes are the same
-      size_ok = true
-      if dst[MTIME_I] - dst[MTIME_I]  >=  0                
-        date_ok = true 
+
+    # deletes file from destination
+    def do_delete (dst_fp)
+      begin
+        if File.exists?(dst_fp)
+          if File.directory?(dst_fp)
+            file_op = 'remove_dir'
+            FileUtils.remove_dir(dst_fp)
+          else
+            file_op = 'rm'
+            FileUtils.rm dst_fp
+          end
+        else
+          puts "ERR: inexistent file/folder : #{dst_fp}"
+        end
+      rescue
+        puts "ERR: (#{file_op}) for #{dst_fp} : (#{$!})"
+      end
+    end
+
+    # if hash => also check hash if sizes are same
+    def same?(src_fp, dst_fp, src, dst)
+      size_ok = false
+      date_ok = false
+      if src[SSIZE_I] == dst[SSIZE_I] # sizes are the same
+        size_ok = true
+        if dst[MTIME_I] - dst[MTIME_I]  >=  0
+          date_ok = true
+        else
+          puts "DATE diff for #{src_fp} vs #{dst_fp}" if @options[:verbose] > 1
+        end
       else
-        puts "DATE diff for #{src_fp} vs #{dst_fp}" if @options[:verbose] > 1
-      end  
-    else
-      puts "SIZE diff for #{src_fp} vs #{dst_fp}" if @options[:verbose] > 1
-    end    
-    return false if !size_ok 
-    return false if !date_ok
-    return true if !@options[:use_hash]
-    h1 = get_hash(src_fp)
-    h2 = get_hash(dst_fp)
-    if h1 != h2
-      puts "SHA1 diff for #{src_fp} vs #{dst_fp}" if @options[:verbose] > 1
-      return false
+        puts "SIZE diff for #{src_fp} vs #{dst_fp}" if @options[:verbose] > 1
+      end
+      return false if !size_ok
+      return false if !date_ok
+      return true if !@options[:use_hash]
+      h1 = get_hash(src_fp)
+      h2 = get_hash(dst_fp)
+      if h1 != h2
+        puts "SHA1 diff for #{src_fp} vs #{dst_fp}" if @options[:verbose] > 1
+        return false
+      end
+      return true
     end
-    return true
-  end
-  def create_hash
-    if !File.exists?(@ll.first)
-      puts "source folder #{@ll.first} does not exist ! "
-      return
+
+    #creates the initial file list hash for the source and potentially the destination
+    def create_hash
+      if !File.exists?(ll.first)
+        puts "source folder #{ll.first} does not exist ! "
+        return
+      end
+      ll.each do |prefix|
+        all_h[prefix] = {}
+        if prefix == ll.last
+          next if @options[:real_copy] == false
+          dest_hash_file_name = File.join(ll.last,KSYNC_FILES_HASH)
+          if File.exists?(dest_hash_file_name) && @options[:force_dest_hash] == false
+            all_h[prefix] = load_files_hash(dest_hash_file_name)
+            next unless @options[:force_dest_hash]
+          end
+        end
+
+        cnt = 0
+        if !File.exists?(prefix)
+          puts "creating folder #{prefix}" if @options[:verbose] > 1
+          FileUtils.mkdir_p prefix
+        end
+
+        Find.find(prefix).each do |f|
+          next if prefix == f
+          ff = f.unpack('U*').pack('U*')  # to get rid of funny encoding related errors
+          all_h[prefix][ff.split(prefix)[1]] = [File.size(ff), File.mtime(ff)]
+          puts "(#{ff})" if @options[:verbose] > 2
+          puts cnt if cnt % 100 == 0 && @options[:verbose] > 1
+          cnt += 1
+        end
+
+        save_files_hash(File.join(ll.last, KSYNC_FILES_HASH)) if prefix == ll.last
+
+      end
     end
-    @ll.each do |prefix|
-      @all_h[prefix] = {}
-      next if prefix == @ll.last && @options[:real_copy] == false
-      puts "creating hash for #{prefix}" if @options[:verbose] > 1
-      cnt = 0 
-      FileUtils.mkdir_p prefix if !File.exists?(prefix)
-      Find.find(prefix).each do |f|
-        next if prefix == f
-        ff = f.unpack('U*').pack('U*')  # to get rid of funny encoding related errors
-        @all_h[prefix][ff.split(prefix)[1]] = [File.size(ff), File.mtime(ff)]
-        puts "(#{ff})" if @options[:verbose] > 2
-        puts cnt if cnt % 100 == 0 && @options[:verbose] > 2
-        cnt += 1
+
+    private
+
+    def copy_del_actions(action, k, src_fp, dst_fp)
+      puts "#{action}: #{k}" if @options[:verbose] > 0
+      if action == 'Deleted'
+        if @options[:real_copy]
+          do_delete dst_fp
+          all_h[ll.last].delete(k)
+        end
+      else
+        if @options[:real_copy]
+          do_copy(src_fp, dst_fp) if @options[:real_copy]
+          all_h[ll.last][k] = [File.size(dst_fp), File.mtime(dst_fp)]
+        end
       end
+      @changes = true unless @changes
     end
+
+
   end
 end

data/test/test_ksync.rb

@@ -1,4 +1,4 @@
-gem 'minitest', '>= 4.4.0' # use the gem version, not the 1.9 bundled version of minitest
+gem 'minitest', '~> 4.4' # use the gem version, not the 1.9 bundled version of minitest
 require 'minitest/autorun'
 
 require 'ksync'
@@ -9,7 +9,8 @@ require 'turn'
 Turn.config.format = :outline
 
 class KSyncTest < MiniTest::Unit::TestCase
-  def setup
+    include KSync
+    def setup
     begin
       @src_path = File.join(File.dirname(__FILE__), "src/folder1/folder2")
       @dst_path = File.join(File.dirname(__FILE__), "src")
@@ -19,7 +20,8 @@ class KSyncTest < MiniTest::Unit::TestCase
     rescue
       puts "error trying to do the setup :#{$!}"
     end
-  end
+    end
+
   def teardown
    FileUtils.rm_rf  @opts[:src]
    FileUtils.rm_rf  @opts[:dst]
@@ -30,6 +32,7 @@ class KSyncTest < MiniTest::Unit::TestCase
     l1 = File.join(File.dirname(__FILE__), base_path)
     l2 = File.join(File.dirname(__FILE__), "#{base_path}/folder1")
     l3 = File.join(File.dirname(__FILE__), "#{base_path}/folder1/folder2")
+    l4 = File.join(File.dirname(__FILE__), "#{base_path}/folder_empty")
     FileUtils.mkdir_p File.join(File.dirname(__FILE__),File.join(base_path, "/folder1/folder2"))
     # create some files and write some content
     [l1,l2,l3].each do |folder|
@@ -60,34 +63,46 @@ class KSyncTest < MiniTest::Unit::TestCase
   end
 
   # tests
-  def test_new
+  def test_new_tree
     assert_equal(File.exists?(@opts[:dst]), false, "initial conditions not correct")
-    KSync.new(@opts).do_sync
+    KSync::Base.new(@opts).do_sync
     assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
   end
 
   def test_nothing_changed
     create_tree("dst")
     assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
-    assert_equal(KSync.new(@opts).do_sync, false)
+    assert_equal(KSync::Base.new(@opts).do_sync, false)
   end
 
   def test_more_files_in_source
     create_fill_file(File.join(@opts[:src],"NEWFILE.txt"))
-    assert_equal(KSync.new(@opts).do_sync, true)
+    assert_equal(KSync::Base.new(@opts).do_sync, true)
+    assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
+  end
+
+  def test_more_folders_in_source
+    FileUtils.mkdir_p (File.join(@opts[:src],"NEW_FOLDER/NEW_FOLDER"))
+    assert_equal(KSync::Base.new(@opts).do_sync, true)
     assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
   end
 
   def test_less_files_in_source
     FileUtils.rm(File.join(@opts[:src],"File01"))
-    assert_equal(KSync.new(@opts).do_sync, true)
+    assert_equal(KSync::Base.new(@opts).do_sync, true)
+    assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
+  end
+
+  def test_less_folders_in_source
+    FileUtils.rm_rf "#{@opts[:src]}/folder_empty"
+    assert_equal(KSync::Base.new(@opts).do_sync, true)
     assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
   end
 
   def test_source_file_changed_using_hash
     create_fill_file(File.join(@opts[:src],"File01"))
     @opts[:use_hash] = true
-    assert_equal(KSync.new(@opts).do_sync, true)
+    assert_equal(KSync::Base.new(@opts).do_sync, true)
     assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
   end
 
@@ -95,15 +110,29 @@ class KSyncTest < MiniTest::Unit::TestCase
     @opts2 = {:src => File.join(File.dirname(__FILE__), "src_non"), :dst => File.join(File.dirname(__FILE__), "dst_non")}
     assert_equal(File.exists?(@opts2[:src]), false, "initial conditions not correct")
     @opts2[:verbose] = 3
-    assert_equal(KSync.new(@opts2).do_sync, false)
+    assert_equal(KSync::Base.new(@opts2).do_sync, false)
   end
 
   def test_dry_run
     assert_equal(File.exists?(@opts[:dst]), false, "initial conditions not correct")
     @opts[:real_copy] = false
     @opts[:verbose] = 3
-    KSync.new(@opts).do_sync
+    @opts[:force_dest_hash] = true
+    KSync::Base.new(@opts).do_sync
     assert_equal(File.exists?(@opts[:dst]), false)
   end
 
+  def test_use_existing_files_hash
+    assert_equal(File.exists?(@opts[:dst]), false, "initial conditions not correct")
+    k = KSync::Base.new(@opts)
+    k.do_sync
+    assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
+    saved_hash = File.open(File.join(@opts[:dst], KSync::Base::KSYNC_FILES_HASH),'r').read
+    teardown
+    setup
+    File.open(File.join(@opts[:dst], KSync::Base::KSYNC_FILES_HASH),'w'){|f| f.write saved_hash}
+    assert_equal(KSync::Base.new(@opts).do_sync, false)
+    FileUtils.rm File.join(@opts[:dst], KSync::Base::KSYNC_FILES_HASH) rescue nil
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ksync
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
   prerelease:
 platform: ruby
 authors:
@@ -15,25 +15,25 @@ dependencies:
   name: turn
   version_requirements: &2056 !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.9'
     none: false
   requirement: *2056
   prerelease: false
-  type: :runtime
+  type: :development
 - !ruby/object:Gem::Dependency
   name: minitest
   version_requirements: &2074 !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 4.4.0
+        version: '4.4'
     none: false
   requirement: *2074
   prerelease: false
-  type: :runtime
-description: ksync is a simple script which is used to sync between 2 folders, the destination folder being used as a repository
+  type: :development
+description: ksync is a simple class which is used to sync between 2 folders, the destination folder being used as a backup repository
 email: kiriakos.adoniadis@gmail.com
 executables:
 - ksync
@@ -42,11 +42,11 @@ extra_rdoc_files:
 - README
 - MIT-LICENSE
 files:
+- README
+- MIT-LICENSE
 - Rakefile.rb
 - lib/ksync.rb
 - bin/ksync
-- README
-- MIT-LICENSE
 - test/test_ksync.rb
 homepage: http://rubygems.org/gems/ksync
 licenses:
@@ -72,7 +72,8 @@ rubyforge_project:
 rubygems_version: 1.8.9
 signing_key:
 specification_version: 3
-summary: ! 'Ksync: A simple file backup/syncing class'
+summary: A simple file backup/syncing class
 test_files:
 - test/test_ksync.rb
+has_rdoc:
 ...