ksync 0.5.0

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2013 Kirk Adoniadis (kiriakos.adoniadis@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,31 @@
+This gem is a simple folder synchroniser between a changing source folder and a
+'backup' destination folder (presumed to be unchanged). It uses a list of files hash which is a
+hash containing the filepath as the key and file information [FileSize, FileMTime] as value.
+
+In order to compare the source and destination folders:
+  a. it always creates a list of files hash for the source
+  b. it uses an existing list of files hash for the destination or creates one if non existing
+  c. It compares the 2 lists of files hash and copies/deletes from destination accordingly.
+  d. It compares files based on size and then on last modification date. It may optionally use
+     hash calculation (inactive by default because it is slow)
+
+NOTE: If any files in the destination folder have changed and the files hash exists in the
+destination and the -u option (use hash) is not specified, then these changes will NOT be detected.
+This is a fast backup solution and presumes that the destination never changes.
+
+The gem comes with a frontend, 'ksync' which can be used directly to invoke the object
+
+Examples of usage:
+
+To backup folder c:/dev to c:/dev_backup with default options :
+> ksync c:/dev c:/dev_backup
+The above, but inside rour ruby code :
+>  require 'ksync'
+>  KSync.new({:src => 'c:/dev', :dst => 'c:/dev_backup'}).do_sync
+the method do_sync will return false if there were no changes, true otherwise
+
+To backup folder c:/dev to c:/dev_backup forcing hash calculation (hash calculation will only be used if the files have
+the same size and modification date):
+> ksync --use_hash c:/dev c:/dev_backup
+or
+> ksync -u c:/dev c:/dev_backup

data/Rakefile.rb

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/bin/ksync

@@ -0,0 +1,30 @@
+#!/usr/bin/env jruby
+
+require 'optparse'
+require 'ksync'
+
+def message(o)
+    puts o
+    exit
+end
+
+options = {}
+options[:real_copy] = true
+options[:use_hash] = false
+options[:verbose] = 0
+o = OptionParser.new
+o.banner = "Usage: #{$0} [options] source_folder destination_folder"
+o.on('-d', '--dry_run', 'dry run (default : do the real copy - no dry run)') { |s| options[:real_copy] = false }
+o.on('-v', '--verbosity=value', 'The level of verbosity (1..3) (default = 0 : very silent)') { |s| options[:verbose] = s.to_i }
+o.on('-u', '--use_hash', 'use hash calculation (default : dont use hash)') { |s| options[:use_hash] = true }
+o.on('-h', '--help') { message(o) }
+
+o.parse!
+
+if !ARGV[0] || !ARGV[1]
+    message(o)
+else
+    options[:src] =  ARGV[0]
+    options[:dst] =  ARGV[1]
+    KSync.new(options).do_sync
+end

data/lib/ksync.rb

@@ -0,0 +1,159 @@
+require 'fileutils'
+require 'digest'
+require 'find'
+
+class KSync
+
+  SSIZE_I=0
+  MTIME_I=1
+
+  attr_accessor :all_h, :ll
+  def initialize(init_opt)
+    @all_h={}
+    @changes = false
+    init_opt[:real_copy] = init_opt[:real_copy] == nil ? true : init_opt[:real_copy]
+    init_opt[:use_hash] = init_opt[:use_hash] == nil ? false : init_opt[:use_hash]
+    init_opt[:verbose] = init_opt[:verbose] == nil ? 0 : init_opt[:verbose]
+    return nil if !init_opt[:src] | !init_opt[:dst]
+    @options = init_opt
+    @ll = [init_opt[:src], init_opt[:dst]]
+    create_hash    
+  end
+
+  def do_sync    
+    src=0    
+    dst=1    
+    # browse source hash to detect new and changed files
+    return false unless File.exists?(ll.first)
+    @all_h[ll[src]].each do |k,v|
+      src_fp = File.join(@ll[src],k)
+      dst_fp = File.join(@ll[dst],k)
+      if !@all_h[@ll[dst]][k]
+        puts "N: #{k}" if @options[:verbose] > 0
+        do_copy(src_fp, dst_fp, v) if @options[:real_copy]
+        @changes = true
+      else
+        if !File.directory?(File.join(@ll[src],k))
+          if !same?(src_fp, dst_fp, v, all_h[@ll[dst]][k])
+            puts "C: #{k}" if @options[:verbose] > 0
+            do_copy(src_fp, dst_fp, v) if @options[:real_copy]
+            @changes = true
+          end
+        end   
+      end   
+    end  
+    # browse destination hash to detect obsolete files
+    @all_h[ll[dst]].each do |k,v|
+      if !all_h[ll[src]][k] 
+        dst_fp = File.join(ll[dst],k)
+        puts "D: #{dst_fp}" if @options[:verbose] > 0
+        do_delete dst_fp if @options[:real_copy]
+        @changes = true
+      end  
+    end  
+    #puts "there were changes" if @changes
+    #puts "nothing changed" if !@changes
+    return @changes
+  end  
+  
+  def get_hash(in_file)    
+    if !File.exists?(in_file)
+      puts "ERR: inexistent file : #{in_file}"
+      return nil
+    end  
+    sha1 = Digest::SHA1.new
+    begin
+      File.open(in_file) do |file|
+        buffer = ''
+        # Read the file 512 bytes at a time
+        while not file.eof
+          file.read(512, buffer)
+          sha1.update(buffer)
+        end
+      end
+    rescue 
+      puts "ERR: while calculating hash for : #{in_file}: (#{$!})"
+      return nil
+    end  
+    return sha1.to_s
+  end 
+
+  def do_copy(src_fp, dst_fp, src)
+    begin       
+      if File.directory?(src_fp)
+        file_op = "mkdir"
+        FileUtils.mkdir_p dst_fp 
+      else
+        file_op = "cp"
+        FileUtils.copy_file src_fp, dst_fp        
+      end
+    rescue 
+      puts "ERR: while copying (#{file_op}) [#{src_fp} to #{dst_fp}] : (#{$!})"
+    end    
+  end
+
+  def do_delete (dst_fp)
+    begin 
+      if File.exists?(dst_fp) 
+        if File.directory?(dst_fp)
+          file_op = 'remove_dir'
+          FileUtils.remove_dir(dst_fp)
+        else
+          file_op = 'rm'
+          FileUtils.rm dst_fp
+        end          
+      else
+        puts "ERR: inexistent file/folder : #{dst_fp}"
+      end      
+    rescue 
+      puts "ERR: (#{file_op}) for #{dst_fp} : (#{$!})"
+    end
+  end
+  
+  # if hash => also check hash if sizes are same
+  def same?(src_fp, dst_fp, src, dst)
+    size_ok = false
+    date_ok = false   
+    if src[SSIZE_I] == dst[SSIZE_I] # sizes are the same
+      size_ok = true
+      if dst[MTIME_I] - dst[MTIME_I]  >=  0                
+        date_ok = true 
+      else
+        puts "DATE diff for #{src_fp} vs #{dst_fp}" if @options[:verbose] > 1
+      end  
+    else
+      puts "SIZE diff for #{src_fp} vs #{dst_fp}" if @options[:verbose] > 1
+    end    
+    return false if !size_ok 
+    return false if !date_ok
+    return true if !@options[:use_hash]
+    h1 = get_hash(src_fp)
+    h2 = get_hash(dst_fp)
+    if h1 != h2
+      puts "SHA1 diff for #{src_fp} vs #{dst_fp}" if @options[:verbose] > 1
+      return false
+    end
+    return true
+  end
+  def create_hash
+    if !File.exists?(@ll.first)
+      puts "source folder #{@ll.first} does not exist ! "
+      return
+    end
+    @ll.each do |prefix|
+      @all_h[prefix] = {}
+      next if prefix == @ll.last && @options[:real_copy] == false
+      puts "creating hash for #{prefix}" if @options[:verbose] > 1
+      cnt = 0 
+      FileUtils.mkdir_p prefix if !File.exists?(prefix)
+      Find.find(prefix).each do |f|
+        next if prefix == f
+        ff = f.unpack('U*').pack('U*')  # to get rid of funny encoding related errors
+        @all_h[prefix][ff.split(prefix)[1]] = [File.size(ff), File.mtime(ff)]
+        puts "(#{ff})" if @options[:verbose] > 2
+        puts cnt if cnt % 100 == 0 && @options[:verbose] > 2
+        cnt += 1
+      end
+    end
+  end
+end

data/test/test_ksync.rb

@@ -0,0 +1,109 @@
+gem 'minitest', '>= 4.4.0' # use the gem version, not the 1.9 bundled version of minitest
+require 'minitest/autorun'
+
+require 'ksync'
+require 'fileutils'
+
+require 'turn'
+
+Turn.config.format = :outline
+
+class KSyncTest < MiniTest::Unit::TestCase
+  def setup
+    begin
+      @src_path = File.join(File.dirname(__FILE__), "src/folder1/folder2")
+      @dst_path = File.join(File.dirname(__FILE__), "src")
+      create_tree("src")
+      @opts = {}
+      @opts = {:src => File.join(File.dirname(__FILE__), "src"), :dst => File.join(File.dirname(__FILE__), "dst")}
+    rescue
+      puts "error trying to do the setup :#{$!}"
+    end
+  end
+  def teardown
+   FileUtils.rm_rf  @opts[:src]
+   FileUtils.rm_rf  @opts[:dst]
+  end
+
+  # support methods
+  def create_tree(base_path)
+    l1 = File.join(File.dirname(__FILE__), base_path)
+    l2 = File.join(File.dirname(__FILE__), "#{base_path}/folder1")
+    l3 = File.join(File.dirname(__FILE__), "#{base_path}/folder1/folder2")
+    FileUtils.mkdir_p File.join(File.dirname(__FILE__),File.join(base_path, "/folder1/folder2"))
+    # create some files and write some content
+    [l1,l2,l3].each do |folder|
+      (1..10).each do |f|
+        create_fill_file(File.join(folder, "File%02d" % f))
+      end
+    end
+  end
+
+  def create_fill_file(filename)
+    File.open(filename, "w") do |fh|
+      (1..10).each do
+        fh.write ('a'..'z').to_a.shuffle[0,8].join + "\n"
+      end
+    end
+  end
+  # only compares folders based on file existence not file size or other
+  def compare_folders(src,dst)
+    Find.find(src).each do |f|
+      dst_file_folder = f.gsub(src,dst)
+      #puts "=> #{dst_file_folder}"
+      if !File.exists?(dst_file_folder)
+        puts "non existent file #{File.join(dst,f.split('/')[1..-1])}"
+        return false
+      end
+    end
+    return true
+  end
+
+  # tests
+  def test_new
+    assert_equal(File.exists?(@opts[:dst]), false, "initial conditions not correct")
+    KSync.new(@opts).do_sync
+    assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
+  end
+
+  def test_nothing_changed
+    create_tree("dst")
+    assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
+    assert_equal(KSync.new(@opts).do_sync, false)
+  end
+
+  def test_more_files_in_source
+    create_fill_file(File.join(@opts[:src],"NEWFILE.txt"))
+    assert_equal(KSync.new(@opts).do_sync, true)
+    assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
+  end
+
+  def test_less_files_in_source
+    FileUtils.rm(File.join(@opts[:src],"File01"))
+    assert_equal(KSync.new(@opts).do_sync, true)
+    assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
+  end
+
+  def test_source_file_changed_using_hash
+    create_fill_file(File.join(@opts[:src],"File01"))
+    @opts[:use_hash] = true
+    assert_equal(KSync.new(@opts).do_sync, true)
+    assert_equal(compare_folders(@opts[:src], @opts[:dst]), true)
+  end
+
+  def test_dont_create_src_folder
+    @opts2 = {:src => File.join(File.dirname(__FILE__), "src_non"), :dst => File.join(File.dirname(__FILE__), "dst_non")}
+    assert_equal(File.exists?(@opts2[:src]), false, "initial conditions not correct")
+    @opts2[:verbose] = 3
+    assert_equal(KSync.new(@opts2).do_sync, false)
+  end
+
+  def test_dry_run
+    assert_equal(File.exists?(@opts[:dst]), false, "initial conditions not correct")
+    @opts[:real_copy] = false
+    @opts[:verbose] = 3
+    KSync.new(@opts).do_sync
+    assert_equal(File.exists?(@opts[:dst]), false)
+  end
+
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification
+name: ksync
+version: !ruby/object:Gem::Version
+  version: 0.5.0
+  prerelease:
+platform: ruby
+authors:
+- Kirk Adoniadis
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2013-01-07 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: turn
+  version_requirements: &2056 !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  requirement: *2056
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: minitest
+  version_requirements: &2074 !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 4.4.0
+    none: false
+  requirement: *2074
+  prerelease: false
+  type: :runtime
+description: ksync is a simple script which is used to sync between 2 folders, the destination folder being used as a repository
+email: kiriakos.adoniadis@gmail.com
+executables:
+- ksync
+extensions: []
+extra_rdoc_files:
+- README
+- MIT-LICENSE
+files:
+- Rakefile.rb
+- lib/ksync.rb
+- bin/ksync
+- README
+- MIT-LICENSE
+- test/test_ksync.rb
+homepage: http://rubygems.org/gems/ksync
+licenses:
+- MIT
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+  none: false
+requirements: []
+rubyforge_project:
+rubygems_version: 1.8.9
+signing_key:
+specification_version: 3
+summary: ! 'Ksync: A simple file backup/syncing class'
+test_files:
+- test/test_ksync.rb
+...