dir_sync 0.1.0

data/HISTORY.rdoc

@@ -0,0 +1,4 @@
+= 0.1.0
+
+Fri 27 Jan 2012 20:57:18 EST
+Initial release

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Mark Ryall
+ 
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+ 
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+ 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,29 @@
+= dir_sync
+
+Produce a script to synchronise two directories.  Kind of like unison but a bit simpler.
+
+= Usage
+
+    dir_sync name folder1 folder2 ...
+
+This will output all require mkdir and cp commands that would be required to sync the
+folders.  It will also produce the index file (~/.dir_sync/NAME) which will be used
+in subsequent synchronisations to detect deletions.  You can then review the commands
+before executing them (in case they seem likely to do something undesirable). Note that
+files that differ only in modification time will be overwritten with the most recent one.
+
+    drain script
+
+This simply runs each line in the specified script.  As each line is executed, a new script is
+produced.  The purpose of this is to allow the script to be interrupted and able to resume.
+
+Together, these scripts can be used as follows:
+
+    dir_sync photos /Volumes/BACKUP/sync/photos /Users/Shared/Photos | tee sync.sh
+    drain sync.sh
+
+= Future Plans
+
+* actually execute file commands perhaps with optional confirmation
+* consider introducing checksums/filesize comparisons to avoid copying identical files that
+differ only in modification time.

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler/gem_tasks'
+
+task :default => :test
+task :test => [:spec, :features]
+
+task :spec do
+  sh 'rspec spec'
+end
+
+task :features do
+  sh 'cucumber'
+end

data/bin/dir_sync

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname(__FILE__)+'/../lib'
+
+require 'synchroniser'
+
+if ARGV.size < 3
+  puts "usage: #{__FILE__} name directory1 directory2 ..."
+  puts " set DEBUG for verbose output"
+  exit 1
+end
+
+Synchroniser.iterate *ARGV

data/bin/drain

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+if ARGV.empty?
+  puts "Runs a set of single line commands one by one (so that you can interrupt and the script will resume)"
+  puts "usage: #{__FILE__} script"
+  exit 1
+end
+
+file = ARGV.shift
+lines = File.readlines(file).map {|s| s.chomp.strip }.select {|s| !s.empty? and !s.start_with?('#')}
+
+loop do
+  break if lines.empty?
+  lines.shift.tap {|s| puts "> #{s}" }.tap {|s| puts `#{s}` }
+  File.open("#{file}.tmp", 'w') {|f| f.puts lines.join "\n" }
+  `mv #{file}.tmp #{file}`
+end
+`rm #{file}`

data/lib/change_log_file_system.rb

@@ -0,0 +1,15 @@
+class ChangeLogFileSystem
+  def initialize io
+    @io = io
+  end
+
+  def cp from, to
+    to_dir = File.dirname to
+    @io.puts "mkdir -p \"#{to_dir}\"" unless File.exist? to_dir
+    @io.puts "cp -p \"#{from}\" \"#{to}\""
+  end
+
+  def rm path
+    @io.puts "rm \"#{path}\""
+  end
+end

data/lib/change_resolver.rb

@@ -0,0 +1,74 @@
+class ChangeResolver
+  def debug message
+    puts "# #{message}" if ENV['DEBUG']
+  end
+
+  def initialize history, *traversers
+    @history, @traversers = history, traversers
+  end
+
+  def iterate
+    report_traversers
+    first = candidates.first
+    return false unless first
+    debug "first is #{first.description}"
+    send dispatch(first), first
+    advance_matching_traversers first.name, @history, *non_empty_traversers
+  end
+
+  def candidate
+    candidates.first
+  end
+private
+  def dispatch traverser
+    return :rm if traverser.ignored?
+    if traverser.equivalent? @history
+      return all_equivalent_traversers?(traverser) ? :ignore : :rm
+    end
+    :cp
+  end
+
+  def rm traverser
+    traverser.rm
+  end
+
+  def cp traverser
+    @history.report traverser
+    traverser.cp *@traversers
+  end
+
+  def ignore traverser
+  end
+
+  def report_traversers
+    report @history
+    @traversers.each { |traverser| report traverser }
+  end
+
+  def report traverser
+    debug "#{traverser.base}: #{traverser.description}"
+  end
+
+  def advance_matching_traversers name, *traversers
+    traversers.select{|t| t.name == name}.each &:advance
+  end
+
+  def all_equivalent_traversers? traverser
+    @traversers.all? {|t| traverser.equivalent? t }
+  end
+
+  def non_empty_traversers
+    @traversers.select {|t| !t.empty? }
+  end
+
+  def candidates
+    non_empty_traversers.sort do |left,right|
+      combine left.name <=> right.name, right.ts <=> left.ts
+    end
+  end
+
+  def combine *exps
+    exps.each { |exp| return exp unless exp == 0 }
+    0
+  end
+end

data/lib/historical_traverser.rb

@@ -0,0 +1,46 @@
+class HistoricalTraverser
+  attr_reader :name, :ts, :base, :description
+  REGEXP = /:(\d+)$/
+
+  def initialize name
+    home = File.expand_path '~'
+    Pathname.new("#{home}/.dir_sync").mkpath
+    @path_old = "#{home}/.dir_sync/#{name}"
+    @path_new = "#{@path_old}.new"
+    @new = File.open @path_new, 'w'
+    @base = '<history>'
+    if File.exist? @path_old
+      @fiber = Fiber.new do
+        File.open(@path_old) do |file|
+          file.each {|line| Fiber.yield line.chomp}
+        end
+        Fiber.yield nil
+      end
+      @description = :nothing
+      advance
+    end
+  end
+
+  def advance
+    @description = @fiber.resume if @description
+    if @description
+      match = REGEXP.match @description
+      raise "unable to parse line \"#{@description}\"" unless match
+      @name = match.pre_match
+      @ts = match[1].to_i
+    end
+  end
+
+  def close
+    @new.close
+    `mv #{@path_new} #{@path_old}`
+  end
+
+  def report traverser
+    @new.puts traverser.description
+  end
+
+  def empty?
+    @description.nil?
+  end
+end

data/lib/synchroniser.rb

@@ -0,0 +1,15 @@
+require 'traverser'
+require 'historical_traverser'
+require 'change_resolver'
+require 'change_log_file_system'
+
+module Synchroniser
+  def self.iterate name, *paths
+    file_system = ChangeLogFileSystem.new $stdout
+    traversers = paths.map {|path| Traverser.new path, file_system }
+    history = HistoricalTraverser.new name
+    resolver = ChangeResolver.new history, *traversers
+    loop { break unless resolver.iterate }
+    history.close
+  end
+end

data/lib/traverser.rb

@@ -0,0 +1,60 @@
+require 'pathname'
+
+class Traverser
+  attr_reader :base
+
+  TOLERANCE=5
+  IGNORE_PATTERNS = [
+    /\.DS_Store$/,
+    /\/\._/
+  ]
+
+  def initialize path, file_system
+    @file_system = file_system
+    @base = Pathname.new path
+    @base.mkpath
+    @fiber = Fiber.new do
+      @base.find { |child| Fiber.yield child if child.file? }
+      Fiber.yield nil
+    end
+    @current = @fiber.resume
+  end
+
+  def description
+    empty? ? 'empty' : "#{name}:#{ts}"
+  end
+
+  def advance
+    @current = @fiber.resume if @current
+  end
+
+  def name
+    @current.relative_path_from(@base).to_s if @current
+  end
+
+  def ts
+    @current.mtime.to_i if @current
+  end
+
+  def empty?
+    @current.nil?
+  end
+
+  def ignored?
+    IGNORE_PATTERNS.any? {|pattern| pattern.match name }
+  end
+
+  def cp *traversers
+    traversers.each do |t|
+      @file_system.cp @current.to_s, "#{t.base}/#{name}" unless equivalent? t
+    end
+  end
+
+  def rm
+    @file_system.rm @current.to_s
+  end
+
+  def equivalent? traverser
+    name == traverser.name and (ts - traverser.ts).abs <= TOLERANCE
+  end
+end

data/spec/change_resolver_spec.rb

@@ -0,0 +1,165 @@
+$: << File.dirname(__FILE__)+'/../lib'
+
+require 'change_resolver'
+
+describe ChangeResolver do
+  let(:history) { stub 'history', name: nil, base: nil, description: nil, report: nil, advance: nil }
+  let(:traversers) { [] }
+  let(:resolver) { ChangeResolver.new history, *traversers }
+
+  def stub_history hash
+    hash.each do |meth,ret|
+       history.stub!(meth).and_return ret
+    end
+  end
+
+  def stub_traversers hashes
+    hashes.each_with_index do |traverser_stubs, index|
+      stubs = {
+        ts: 0,
+        cp: nil,
+        advance: nil,
+        empty?: false,
+        ignored?: false,
+        base: nil,
+        description: nil,
+        equivalent?: false
+      }.merge traverser_stubs
+      traversers << stub("traverser#{index}").tap do |traverser|
+        stubs.each do |meth,ret|
+           traverser.stub!(meth).and_return ret
+        end
+      end
+    end
+  end
+
+  def candidate
+    resolver.candidate
+  end
+
+  describe '#iterate' do
+    it 'should return true if any traverser is not empty' do
+      stub_traversers [
+        { name: 'a'},
+        { name: 'a'}
+      ]
+      resolver.iterate.should be_true
+    end
+
+    it 'should return false if all traversers are empty' do
+      stub_traversers [
+        { name: 'a'},
+        { name: 'b'}
+      ]
+      traversers[0].should_receive(:empty?).and_return true
+      traversers[1].should_receive(:empty?).and_return true
+      resolver.iterate.should be_false
+    end
+
+    it 'should copy all other traversers to candidate' do
+      stub_traversers [
+        { name: 'a'},
+        { name: 'b'},
+        { name: 'c'},
+      ]
+      traversers[0].should_receive(:cp).with *traversers
+      resolver.iterate
+    end
+
+    it 'should remove files that are ignored' do
+      stub_traversers [
+        { name: 'a', ignored?: true},
+        { name: 'b'},
+        { name: 'c'},
+      ]
+      traversers[0].should_receive :rm
+      resolver.iterate
+    end
+
+    it 'should ignore files that appear in history and have not been removed from any traverser' do
+      stub_history name: 'a'
+      stub_traversers [
+        { name: 'a'},
+        { name: 'a'}
+      ]
+      traversers[0].should_receive(:equivalent?).with(history).and_return true
+      traversers[0].should_receive(:equivalent?).with(traversers[0]).and_return true
+      traversers[0].should_receive(:equivalent?).with(traversers[1]).and_return true
+      traversers[0].should_not_receive :rm
+      resolver.iterate
+    end
+
+    it 'should remove files that appear in history but have been removed from a traverser' do
+      stub_history name: 'a'
+      stub_traversers [
+        { name: 'a'},
+        { name: 'b'},
+        { name: 'a'}
+      ]
+      traversers[0].should_receive(:equivalent?).with(history).and_return true
+      traversers[0].should_receive(:equivalent?).with(traversers[0]).and_return true
+      traversers[0].should_receive(:equivalent?).with(traversers[1]).and_return false
+      traversers[0].should_receive :rm
+      resolver.iterate
+    end
+
+    it 'should advance history when it has candidate name' do
+      stub_history name: 'a'
+      stub_traversers [{ name: 'a'}, { name: 'b'}]
+      history.should_receive :advance
+      resolver.iterate
+    end
+
+    it 'should advance traversers with the candidate name' do
+      stub_history name: 'b'
+      stub_traversers [
+        { name: 'a'},
+        { name: 'a'},
+        { name: 'b'},
+      ]
+      history.should_not_receive :advance
+      traversers[0].should_receive :advance
+      traversers[1].should_receive :advance
+      traversers[2].should_not_receive :advance
+      resolver.iterate
+    end
+  end
+
+  describe '#candidate' do
+    it 'should determine the next candidate according to the name collation order' do
+      stub_traversers [
+        { name: 'a'},
+        { name: 'b'},
+        { name: 'c'},
+      ]
+      candidate.should == traversers[0]
+    end
+
+    it 'should determine the next candidate according to the name collation order' do
+      stub_traversers [
+        { name: 'c'},
+        { name: 'b'},
+        { name: 'a'},
+      ]
+      candidate.should == traversers[2]
+    end
+
+    it 'should determine the next candidate for the same name by most recent timestamp' do
+      stub_traversers [
+        { name: 'a', ts: 10},
+        { name: 'a', ts: 20},
+        { name: 'b'},
+      ]
+      candidate.should == traversers[1]
+    end
+
+    it 'should determine the next candidate for the same name by most recent timestamp' do
+      stub_traversers [
+        { name: 'a', ts: 20},
+        { name: 'a', ts: 10},
+        { name: 'b'},
+      ]
+      candidate.should == traversers[0]
+    end
+  end
+end

data/spec/synchroniser_spec.rb

@@ -0,0 +1,31 @@
+$: << File.dirname(__FILE__)+'/../lib'
+
+require 'synchroniser'
+
+describe Synchroniser do
+  let(:file_system) { stub 'file_system' }
+  let(:resolver) { stub 'resolver'}
+  let(:history) { stub 'history', close: nil }
+  let(:traverser_a) { stub 'traverser_a' }
+  let(:traverser_b) { stub 'traverser_b' }
+
+  before do
+    ChangeLogFileSystem.should_receive(:new).with($stdout).and_return file_system
+    HistoricalTraverser.should_receive(:new).with('test').and_return history
+    Traverser.should_receive(:new).with('a', file_system).and_return traverser_a
+    Traverser.should_receive(:new).with('b', file_system).and_return traverser_b
+    ChangeResolver.should_receive(:new).with(history, traverser_a, traverser_b).and_return resolver
+  end
+
+  it 'should call iterate once resolved if it returns false' do
+    resolver.should_receive(:iterate).and_return false
+    Synchroniser.iterate 'test', 'a', 'b'
+  end
+
+  it 'should repeatedly call iterate on resolved until it returns false' do
+    resolver.should_receive(:iterate).and_return true
+    resolver.should_receive(:iterate).and_return true
+    resolver.should_receive(:iterate).and_return false
+    Synchroniser.iterate 'test', 'a', 'b'
+  end
+end

data/spec/traverser_spec.rb

@@ -0,0 +1,58 @@
+$: << File.dirname(__FILE__)+'/../lib'
+
+require 'traverser'
+
+describe Traverser do
+  let(:pathname) { stub 'pathname', mkpath: nil, find: nil}
+  let(:file_system) { stub 'file_system'}
+  let(:traverser) { Traverser.new 'a', file_system }
+
+  before do
+    Pathname.should_receive(:new).with('a').and_return pathname
+  end
+
+  it 'should create path' do
+    pathname.should_receive :mkpath
+    traverser
+  end
+
+  def with_file name, mtime=0
+    child_pathname = stub 'current_pathname', file?: true, mtime: mtime, to_s: "a/#{name}"
+    child_pathname.stub!(:relative_path_from).with(pathname).and_return name
+    pathname.stub!(:find).and_yield child_pathname
+  end
+
+  it 'should skip copying to itself' do
+    with_file '1.txt'
+    file_system.should_not_receive :cp
+    traverser.cp traverser
+  end
+
+  it 'should copy to traversers with a different file name' do
+    with_file '1.txt'
+    other_traverser = stub 'other_traverser', base: 'b', name: '2.txt'
+    file_system.should_receive(:cp).with 'a/1.txt', 'b/1.txt'
+    traverser.cp other_traverser
+  end
+
+  it 'should skip copying to traversers with the same file name and timestamp' do
+    with_file '1.txt', 10
+    other_traverser = stub 'other_traverser', base: 'b', name: '1.txt', ts: 10
+    file_system.should_not_receive :cp
+    traverser.cp other_traverser
+  end
+
+  it 'should skip copying to traversers with the same file name and timestamp is within tolerance' do
+    with_file '1.txt', 10
+    other_traverser = stub 'other_traverser', base: 'b', name: '1.txt', ts: 15
+    file_system.should_not_receive :cp
+    traverser.cp other_traverser
+  end
+
+  it 'should skip copying to traversers with the same file name and timestamp is outside tolerance' do
+    with_file '1.txt', 10
+    other_traverser = stub 'other_traverser', base: 'b', name: '1.txt', ts: 16
+    file_system.should_receive(:cp).with 'a/1.txt', 'b/1.txt'
+    traverser.cp other_traverser
+  end
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: dir_sync
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Mark Ryall
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-01-27 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70177781836300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70177781836300
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70177781835000 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: *70177781835000
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: &70177781834120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70177781834120
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: &70177781832600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70177781832600
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: &70177781831880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70177781831880
+description: ! 'Bidirectional directory synchronisation for any number of directories
+
+'
+email: mark@ryall.name
+executables:
+- dir_sync
+- drain
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/change_log_file_system.rb
+- lib/change_resolver.rb
+- lib/historical_traverser.rb
+- lib/synchroniser.rb
+- lib/traverser.rb
+- spec/change_resolver_spec.rb
+- spec/synchroniser_spec.rb
+- spec/traverser_spec.rb
+- bin/dir_sync
+- bin/drain
+- README.rdoc
+- MIT-LICENSE
+- HISTORY.rdoc
+- Rakefile
+- .gemtest
+homepage: http://github.com/markryall/dir_sync
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2227443201844382087
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2227443201844382087
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: directory synchroniser
+test_files: []