ftp_sync 0.4.1

data/README.rdoc

@@ -0,0 +1,24 @@
+= FtpSync
+
+A Ruby library for recursively downloading and uploading directories to/from ftp
+servers. Also supports uploading and downloading a list of files relative to 
+the local/remote roots. You can specify a timestamp to only download files 
+newer than that timestamp, or only download files newer than their local copy.
+
+This was originally written to provide the functionality I needed for Munkey,
+a tool for tracking changes on FTP servers with git.
+
+Allows you to supply a 'ignore' class that dictates whether a file is excluded
+from upload / download.
+
+== Quickstart
+
+ ftp = FtpSync.new 'my.ftp.server.com', 'username', 'password'
+ ftp.pull_dir '/tmp/syncdir', 'path/on/server'
+
+make some changes on server
+
+ ftp.pull_dir '/tmp/syncdir', 'path/on/server', :since => true, :delete => true
+
+ :since => true - means only files which are newer than the local copy will be downloaded
+ :delete => true - means if the file is removed from the server, then the local copy is removed

data/lib/ftp_sync.rb

@@ -0,0 +1,189 @@
+require 'net/ftp'
+require 'rubygems'
+require 'net/ftp/list'
+require 'fileutils'
+
+# A Ruby library for recursively downloading and uploading directories to/from ftp
+# servers. Also supports uploading and downloading a list of files relative to 
+# the local/remote roots. You can specify a timestamp to only download files 
+# newer than that timestamp, or only download files newer than their local copy.
+class FtpSync
+  
+  attr_accessor :verbose, :server, :user, :password
+  
+  # Creates a new instance for accessing a ftp server 
+  # requires +server+, +user+, and +password+ options
+  # * :ignore - Accepts an instance of class which has an ignore? method, taking a path and returns true or false, for whether to ignore the file or not.
+  # * :verbose - Whether should be verbose
+  def initialize(server, user, password, options = {})
+    @server = server
+    @user = user
+    @password = password
+    @connection = nil
+    @ignore = options[:ignore]
+    @recursion_level = 0
+    @verbose = options[:verbose] || false
+  end
+  
+  # Recursively pull down files
+  # :since => true - only pull down files newer than their local counterpart, or with a different filesize
+  # :since => Time.now - only pull down files newer than the supplied timestamp, or with a different filesize
+  # :delete => Remove local files which don't exist on the FTP server
+  # If a block is supplied then it will be called to remove a local file
+  
+  def pull_dir(localpath, remotepath, options = {}, &block)
+    connect! unless @connection
+    @recursion_level += 1
+
+    todelete = Dir.glob(File.join(localpath, '*'))
+    
+    tocopy = []
+    recurse = []
+
+    # To trigger error if path doesnt exist since list will
+    # just return and empty array
+    @connection.chdir(remotepath) 
+
+    @connection.list(remotepath) do |e|
+      entry = Net::FTP::List.parse(e)
+      
+      paths = [ File.join(localpath, entry.basename), "#{remotepath}/#{entry.basename}".gsub(/\/+/, '/') ]
+
+      if entry.dir?
+        recurse << paths
+      elsif entry.file?
+        if options[:since] == :src
+          tocopy << paths unless File.exist?(paths[0]) and entry.mtime < File.mtime(paths[0]) and entry.filesize == File.size(paths[0])
+        elsif options[:since].is_a?(Time)
+          tocopy << paths unless entry.mtime < options[:since] and File.exist?(paths[0]) and entry.filesize == File.size(paths[0])
+        else
+          tocopy << paths
+        end
+      end
+      todelete.delete paths[0]
+    end
+    
+    tocopy.each do |paths|
+      localfile, remotefile = paths
+      unless should_ignore?(localfile)
+        @connection.get(remotefile, localfile)
+        log "Pulled file #{remotefile}"
+      end
+    end
+    
+    recurse.each do |paths|
+      localdir, remotedir = paths
+      Dir.mkdir(localdir) unless File.exist?(localdir)
+      pull_dir(localdir, remotedir, options, &block)
+    end
+    
+    if options[:delete]
+      todelete.each do |p|
+        block_given? ? yield(p) : FileUtils.rm_rf(p)
+        log "Removed path #{p}"
+      end
+    end
+    
+    @recursion_level -= 1
+    close! if @recursion_level == 0
+  rescue Net::FTPPermError
+    close!
+    raise Net::FTPPermError
+  end
+  
+  # Recursively push a local directory of files onto an FTP server
+  def push_dir(localpath, remotepath)
+    connect!
+    
+    Dir.glob(File.join(localpath, '**', '*')) do |f|
+      f.gsub!("#{localpath}/", '')
+      local = File.join localpath, f
+      remote = "#{remotepath}/#{f}".gsub(/\/+/, '/')
+            
+      if File.directory?(local)
+        @connection.mkdir remote rescue Net::FTPPermError
+        log "Created Remote Directory #{local}"
+      elsif File.file?(local)
+        @connection.put local, remote
+        log "Pushed file #{remote}"
+      end
+    end
+    
+    close!
+  end
+  
+  # Pull a supplied list of files from the remote ftp path into the local path
+  def pull_files(localpath, remotepath, filelist)
+    connect!
+    filelist.each do |f|
+      localdir = File.join(localpath, File.dirname(f))
+      FileUtils.mkdir_p localdir unless File.exist?(localdir)
+      @connection.get "#{remotepath}/#{f}", File.join(localpath, f)
+      log "Pulled file #{remotepath}/#{f}"
+    end
+    close!
+  end
+  
+  # Push a supplied list of files from the local path into the remote ftp path
+  def push_files(localpath, remotepath, filelist)
+    connect!
+    
+    remote_paths = filelist.map {|f| File.dirname(f) }.uniq.reject{|p| p == '.' }
+    create_remote_paths(remotepath, remote_paths)
+    
+    filelist.each do |f|
+      @connection.put File.join(localpath, f), "#{remotepath}/#{f}"
+      log "Pushed file #{remotepath}/#{f}"
+    end
+    close!
+  end
+  
+  # Remove listed files from the FTP server
+  def remove_files(basepath, filelist)
+    connect!
+    
+    filelist.each do |f| 
+      begin
+        @connection.delete "#{basepath}/#{f}".gsub(/\/+/, '/') 
+        log "Removed file #{basepath}/#{f}"
+      rescue Net::FTPPermError => e
+        raise e unless /^550/ =~ e.message
+      end
+    end
+    
+    close!
+  end
+  
+  # Chains off to the (if supplied) Ignore class, ie GitIgnores.new.ignore?('path/to/my/file')
+  def should_ignore?(path)
+    @ignore && @ignore.ignore?(path)
+  end
+  
+  private
+    def connect!
+      @connection = Net::FTP.new(@server)
+      @connection.login(@user, @password)
+      log "Opened connection to #{@server}"
+    end
+  
+    def close!
+      @connection.close
+      log "Closed Connection to #{@server}"
+    end
+    
+    def create_remote_paths(base, pathlist)
+      base = '' if base == '/'
+      pathlist.each do |remotepath|
+        parent = base
+        remotepath.split('/').each do |p|
+          parent = "#{parent}/#{p}"
+          @connection.mkdir(parent) rescue Net::FTPPermError
+          log "Creating Remote Directory #{parent}"
+        end
+      end
+    end
+    
+    def log(msg)
+      puts msg if @verbose
+    end
+end

data/test/ftp_sync_test.rb

@@ -0,0 +1,219 @@
+test_path = File.expand_path(File.dirname(__FILE__))
+lib_path = File.join(File.expand_path(File.dirname(__FILE__)), '..', 'lib')
+
+$:.unshift test_path unless $:.include?(test_path)
+$:.unshift lib_path unless $:.include?(lib_path)
+
+require 'rubygems'
+require 'test/unit'
+require 'net/ftp'
+require 'ftp_sync'
+require 'tmpdir'
+require 'fileutils'
+
+class Ignore
+  def ignore?(p); p == 'ignore' ? true : false; end
+end
+
+class FtpSyncTest < Test::Unit::TestCase
+  
+  def setup
+    Net::FTP.create_ftp_src
+    Net::FTP.listing_overrides = {}
+    @local = File.join Dir.tmpdir, create_tmpname
+    FileUtils.mkdir_p @local
+    @ftp = FtpSync.new('test.server', 'user', 'pass')
+  end
+  
+  def teardown
+    FileUtils.rm_rf @local
+    FileUtils.rm_rf Net::FTP.ftp_src
+    FileUtils.rm_rf Net::FTP.ftp_dst if File.exist?(Net::FTP.ftp_dst)
+  end
+  
+  def test_can_initialize_with_params
+    assert_equal 'test.server', @ftp.server
+    assert_equal 'user', @ftp.user
+    assert_equal 'pass', @ftp.password
+  end
+  
+  def test_can_set_verbose
+    @ftp.verbose = true
+    assert_equal true, @ftp.verbose
+    @ftp.verbose = false
+    assert_equal false, @ftp.verbose  
+  end
+  
+  def test_setting_an_ignore_object    
+    ftp = FtpSync.new('localhost', 'user', 'pass', { :ignore => Ignore.new })
+    assert ftp.should_ignore?('ignore')
+    assert !ftp.should_ignore?('something')
+  end  
+  
+  def test_pulling_from_an_unknown_server
+    assert_raise SocketError do
+      ftp = FtpSync.new('unknown.server', 'user', 'pass')
+      ftp.pull_files(@local, '/', ['README'])
+    end
+  end
+  
+  def test_pulling_files_with_bad_account_details
+    assert_raise Net::FTPPermError do
+      ftp = FtpSync.new('test.server', 'unknown', 'unknown')
+      ftp.pull_files(@local, '/', ['README'])
+    end
+  end
+  
+  def test_pulling_files
+    @ftp.pull_files(@local, '/', ['README', 'fileA'])
+    assert File.exist?(File.join(@local, 'README'))
+    assert File.exist?(File.join(@local, 'fileA'))
+  end
+  
+  def test_pulling_unknown_files
+    assert_raise Net::FTPPermError do
+      @ftp.pull_files(@local, '/', ['unknown' ])
+    end
+  end
+  
+  def test_pulling_files_from_subdirs
+    @ftp.pull_files(@local, '/', ['dirA/fileAA'])
+    assert File.exist?(File.join(@local, 'dirA/fileAA'))
+  end
+  
+  def test_pull_dir_from_root
+    @ftp.pull_dir(@local, '/')
+    assert File.exist?(File.join(@local, 'fileA'))
+    assert File.exist?(File.join(@local, 'fileB'))
+    assert File.exist?(File.join(@local, 'dirA/fileAA'))
+    assert File.exist?(File.join(@local, 'dirA/dirAA/fileAAA'))
+    assert File.exist?(File.join(@local, 'dirB/fileBA'))
+    assert File.exist?(File.join(@local, 'dirB/fileBB'))
+  end
+  
+  def test_pull_dir_from_subdir
+    @ftp.pull_dir(@local, '/dirA')
+    assert File.exist?(File.join(@local, 'fileAA'))
+    assert File.exist?(File.join(@local, 'dirAA/fileAAA'))
+  end
+  
+  def test_pull_dir_from_nonexistant_dir
+    assert_raise Net::FTPPermError do
+      @ftp.pull_dir(@local, 'something')
+    end
+  end
+  
+  def test_pulling_dir_over_existing_files
+    assert_nothing_raised do
+      @ftp.pull_dir(@local, '/')
+      FileUtils.rm File.join(@local, 'README')
+      @ftp.pull_dir(@local, '/')
+      assert File.exist?(File.join(@local, 'README'))
+    end
+  end
+  
+  def test_pulling_dir_with_deleting_files
+    @ftp.pull_dir(@local, '/')
+    FileUtils.rm_r File.join(Net::FTP.ftp_src, 'README')
+    @ftp.pull_dir(@local, '/', :delete => true)
+    assert !File.exist?(File.join(@local, 'README'))
+  end
+  
+  def test_pulling_dir_with_not_deleting_files
+    @ftp.pull_dir(@local, '/')
+    assert File.exist?(File.join(@local, 'README'))
+    FileUtils.rm_r File.join(Net::FTP.ftp_src, 'README')
+    @ftp.pull_dir(@local, '/')
+    assert File.exist?(File.join(@local, 'README'))
+  end
+  
+  def test_quick_pull_of_file_older_than_change_date
+    @ftp.pull_dir(@local, '/')
+    File.open(File.join(Net::FTP.ftp_src, 'README'), 'w') {|f| f.write 'quicktest' }
+    Net::FTP.listing_overrides['/'] = ["-rw-r--r--   1 root     other         0 #{(Time.now - 600).strftime('%b %d %H:%M')} README"]    
+    @ftp.pull_dir(@local, '/', :since => Time.now - 120)
+    assert_no_match /quicktest/, File.read(File.join(@local, 'README'))
+  end
+  
+  def test_quick_pull_of_file_newer_than_change_date
+    @ftp.pull_dir(@local, '/')
+    File.open(File.join(Net::FTP.ftp_src, 'README'), 'w') {|f| f.write 'quicktest' }
+    Net::FTP.listing_overrides['/'] = ["-rw-r--r--   1 root     other          0 #{(Time.now - 30).strftime('%b %d %H:%M')} README"]
+    @ftp.pull_dir(@local, '/', :since => Time.now - 120)
+    assert_match /quicktest/, File.read(File.join(@local, 'README'))
+  end
+  
+  def test_quick_pull_of_file_older_than_change_date_with_incorrect_file_size
+    @ftp.pull_dir(@local, '/')
+    File.open(File.join(Net::FTP.ftp_src, 'README'), 'w') {|f| f.write 'quicktest' }
+    Net::FTP.listing_overrides['/'] = ["-rw-r--r--   1 root     other          9 #{(Time.now - 600).strftime('%b %d %H:%M')} README"]
+    @ftp.pull_dir(@local, '/', :since => Time.now - 120)
+    assert_match /quicktest/, File.read(File.join(@local, 'README'))
+  end
+  
+  def test_quick_pull_of_file_older_than_dst_file
+    @ftp.pull_dir(@local, '/')
+    File.open(File.join(Net::FTP.ftp_src, 'README'), 'w') {|f| f.write 'quicktest' }
+    Net::FTP.listing_overrides['/'] = ["-rw-r--r--   1 root     other          0 #{(Time.now - 90).strftime('%b %d %H:%M')} README"]
+    @ftp.pull_dir(@local, '/', :since => :src)
+    assert_no_match /quicktest/, File.read(File.join(@local, 'README'))
+  end
+  
+  def test_quick_pull_of_file_newer_than_dst_file
+    @ftp.pull_dir(@local, '/')
+    File.open(File.join(Net::FTP.ftp_src, 'README'), 'w') {|f| f.write 'quicktest' }
+    Net::FTP.listing_overrides['/'] = ["-rw-r--r--   1 root     other          0 #{(Time.now + 90).strftime('%b %d %H:%M')} README"]
+    @ftp.pull_dir(@local, '/', :since => :src)
+    assert_match /quicktest/, File.read(File.join(@local, 'README'))
+  end
+  
+  def test_quick_pull_for_first_download
+    @ftp.pull_dir(@local, '/', :since => true)
+    assert File.exist?(File.join(@local, 'README'))
+  end
+  
+  def test_quick_pull_since_date_for_first_download
+    @ftp.pull_dir(@local, '/', :since => Time.now)
+    assert File.exist?(File.join(@local, 'README'))
+  end
+  
+  def test_pushing_files
+    Net::FTP.create_ftp_dst
+    FileUtils.touch(File.join(@local, 'localA'))
+    FileUtils.mkdir_p(File.join(@local, 'localdirA'))
+    FileUtils.touch(File.join(@local, 'localdirA', 'localAA'))
+    @ftp.push_files(@local, '/', ['localA', File.join('localdirA', 'localAA')])
+    assert File.exist?(File.join(Net::FTP.ftp_dst, 'localA'))
+    assert File.exist?(File.join(Net::FTP.ftp_dst, 'localdirA', 'localAA'))
+  end
+  
+  def test_pushing_dir
+    Net::FTP.create_ftp_dst
+    FileUtils.touch(File.join(@local, 'localA'))
+    FileUtils.mkdir_p(File.join(@local, 'localdirA'))
+    FileUtils.touch(File.join(@local, 'localdirA', 'localAA'))
+    @ftp.push_dir(@local, '/')
+    assert File.exist?(File.join(Net::FTP.ftp_dst, 'localA'))
+    assert File.exist?(File.join(Net::FTP.ftp_dst, 'localdirA', 'localAA'))
+  end
+  
+  def test_deleting_files
+    Net::FTP.create_ftp_dst
+    FileUtils.touch File.join(Net::FTP.ftp_dst, 'fileA')
+    FileUtils.mkdir File.join(Net::FTP.ftp_dst, 'dirB')
+    FileUtils.touch File.join(Net::FTP.ftp_dst, 'dirB', 'fileB')
+    FileUtils.touch File.join(Net::FTP.ftp_dst, 'fileC')
+    @ftp.remove_files('/', [ 'fileA', 'dirB/fileB' ])
+    assert !File.exist?(File.join(Net::FTP.ftp_dst, 'fileA'))
+    assert !File.exist?(File.join(Net::FTP.ftp_dst, 'dirB', 'fileB'))
+    assert File.exist?(File.join(Net::FTP.ftp_dst, 'fileC'))
+  end
+    
+  protected
+    def create_tmpname
+      tmpname = ''
+      char_list = ("a".."z").to_a + ("0".."9").to_a
+			1.upto(20) { |i| tmpname << char_list[rand(char_list.size)] }
+			return tmpname
+    end
+end

data/test/net/ftp.rb

@@ -0,0 +1,117 @@
+require 'tmpdir'
+
+module Net
+  class FTP
+    @@listing_overrides = {}
+    
+    class << self
+      
+      def ftp_src
+        @ftp_src ||= File.join(Dir.tmpdir, 'munkey_ftp_src')
+      end
+      
+      def ftp_src=(src)
+        @ftp_src = src
+      end
+      
+      def create_ftp_src
+        FileUtils.mkdir_p File.join(ftp_src, 'dirA', 'dirAA')
+        FileUtils.mkdir_p File.join(ftp_src, 'dirB')
+        FileUtils.touch File.join(ftp_src, 'README')
+        FileUtils.touch File.join(ftp_src, 'fileA')
+        FileUtils.touch File.join(ftp_src, 'fileB')
+        FileUtils.touch File.join(ftp_src, 'dirA', 'fileAA')
+        FileUtils.touch File.join(ftp_src, 'dirA', 'dirAA', 'fileAAA')
+        FileUtils.touch File.join(ftp_src, 'dirB', 'fileBA')
+        FileUtils.touch File.join(ftp_src, 'dirB', 'fileBB')
+      end
+      
+      def ftp_dst
+        @ftp_dst ||= File.join(Dir.tmpdir, 'munkey_ftp_dst')
+      end
+      
+      def ftp_dst=(dst)
+        @ftp_dst = dst
+      end
+    
+      def create_ftp_dst
+        FileUtils.mkdir_p ftp_dst
+      end
+      
+      def listing_overrides
+        @@listing_overrides ||= {}
+      end
+      
+      def listing_overrides=(overrides)
+        @@listing_overrides = overrides
+      end
+    end
+    
+    def initialize(server)
+      raise SocketError unless server == 'test.server'
+    end
+    
+    def inspect
+      "Mocked Net::FTP server=#{@server}"
+    end
+    
+    def login(user, pass)
+      raise Net::FTPPermError unless user == 'user' && pass == 'pass'
+    end
+    
+    def get(src, dst)
+      raise Net::FTPPermError unless File.exist?(src_path(src))
+      FileUtils.cp src_path(src), dst
+    end
+    
+    def put(src, dst)
+      d,f = File.split(dst)
+      raise Net::FTPPermError unless File.exist?(dst_path(d))
+      FileUtils.cp src, dst_path(dst)
+    end
+    
+    def mkdir(dir)
+      d,sd = File.split(dir)
+      raise Net::FTPPermError if File.exist?(dst_path(dir))
+      raise Net::FTPPermError unless File.exist?(dst_path(d))
+      FileUtils.mkdir dst_path(dir)
+    end
+    
+    def chdir(dir)
+      raise Net::FTPPermError unless File.exist?(src_path(dir))
+    end
+    
+    def list(dir)
+      paths = if @@listing_overrides[dir]
+        @@listing_overrides[dir]
+      elsif File.exist?(src_path(dir))
+        `ls -l #{src_path(dir)}`.strip.split("\n")
+      else
+        []
+      end
+
+      paths.each {|e| yield(e) } if block_given?
+      paths
+    end
+    
+    def delete(file)
+      raise Net::FTPPermError unless File.exist?(dst_path(file))
+      File.unlink(dst_path(file))
+    end
+    
+    def close; end
+    
+    private
+      def src_path(p)
+        File.join(self.class.ftp_src, p)
+      end
+      
+      def dst_path(p)
+        File.join(self.class.ftp_dst, p)
+      end
+  end
+  
+  class FTPPermError < RuntimeError; end  
+end
+
+class SocketError < RuntimeError; end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification 
+name: ftp_sync
+version: !ruby/object:Gem::Version 
+  hash: 13
+  prerelease: false
+  segments: 
+  - 0
+  - 4
+  - 1
+  version: 0.4.1
+platform: ruby
+authors: 
+- jebw
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-12 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: net-ftp-list
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 2
+        - 1
+        - 1
+        version: 2.1.1
+  type: :runtime
+  version_requirements: *id001
+description: Library for recursively downloading and uploading entire directories from FTP servers. Supports 'quick' downloads pulling only files changed since a specified date and uploading downloading lists of files. Split out from Munkey - a Git <-> FTP tool
+email: jeb@jdwilkins.co.uk
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- lib/ftp_sync.rb
+- README.rdoc
+- test/ftp_sync_test.rb
+- test/net/ftp.rb
+has_rdoc: true
+homepage: http://github.com/jebw/ftp_sync
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --title
+- Library for syncing files and dirs with ftp servers
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Library for syncing files and dirs with ftp servers
+test_files: 
+- test/ftp_sync_test.rb
+- test/net/ftp.rb