s3ranger 0.1.0

data/lib/s3ranger/sync.rb

@@ -0,0 +1,338 @@
+# (c) 2013  Lincoln de Sousa <lincoln@clarete.li>
+# (c) 2007 s3sync.net
+#
+# This software code is made available "AS IS" without warranties of any
+# kind.  You may copy, display, modify and redistribute the software
+# code either by itself or as incorporated into your code; provided that
+# you do not remove any proprietary notices.  Your use of this software
+# code is at your own risk and you waive any claim against the author
+# with respect to your use of this software code.
+
+require 's3ranger/util'
+require 'fileutils'
+
+module S3Ranger
+
+  class Location
+    attr_accessor :path
+    attr_accessor :bucket
+
+    def initialize path, bucket=nil
+      raise RuntimeError if path.nil?
+      @path = path
+      @bucket = bucket || nil
+    end
+
+    def to_s
+      out = []
+      out << @bucket unless @bucket.nil?
+      out << @path
+      out.join ':'
+    end
+
+    def local?
+      @bucket.nil?
+    end
+
+    def == other
+      @path == other.path and @bucket == other.bucket
+    end
+
+    alias eql? ==
+  end
+
+  class Node
+    include Comparable
+
+    attr_accessor :base
+    attr_accessor :path
+    attr_accessor :size
+
+    def initialize base, path, size
+      @base = base.squeeze '/'
+      @path = path.squeeze '/'
+      @size = size
+    end
+
+    def full
+      S3Ranger.safe_join [@base, @path]
+    end
+
+    def == other
+      full == other.full and @size == other.size
+    end
+
+    def <=> other
+      if self.size < other.size
+        -1
+      elsif self.size > other.size
+        1
+      else
+        0
+      end
+    end
+
+    alias eql? ==
+  end
+
+  class LocalDirectory
+    attr_accessor :source
+
+    def initialize source
+      @source = source
+    end
+
+    def list_files
+      Dir["#{@source}/**/*"].collect { |file|
+        unless File.directory? file
+          file = Pathname.new(file).cleanpath.to_s
+          file_name = file.gsub(/^#{@source}\/?/, '')
+          Node.new @source, file_name, File.stat(file).size
+        end
+      }.compact
+    end
+  end
+
+  class SyncCommand
+
+    def SyncCommand.cmp list1, list2
+      l1 = {}; list1.each {|e| l1[e.path] = e}
+      l2 = {}; list2.each {|e| l2[e.path] = e}
+
+      same, to_add_to_2, to_remove_from_2 = [], [], []
+
+      l1.each do |key, value|
+        value2 = l2.delete key
+        if value2.nil?
+          to_add_to_2 << value
+        elsif value2.size == value.size
+          same << value
+        else
+          to_add_to_2 << value
+        end
+      end
+
+      to_remove_from_2 = l2.values
+
+      [same, to_add_to_2, to_remove_from_2]
+    end
+
+    def initialize args, source, destination
+      @args = args
+      @source = source
+      @destination = destination
+    end
+
+    def run
+      # Reading the source and destination using our helper method
+      if (source, destination, bucket = SyncCommand.parse_params [@source, @destination]).nil?
+        raise WrongUsage.new(nil, 'Need a source and a destination')
+      end
+
+      # Getting the trees
+      source_tree, destination_tree = read_trees source, destination
+
+      # Getting the list of resources to be exchanged between the two peers
+      _, to_add, to_remove = SyncCommand.cmp source_tree, destination_tree
+
+      # Removing the items matching the exclude pattern if requested
+      to_add.select! { |e|
+        begin
+          (e.path =~ /#{@args[:options]["--exclude"]}/).nil?
+        rescue RegexpError => exc
+          raise WrongUsage.new nil, exc.message
+        end
+      } if @args[:options]["--exclude"]
+
+      # Calling the methods that perform the actual IO
+      if source.local?
+        upload_files destination, to_add
+        remove_files destination, to_remove unless @args[:options]["--keep"]
+      else
+        download_files destination, source, to_add
+        remove_local_files destination, source, to_remove unless @args[:options]["--keep"]
+      end
+    end
+
+    def SyncCommand.parse_params args
+      # Reading the arbitrary parameters from the command line and getting
+      # modifiable copies to parse
+      source, destination = args; return nil if source.nil? or destination.nil?
+
+      # Sync from one s3 to another is currently not supported
+      if SyncCommand.remote_prefix? source and SyncCommand.remote_prefix? destination
+        raise WrongUsage.new(nil, 'Both arguments can\'t be on S3')
+      end
+
+      # C'mon, there's rsync out there
+      if !SyncCommand.remote_prefix? source and !SyncCommand.remote_prefix? destination
+        raise WrongUsage.new(nil, 'One argument must be on S3')
+      end
+
+      source, destination = SyncCommand.process_destination source, destination
+      return [Location.new(*source), Location.new(*destination)]
+    end
+
+    def SyncCommand.remote_prefix?(prefix)
+      # allow for dos-like things e.g. C:\ to be treated as local even with
+      # colon.
+      prefix.include? ':' and not prefix.match '^[A-Za-z]:[\\\\/]'
+    end
+
+    def SyncCommand.process_file_destination source, destination, file=""
+      if not file.empty?
+        sub = (remote_prefix? source) ? source.split(":")[1] : source
+        file = file.gsub /^#{sub}/, ''
+      end
+
+      # no slash on end of source means we need to append the last src dir to
+      # dst prefix testing for empty isn't good enough here.. needs to be
+      # "empty apart from potentially having 'bucket:'"
+      if source =~ %r{/$}
+        File.join [destination, file]
+      else
+        if remote_prefix? source
+          _, name = source.split ":"
+          File.join [destination, File.basename(name || ""), file]
+        else
+          source = /^\/?(.*)/.match(source)[1]
+
+          # Corner case: the root of the remote path is empty, we don't want to
+          # add an unnecessary slash here.
+          if destination.end_with? ':'
+            File.join [destination + source, file]
+          else
+            File.join [destination, source, file]
+          end
+        end
+      end
+    end
+
+    def SyncCommand.process_destination source, destination
+      source, destination = source.dup, destination.dup
+
+      # don't repeat slashes
+      source.squeeze! '/'
+      destination.squeeze! '/'
+
+      # Making sure that local paths won't break our stuff later
+      source.gsub! /^\.\//, ''
+      destination.gsub! /^\.\//, ''
+
+      # Parsing the final destination
+      destination = SyncCommand.process_file_destination source, destination, ""
+
+      # here's where we find out what direction we're going
+      source_is_s3 = remote_prefix? source
+
+      # alias these variables to the other strings (in ruby = does not make
+      # copies of strings)
+      remote_prefix = source_is_s3 ? source : destination
+      local_prefix = source_is_s3 ? destination : source
+
+      # canonicalize the S3 stuff
+      bucket, remote_prefix = remote_prefix.split ":"
+      remote_prefix ||= ""
+
+      # Just making sure we preserve the direction
+      if source_is_s3
+        [[remote_prefix, bucket], destination]
+      else
+        [source, [remote_prefix, bucket]]
+      end
+    end
+
+    def read_tree_remote location
+      begin
+        dir = location.path
+        dir += '/' if not (dir.empty? or dir.end_with? '/')
+        @args[:s3].buckets[location.bucket].objects.with_prefix(dir || "").to_a.collect {|obj|
+          Node.new location.path, obj.key, obj.content_length
+        }
+      rescue AWS::S3::Errors::NoSuchBucket
+        raise FailureFeedback.new("There's no bucket named `#{location.bucket}'")
+      rescue AWS::S3::Errors::NoSuchKey
+        raise FailureFeedback.new("There's no key named `#{location.path}' in the bucket `#{location.bucket}'")
+      rescue AWS::S3::Errors::AccessDenied
+        raise FailureFeedback.new("Access denied")
+      end
+    end
+
+    def read_trees source, destination
+      if source.local?
+        source_tree = LocalDirectory.new(source.path).list_files
+        destination_tree = read_tree_remote destination
+      else
+        source_tree = read_tree_remote source
+        destination_tree = LocalDirectory.new(destination.path).list_files
+      end
+
+      [source_tree, destination_tree]
+    end
+
+    def upload_files remote, list
+      list.each do |e|
+        if @args[:options]["--dry-run"] or @args[:options]["--verbose"]
+          puts " + #{e.full} => #{remote}#{e.path}"
+        end
+
+        unless @args[:options]["--dry-run"]
+          if File.file? e.path
+            @args[:s3].buckets[remote.bucket].objects[e.path].write Pathname.new e.path
+          end
+        end
+      end
+    end
+
+    def remove_files remote, list
+
+      if @args[:options]["--dry-run"] or @args[:options]["--verbose"]
+        list.each {|e|
+          puts " - #{remote}#{e.path}"
+        }
+      end
+
+      unless @args[:options]["--dry-run"]
+        @args[:s3].buckets[remote.bucket].objects.delete_if { |obj| list.include? obj.key }
+      end
+    end
+
+    def download_files destination, source, list
+      list.each {|e|
+        path = File.join destination.path, e.path
+
+        if @args[:options]["--dry-run"] or @args[:options]["--verbose"]
+          puts " + #{source}#{e.path} => #{path}"
+        end
+
+        unless @args[:options]["--dry-run"]
+          obj = @args[:s3].buckets[source.bucket].objects[e.path]
+
+          # Making sure this new file will have a safe shelter
+          FileUtils.mkdir_p File.dirname(path)
+
+          # Downloading and saving the files
+          File.open(path, 'wb') do |file|
+            obj.read do |chunk|
+              file.write chunk
+            end
+          end
+        end
+      }
+    end
+
+    def remove_local_files destination, source, list
+      list.each {|e|
+        path = File.join destination.path, e.path
+
+        if @args[:options]["--dry-run"] or @args[:options]["--verbose"]
+          puts " * #{e.path} => #{path}"
+        end
+
+        unless @args[:options]["--dry-run"]
+          FileUtils.rm_rf path
+        end
+      }
+    end
+  end
+end

data/lib/s3ranger/util.rb

@@ -0,0 +1,5 @@
+module S3Ranger
+  def S3Ranger.safe_join(parts)
+    File.join *(parts.select {|v| !v.nil? && !v.empty? })
+  end
+end

data/lib/s3ranger/version.rb

@@ -0,0 +1,3 @@
+module S3Ranger
+  VERSION = "0.1.0"
+end

data/lib/s3ranger.rb

@@ -0,0 +1,2 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))

data/s3config.yml.example

@@ -0,0 +1,3 @@
+aws_access_key_id: 11111111111111111111111
+aws_secret_access_key: 222222222222222222222
+ssl_cert_dir: /home/user/s3ranger/certs

data/s3ranger.gemspec

@@ -0,0 +1,31 @@
+# -*- mode: ruby; coding: utf-8; -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 's3ranger/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "s3ranger"
+  spec.version       = S3Ranger::VERSION
+  spec.authors       = ["Lincoln de Sousa"]
+  spec.email         = ["lincoln@comum.org"]
+  spec.description   = 'Tool belt for managing your S3 buckets'
+  spec.summary       = 's3ranger is a library that aggregates a good range of features for managing your Amazon S3 buckets. It also provides basic interactive client'
+
+  spec.homepage      = "http://github.com/clarete/s3ranger"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  # Library requirements
+  spec.add_dependency "aws-sdk"
+
+  # Development requirements
+  spec.add_development_dependency "debugger"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/spec/fixtures/nodes/1.txt

@@ -0,0 +1 @@
+s3ranger

data/spec/local_source_spec.rb

@@ -0,0 +1,55 @@
+require 'spec_helper.rb'
+require 's3ranger/sync'
+
+include S3Ranger
+
+
+describe "Local file system IO" do
+
+  it "should list local files" do
+
+    # Given that I have remote source and a local destination with a couple
+    # files
+    source = "mybucket:path"
+    destination = directory "directory2"
+    file destination, "file1.txt", "First file"
+    file destination, "file2.txt", "Second file"
+
+    # When I create a new local directory based on that path
+    local = LocalDirectory.new destination
+
+    # Then I see that the directory nodes contain both their parent paths and
+    # their names
+    local.list_files.should be_eql [
+      Node.new(fixture("directory2"), "file1.txt", 10),
+      Node.new(fixture("directory2"), "file2.txt", 11),
+    ]
+
+    rm destination
+  end
+
+  it "should skip local folders while listing files" do
+    # Given that I have remote source and a local destination with files
+    source = "mybucket:path"
+    destination = directory "directory2"
+    file destination, "file1.txt", "First file"
+    file destination, "file2.txt", "Second file"
+
+    # And with a sub-directory
+    subdir = directory "directory2/subd"
+    file subdir, "sub1.txt", "Sub content"
+
+    # When I create a new local directory based on that path
+    local = LocalDirectory.new destination
+
+    # Then I see that the directory nodes contain both their parent paths and
+    # their names
+    local.list_files.should be_eql [
+      Node.new(fixture("directory2"), "file1.txt", 10),
+      Node.new(fixture("directory2"), "file2.txt", 11),
+      Node.new(fixture("directory2"), "subd/sub1.txt", 11),
+    ]
+
+    rm destination
+  end
+end

data/spec/main_spec.rb

@@ -0,0 +1,141 @@
+require 'spec_helper.rb'
+require 's3ranger/cmd'
+require 's3ranger/config'
+require 's3ranger/commands'
+require 's3ranger/sync'
+
+include S3Ranger
+
+describe "Parsing command line arguments" do
+
+  describe "Processing the final destination based on how the user expressed the source" do
+
+    it "Put the local etc directory itself into S3" do
+      source = "/etc"
+      destination = "mybucket:pre"
+
+      # This will yield S3 keys named  pre/etc/...
+      SyncCommand.process_destination(source, destination).should be_eql ["/etc", ["pre/etc/", "mybucket"]]
+    end
+
+    it "Put the contents of the local /etc dir into S3, rename dir" do
+      source = "/etc/"
+      destination = "mybucket:pre/etcbackup"
+
+      # This will yield S3 keys named  pre/etcbackup/...
+      SyncCommand.process_destination(source, destination).should be_eql ["/etc/", ["pre/etcbackup/", "mybucket"]]
+    end
+
+    it "Put contents of S3 \"directory\" etc into local dir" do
+      source = "mybucket:pre/etc/"
+      destination = "/root/etcrestore"
+
+      # This will yield local files at  /root/etcrestore/...
+      SyncCommand.process_destination(source, destination).should be_eql [["pre/etc/", "mybucket"], "/root/etcrestore/"]
+    end
+
+    it "Put the contents of S3 \"directory\" etc into a local dir named etc" do
+      source = "mybucket:pre/etc"
+      destination = "/root"
+
+      # This will yield local files at  /root/etc/...
+      SyncCommand.process_destination(source, destination).should be_eql [["pre/etc", "mybucket"], "/root/etc/"]
+    end
+
+    it "Put S3 nodes under the key pre/etc/ to the local dir etcrestore" do
+      source = "mybucket:pre/etc/"
+      destination = "/root/etcrestore"
+
+      # This will yield local files at  /root/etcrestore/...
+      SyncCommand.process_destination(source, destination).should be_eql [["pre/etc/", "mybucket"], "/root/etcrestore/"]
+    end
+
+    it "Put S3 nodes under an empty key (root) to the local dir /tmp/lib" do
+      source = "mybucket:"
+      destination = "/tmp/lib"
+
+      # This will yield local files at  /root/etcrestore/...
+      SyncCommand.process_destination(source, destination).should be_eql [["", "mybucket"], "/tmp/lib/"]
+    end
+  end
+
+  it "Should calculate the right destination for each path" do
+    file = "pre/etc/sub/path/blah.txt"   # This is how it comes from s3
+    source = "mybucket:pre/etc/"
+    destination = "/root/etcrestore"
+
+    SyncCommand.process_file_destination(source, destination, file).should be_eql "/root/etcrestore/sub/path/blah.txt"
+  end
+
+  it "Put S3 files under an empty key (root) to the local dir /tmp/lib" do
+    source = "mybucket:"
+    destination = "/tmp/lib"
+    file = "myfile.rb"
+
+    # This will yield local files at  /tmp/lib/...
+    SyncCommand.process_file_destination(source, destination, file).should be_eql "/tmp/lib/myfile.rb"
+  end
+
+  it "Returning locations based on the parsed destination" do
+    source = "/etc"
+    destination = "mybucket:pre"
+
+    # When I parse the above arguments using the SyncCommand
+    src_location, dst_location = SyncCommand.parse_params [source, destination]
+
+    # Then I see I got the locations with the right params
+    src_location.should be_eql S3Ranger::Location.new("/etc")
+    dst_location.should be_eql S3Ranger::Location.new("pre/etc/", "mybucket")
+  end
+
+  it "Location should be parsed when it is remote with no path" do
+    source = "/etc"
+    destination = "mybucket:"
+
+    # When I parse the above arguments using the SyncCommand
+    src_location, dst_location = SyncCommand.parse_params [source, destination]
+
+    # Then I see I got the locations with the right params
+    src_location.should be_eql S3Ranger::Location.new("/etc")
+    dst_location.should be_eql S3Ranger::Location.new("etc/", "mybucket")
+  end
+
+  it "should be possible to detect if a location is remote" do
+    SyncCommand.remote_prefix?("bucket:prefix").should be_true
+    SyncCommand.remote_prefix?("path").should be_false
+    SyncCommand.remote_prefix?("C://blah").should be_false  # We support windows, LOL
+  end
+end
+
+describe "Comparing file lists" do
+
+  it "should be possible to describe nodes with their paths and size" do
+
+    # Full test
+    node = Node.new "path//to", "file1", 10
+    node.path.should be_eql "file1"
+    node.full.should be_eql "path/to/file1"
+    node.size.should be_eql 10
+
+    # Alternative constructor scenarios
+    node = Node.new "", "file1", 10
+    node.path.should be_eql "file1"
+  end
+
+  it "should be possible to compare two lists of files" do
+
+    # Given that I have two lists of Nodes to compare
+    list1 = [Node.new("", "file1", 10), Node.new("", "file2", 12), Node.new("", "file3", 12)]
+    list2 = [Node.new("", "file1", 10), Node.new("", "file2", 22), Node.new("", "file4", 22),]
+
+    # When I compare those two file lists
+    same_in_both, to_be_added_to_list2, to_be_removed_from_list2 = SyncCommand.cmp list1, list2
+
+    # Then I see that the three lists that I requested were returned with the
+    # right content
+    same_in_both.should == [Node.new("", "file1", 10)]     # Just testing our == operator
+    same_in_both.should be_eql [Node.new("", "file1", 10)]
+    to_be_added_to_list2.should be_eql [Node.new("", "file2", 12), Node.new("", "file3", 12)]
+    to_be_removed_from_list2.should be_eql [Node.new("", "file4", 22)]
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,25 @@
+require 'fileutils'
+require 'simplecov'
+SimpleCov.start
+
+def fixture *args
+  File.join File.dirname(__FILE__), "fixtures", *args
+end
+
+def directory path
+  full = fixture(path)
+  FileUtils::mkdir_p full
+  return full
+end
+
+def file *args
+  file = File.join *args[0..-2]
+  directory File.dirname(file)
+  File.open(file, 'w') {|f| f.write args[-1] }
+  return file
+end
+
+
+def rm path
+  FileUtils::rm_rf path
+end