s3ranger 0.3.0 → 0.3.1

data/lib/s3ranger/cli.rb

@@ -62,6 +62,15 @@ module S3Ranger
 
         u.join ''
       end
+
+      protected
+
+      def parse_acl(opt)
+        @acl = nil
+        opt.on("-a", "--acl=ACL", "Options: #{AVAILABLE_ACLS.join ', '}") {|acl|
+          @acl = acl.to_sym
+        }
+      end
     end
 
     class ListBuckets < BaseCmd
@@ -86,12 +95,8 @@ module S3Ranger
 
         @short_desc = "Create a new bucket under your user account"
 
-        @acl = nil
-
         self.options = CmdParse::OptionParserWrapper.new do |opt|
-          opt.on("-a", "--acl=ACL", "Options: #{AVAILABLE_ACLS.join ', '}") {|acl|
-            @acl = acl.to_sym
-          }
+          parse_acl(opt)
         end
       end
 
@@ -106,7 +111,7 @@ module S3Ranger
           end
 
           s3.buckets.create bucket, params
-        rescue AWS::S3::Errors::BucketAlreadyExists => exc
+        rescue AWS::S3::Errors::BucketAlreadyExists
           raise FailureFeedback.new("Bucket `#{bucket}' already exists")
         end
       end
@@ -175,7 +180,7 @@ module S3Ranger
         collection = s3.buckets[bucket].objects.with_prefix(key || "")
 
         if @max_entries > 0
-          collection = collection.page(:per_page => max = @max_entries)
+          collection = collection.page(:per_page => @max_entries)
         end
 
         collection.each {|object|
@@ -236,7 +241,7 @@ module S3Ranger
 
           opt.on("--expires-in=EXPR", "How long the link takes to expire. Format: <# of seconds> | [#d|#h|#m|#s]") { |expr|
             val = 0
-            expr.scan /(\d+\w)/ do |track|
+            expr.scan(/(\d+\w)/) do |track|
               _, num, what = /(\d+)(\w)/.match(track[0]).to_a
               num = num.to_i
 
@@ -264,10 +269,10 @@ module S3Ranger
         opts.merge!({:secure => @secure})
 
         if @public
-          puts (s3.buckets[bucket].objects[key].public_url opts).to_s
+          puts s3.buckets[bucket].objects[key].public_url(opts).to_s
         else
           opts.merge!({:expires => @expires_in}) if @expires_in
-          puts (s3.buckets[bucket].objects[key].url_for method.to_sym, opts).to_s
+          puts s3.buckets[bucket].objects[key].url_for(method.to_sym, opts).to_s
         end
       end
     end
@@ -317,6 +322,7 @@ module S3Ranger
       attr_accessor :keep
       attr_accessor :dry_run
       attr_accessor :verbose
+      attr_accessor :acl
 
       def initialize
         super 'sync', false, false
@@ -337,6 +343,8 @@ module S3Ranger
             @keep = true
           }
 
+          parse_acl(opt)
+
           opt.on("-d", "--dry-run", "Do not download or exclude anything, just show what was planned. Implies `verbose`.") {
             @dry_run = true
             @verbose = true

data/lib/s3ranger/sync.rb

@@ -121,11 +121,11 @@ module S3Ranger
 
   class SyncCommand
 
-    def SyncCommand.cmp list1, list2
+    def self.cmp list1, list2
       l1 = {}; list1.each {|e| l1[e.path] = e}
       l2 = {}; list2.each {|e| l2[e.path] = e}
 
-      same, to_add_to_2, to_remove_from_2 = [], [], []
+      same, to_add_to_2 = [], []
 
       l1.each do |key, value|
         value2 = l2.delete key
@@ -151,7 +151,7 @@ module S3Ranger
 
     def run
       # Reading the source and destination using our helper method
-      if (source, destination, bucket = SyncCommand.parse_params [@source, @destination]).nil?
+      if (source, destination, bucket = self.class.parse_params [@source, @destination]).nil?
         raise WrongUsage.new(nil, 'Need a source and a destination')
       end
 
@@ -159,7 +159,7 @@ module S3Ranger
       source_tree, destination_tree = read_trees source, destination
 
       # Getting the list of resources to be exchanged between the two peers
-      _, to_add, to_remove = SyncCommand.cmp source_tree, destination_tree
+      _, to_add, to_remove = self.class.cmp source_tree, destination_tree
 
       # Removing the items matching the exclude pattern if requested
       to_add.select! { |e|
@@ -180,35 +180,35 @@ module S3Ranger
       end
     end
 
-    def SyncCommand.parse_params args
+    def self.parse_params args
       # Reading the arbitrary parameters from the command line and getting
       # modifiable copies to parse
       source, destination = args; return nil if source.nil? or destination.nil?
 
       # Sync from one s3 to another is currently not supported
-      if SyncCommand.remote_prefix? source and SyncCommand.remote_prefix? destination
+      if remote_prefix? source and remote_prefix? destination
         raise WrongUsage.new(nil, 'Both arguments can\'t be on S3')
       end
 
       # C'mon, there's rsync out there
-      if !SyncCommand.remote_prefix? source and !SyncCommand.remote_prefix? destination
+      if !remote_prefix? source and !remote_prefix? destination
         raise WrongUsage.new(nil, 'One argument must be on S3')
       end
 
-      source, destination = SyncCommand.process_destination source, destination
+      source, destination = process_destination source, destination
       return [Location.new(*source), Location.new(*destination)]
     end
 
-    def SyncCommand.remote_prefix?(prefix)
+    def self.remote_prefix?(prefix)
       # allow for dos-like things e.g. C:\ to be treated as local even with
       # colon.
       prefix.include? ':' and not prefix.match '^[A-Za-z]:[\\\\/]'
     end
 
-    def SyncCommand.process_file_destination source, destination, file=""
+    def self.process_file_destination source, destination, file=""
       if not file.empty?
         sub = (remote_prefix? source) ? source.split(":")[1] : source
-        file = file.gsub /^#{sub}/, ''
+        file = file.gsub(/^#{sub}/, '')
       end
 
       # no slash on end of source means we need to append the last src dir to
@@ -234,7 +234,7 @@ module S3Ranger
       end
     end
 
-    def SyncCommand.process_destination source, destination
+    def self.process_destination source, destination
       source, destination = source.dup, destination.dup
 
       # don't repeat slashes
@@ -242,21 +242,17 @@ module S3Ranger
       destination.squeeze! '/'
 
       # Making sure that local paths won't break our stuff later
-      source.gsub! /^\.\//, ''
-      destination.gsub! /^\.\//, ''
+      source.gsub!(/^\.\//, '')
+      destination.gsub!(/^\.\//, '')
 
       # Parsing the final destination
-      destination = SyncCommand.process_file_destination source, destination, ""
+      destination = process_file_destination source, destination, ""
 
       # here's where we find out what direction we're going
       source_is_s3 = remote_prefix? source
 
-      # alias these variables to the other strings (in ruby = does not make
-      # copies of strings)
-      remote_prefix = source_is_s3 ? source : destination
-      local_prefix = source_is_s3 ? destination : source
-
       # canonicalize the S3 stuff
+      remote_prefix = source_is_s3 ? source : destination
       bucket, remote_prefix = remote_prefix.split ":"
       remote_prefix ||= ""
 
@@ -269,19 +265,11 @@ module S3Ranger
     end
 
     def read_tree_remote location
-      begin
-        dir = location.path
-        dir += '/' if not (dir.empty? or dir.end_with? '/')
-        @args.s3.buckets[location.bucket].objects.with_prefix(dir || "").to_a.collect {|obj|
-          Node.new location.path, obj.key, obj.content_length
-        }
-      rescue AWS::S3::Errors::NoSuchBucket
-        raise FailureFeedback.new("There's no bucket named `#{location.bucket}'")
-      rescue AWS::S3::Errors::NoSuchKey
-        raise FailureFeedback.new("There's no key named `#{location.path}' in the bucket `#{location.bucket}'")
-      rescue AWS::S3::Errors::AccessDenied
-        raise FailureFeedback.new("Access denied")
-      end
+      dir = location.path
+      dir += '/' if not dir.empty? or dir.end_with?('/')
+      @args.s3.buckets[location.bucket].objects.with_prefix(dir || "").to_a.collect {|obj|
+        Node.new location.path, obj.key, obj.content_length
+      }
     end
 
     def read_trees source, destination
@@ -304,7 +292,7 @@ module S3Ranger
 
         unless @args.dry_run
           if File.file? e.path
-            @args.s3.buckets[remote.bucket].objects[e.path].write Pathname.new e.path
+            @args.s3.buckets[remote.bucket].objects[e.path].write Pathname.new(e.path), :acl => @args.acl
           end
         end
       end

data/lib/s3ranger/util.rb

@@ -24,6 +24,6 @@
 
 module S3Ranger
   def S3Ranger.safe_join(parts)
-    File.join *(parts.select {|v| !v.nil? && !v.empty? })
+    File.join(*(parts.select {|v| !v.nil? && !v.empty? }))
   end
 end

data/lib/s3ranger/version.rb

@@ -23,5 +23,5 @@
 # THE SOFTWARE.
 
 module S3Ranger
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3ranger
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-30 00:00:00.000000000 Z
+date: 2013-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -123,6 +123,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: bump
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Tool belt for managing your S3 buckets
 email:
 - lincoln@comum.org
@@ -131,15 +147,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- Gemfile
-- Gemfile.lock
-- LICENSE.txt
-- README.md
-- Rakefile
 - bin/s3ranger
-- doc/old/README.txt
-- doc/old/README_s3cmd.txt
 - lib/s3ranger.rb
 - lib/s3ranger/cli.rb
 - lib/s3ranger/config.rb
@@ -147,13 +155,7 @@ files:
 - lib/s3ranger/sync.rb
 - lib/s3ranger/util.rb
 - lib/s3ranger/version.rb
-- s3ranger.gemspec
-- s3ranger.yml.example
-- spec/fixtures/nodes/1.txt
-- spec/local_source_spec.rb
-- spec/main_spec.rb
-- spec/spec_helper.rb
-homepage: http://github.com/clarete/s3ranger
+homepage: https://github.com/clarete/s3ranger
 licenses:
 - MIT
 post_install_message: 
@@ -168,7 +170,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1697485139363659678
+      hash: -3161220676864238924
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -177,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1697485139363659678
+      hash: -3161220676864238924
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
@@ -185,8 +187,4 @@ signing_key:
 specification_version: 3
 summary: s3ranger is a library that aggregates a good range of features for managing
   your Amazon S3 buckets. It also provides basic interactive client
-test_files:
-- spec/fixtures/nodes/1.txt
-- spec/local_source_spec.rb
-- spec/main_spec.rb
-- spec/spec_helper.rb
+test_files: []

data/.gitignore

@@ -1,17 +0,0 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
-Gemfile.lock
-InstalledFiles
-_yardoc
-coverage
-doc/
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp

data/Gemfile

@@ -1,4 +0,0 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in s3ranger.gemspec
-gemspec

data/Gemfile.lock

@@ -1,51 +0,0 @@
-PATH
-  remote: .
-  specs:
-    s3ranger (0.3.0)
-      aws-sdk
-      cmdparse
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    aws-sdk (1.16.0)
-      json (~> 1.4)
-      nokogiri (< 1.6.0)
-      uuidtools (~> 2.1)
-    cmdparse (2.0.5)
-    columnize (0.3.6)
-    debugger (1.6.1)
-      columnize (>= 0.3.1)
-      debugger-linecache (~> 1.2.0)
-      debugger-ruby_core_source (~> 1.2.3)
-    debugger-linecache (1.2.0)
-    debugger-ruby_core_source (1.2.3)
-    diff-lcs (1.2.4)
-    json (1.8.0)
-    multi_json (1.7.9)
-    nokogiri (1.5.10)
-    rake (10.1.0)
-    rspec (2.14.1)
-      rspec-core (~> 2.14.0)
-      rspec-expectations (~> 2.14.0)
-      rspec-mocks (~> 2.14.0)
-    rspec-core (2.14.5)
-    rspec-expectations (2.14.2)
-      diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.14.3)
-    simplecov (0.7.1)
-      multi_json (~> 1.0)
-      simplecov-html (~> 0.7.1)
-    simplecov-html (0.7.1)
-    uuidtools (2.1.4)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (~> 1.3)
-  debugger
-  rake
-  rspec
-  s3ranger!
-  simplecov

data/LICENSE.txt

@@ -1,22 +0,0 @@
-Copyright (c) 2013 Lincoln de Sousa <lincoln@clarete.li>
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -1,29 +0,0 @@
-# S3Ranger
-
-TODO: Write a gem description
-
-## Installation
-
-Add this line to your application's Gemfile:
-
-    gem 's3ranger'
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install s3ranger
-
-## Usage
-
-TODO: Write usage instructions here
-
-## Contributing
-
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request

data/Rakefile

@@ -1 +0,0 @@
-require "bundler/gem_tasks"

data/doc/old/README.txt

@@ -1,406 +0,0 @@
-Welcome to s3ranger.rb
---------------------
-Home page, wiki, forum, bug reports, etc: http://s3sync.net
-
-This is a ruby program that easily transfers directories between a local
-directory and an S3 bucket:prefix. It behaves somewhat, but not precisely, like
-the rsync program. In particular, it shares rsync's peculiar behavior that
-trailing slashes on the source side are meaningful. See examples below.
-
-One benefit over some other comparable tools is that s3ranger goes out of its way
-to mirror the directory structure on S3.  Meaning you don't *need* to use s3ranger
-later in order to view your files on S3.  You can just as easily use an S3
-shell, a web browser (if you used the --public-read option), etc.  Note that
-s3ranger is NOT necessarily going to be able to read files you uploaded via some
-other tool.  This includes things uploaded with the old perl version!  For best
-results, start fresh!
-
-s3ranger runs happily on linux, probably other *ix, and also Windows (except that
-symlinks and permissions management features don't do anything on Windows). If
-you get it running somewhere interesting let me know (see below)
-
-s3ranger is free, and license terms are included in all the source files. If you
-decide to make it better, or find bugs, please let me know.
-
-The original inspiration for this tool is the perl script by the same name which
-was made by Thorsten von Eicken (and later updated by me). This ruby program
-does not share any components or logic from that utility; the only relation is
-that it performs a similar task.
-
-
-Examples:
----------
-(using S3 bucket 'mybucket' and prefix 'pre')
-  Put the local etc directory itself into S3
-        s3ranger.rb  -r  /etc  mybucket:pre
-        (This will yield S3 keys named  pre/etc/...)
-  Put the contents of the local /etc dir into S3, rename dir:
-        s3ranger.rb  -r  /etc/  mybucket:pre/etcbackup
-        (This will yield S3 keys named  pre/etcbackup/...)
-  Put contents of S3 "directory" etc into local dir
-        s3ranger.rb  -r  mybucket:pre/etc/  /root/etcrestore
-        (This will yield local files at  /root/etcrestore/...)
-  Put the contents of S3 "directory" etc into a local dir named etc
-        s3ranger.rb  -r  mybucket:pre/etc  /root
-        (This will yield local files at  /root/etc/...)
-  Put S3 nodes under the key pre/etc/ to the local dir etcrestore
-  **and create local dirs even if S3 side lacks dir nodes**
-        s3ranger.rb  -r  --make-dirs  mybucket:pre/etc/  /root/etcrestore
-        (This will yield local files at  /root/etcrestore/...)
-
-
-Prerequisites:
---------------
-You need a functioning Ruby (>=1.8.4) installation, as well as the OpenSSL ruby
-library (which may or may not come with your ruby).
-
-How you get these items working on your system is really not any of my
-business, but you might find the following things helpful.  If you're using
-Windows, the ruby site has a useful "one click installer" (although it takes
-more clicks than that, really).  On debian (and ubuntu, and other debian-like
-things), there are apt packages available for ruby and the open ssl lib.
-
-
-Your environment:
------------------
-s3ranger needs to know several interesting values to work right.  It looks for
-them in the following environment variables -or- a s3config.yml file.
-In the yml case, the names need to be lowercase (see example file).
-Furthermore, the yml is searched for in the following locations, in order:
-   $S3CONF/s3config.yml
-   $HOME/.s3conf/s3config.yml
-   /etc/s3conf/s3config.yml
-
-Required:
-	AWS_ACCESS_KEY_ID
-	AWS_SECRET_ACCESS_KEY
-
-	If you don't know what these are, then s3ranger is probably not the
-	right tool for you to be starting out with.
-Optional:
-	AWS_S3_HOST - I don't see why the default would ever be wrong
-   HTTP_PROXY_HOST,HTTP_PROXY_PORT,HTTP_PROXY_USER,HTTP_PROXY_PASSWORD - proxy
-	SSL_CERT_DIR - Where your Cert Authority keys live; for verification
-	SSL_CERT_FILE - If you have just one PEM file for CA verification
-	S3SYNC_RETRIES - How many HTTP errors to tolerate before exiting
-	S3SYNC_WAITONERROR - How many seconds to wait after an http error
-	S3SYNC_MIME_TYPES_FILE - Where is your mime.types file
-	S3SYNC_NATIVE_CHARSET - For example Windows-1252.  Defaults to ISO-8859-1.
-   AWS_CALLING_FORMAT - Defaults to REGULAR
-       REGULAR   # http://s3.amazonaws.com/bucket/key
-       SUBDOMAIN # http://bucket.s3.amazonaws.com/key
-       VANITY    # http://<vanity_domain>/key
-
-Important: For EU-located buckets you should set the calling format to SUBDOMAIN
-Important: For US buckets with CAPS or other weird traits set the calling format
-to REGULAR
-
-I use "envdir" from the daemontools package to set up my env
-variables easily: http://cr.yp.to/daemontools/envdir.html
-For example:
-	envdir /root/s3ranger/env /root/s3ranger/s3ranger.rb -etc etc etc
-I know there are other similar tools out there as well.
-
-You can also just call it in a shell script where you have exported the vars
-first such as:
-#!/bin/bash
-export AWS_ACCESS_KEY_ID=valueGoesHere
-...
-s3ranger.rb -etc etc etc
-
-But by far the easiest (and newest) way to set this up is to put the name:value
-pairs in a file named s3config.yml and let the yaml parser pick them up. There
-is an .example file shipped with the tar.gz to show what a yaml file looks like.
-Thanks to Alastair Brunton for this addition.
-
-You can also use some combination of .yaml and environment variables, if you
-want. Go nuts.
-
-
-Management tasks
-----------------
-For low-level S3 operations not encapsulated by the sync paradigm, try the
-companion utility s3cmd.rb. See README_s3cmd.txt.
-
-
-About single files
-------------------
-s3ranger lacks the special case code that would be needed in order to handle a
-source/dest that's a single file. This isn't one of the supported use cases so
-don't expect it to work. You can use the companion utility s3cmd.rb for single
-get/puts.
-
-
-About Directories, the bane of any S3 sync-er
----------------------------------------------
-In S3 there's no actual concept of folders, just keys and nodes. So, every tool
-uses its own proprietary way of storing dir info (my scheme being the best
-naturally) and in general the methods are not compatible.
-
-If you populate S3 by some means *other than* s3ranger and then try to use s3ranger
-to "get" the S3 stuff to a local filesystem, you will want to use the
---make-dirs option. This causes the local dirs to be created even if there is no
-s3ranger-compatible directory node info stored on the S3 side. In other words,
-local folders are conjured into existence whenever they are needed to make the
-"get" succeed.
-
-
-About MD5 hashes
-----------------
-s3ranger's normal operation is to compare the file size and MD5 hash of each item
-to decide whether it needs syncing.  On the S3 side, these hashes are stored and
-returned to us as the "ETag" of each item when the bucket is listed, so it's
-very easy.  On the local side, the MD5 must be calculated by pushing every byte
-in the file through the MD5 algorithm.  This is CPU and IO intensive!
-
-Thus you can specify the option --no-md5. This will compare the upload time on
-S3 to the "last modified" time on the local item, and not do md5 calculations
-locally at all. This might cause more transfers than are absolutely necessary.
-For example if the file is "touched" to a newer modified date, but its contents
-didn't change. Conversely if a file's contents are modified but the date is not
-updated, then the sync will pass over it.  Lastly, if your clock is very
-different from the one on the S3 servers, then you may see unanticipated
-behavior.
-
-
-A word on SSL_CERT_DIR:
------------------------
-On my debian install I didn't find any root authority public keys.  I installed
-some by running this shell archive:
-http://mirbsd.mirsolutions.de/cvs.cgi/src/etc/ssl.certs.shar
-(You have to click download, and then run it wherever you want the certs to be
-placed).  I do not in any way assert that these certificates are good,
-comprehensive, moral, noble, or otherwise correct.  But I am using them.
-
-If you don't set up a cert dir, and try to use ssl, then you'll 1) get an ugly
-warning message slapped down by ruby, and 2) not have any protection AT ALL from
-malicious servers posing as s3.amazonaws.com.  Seriously... you want to get
-this right if you're going to have any sensitive data being tossed around.
---
-There is a debian package ca-certificates; this is what I'm using now.
-apt-get install ca-certificates
-and then use:
-SSL_CERT_DIR=/etc/ssl/certs
-
-You used to be able to use just one certificate, but recently AWS has started
-using more than one CA.
-
-
-Getting started:
-----------------
-Invoke by typing s3ranger.rb and you should get a nice usage screen.
-Options can be specified in short or long form (except --delete, which has no
-short form)
-
-ALWAYS TEST NEW COMMANDS using --dryrun(-n) if you want to see what will be
-affected before actually doing it. ESPECIALLY if you use --delete. Otherwise, do
-not be surprised if you misplace a '/' or two and end up deleting all your
-precious, precious files.
-
-If you use the --public-read(-p) option, items sent to S3 will be ACL'd so that
-anonymous web users can download them, given the correct URL. This could be
-useful if you intend to publish directories of information for others to see.
-For example, I use s3ranger to publish itself to its home on S3 via the following
-command: s3ranger.rb -v -p publish/ ServEdge_pub:s3ranger Where the files live in a
-local folder called "publish" and I wish them to be copied to the URL:
-http://s3.amazonaws.com/ServEdge_pub/s3ranger/... If you use --ssl(-s) then your
-connections with S3 will be encrypted. Otherwise your data will be sent in clear
-form, i.e. easy to intercept by malicious parties.
-
-If you want to prune items from the destination side which are not found on the
-source side, you can use --delete. Always test this with -n first to make sure
-the command line you specify is not going to do something terrible to your
-cherished and irreplaceable data.
-
-
-Updates and other discussion:
------------------------------
-The latest version of s3ranger should normally be at:
-	http://s3.amazonaws.com/ServEdge_pub/s3ranger/s3ranger.tar.gz
-and the Amazon S3 forums probably have a few threads going on it at any given
-time. I may not always see things posted to the threads, so if you want you can
-contact me at gbs-s3@10forward.com too.
-
-
-Change Log:
------------
-
-2006-09-29:
-Added support for --expires and --cache-control. Eg:
---expires="Thu, 01 Dec 2007 16:00:00 GMT"
---cache-control="no-cache"
-
-Thanks to Charles for pointing out the need for this, and supplying a patch
-proving that it would be trivial to add =) Apologies for not including the short
-form (-e) for the expires. I have a rule that options taking arguments should
-use the long form.
-----------
-
-2006-10-04
-Several minor debugs and edge cases.
-Fixed a bug where retries didn't rewind the stream to start over.
-----------
-
-2006-10-12
-Version 1.0.5
-Finally figured out and fixed bug of trying to follow local symlink-to-directory.
-Fixed a really nasty sorting discrepancy that caused problems when files started
-with the same name as a directory.
-Retry on connection-reset on the S3 side.
-Skip files that we can't read instead of dying.
-----------
-
-2006-10-12
-Version 1.0.6
-Some GC voodoo to try and keep a handle on the memory footprint a little better.
-There is still room for improvement here.
-----------
-
-2006-10-13
-Version 1.0.7
-Fixed symlink dirs being stored to S3 as real dirs (and failing with 400)
-Added a retry catch for connection timeout error.
-(Hopefully) caught a bug that expected every S3 listing to contain results
-----------
-
-2006-10-14
-Version 1.0.8
-Was testing for file? before symlink? in localnode.stream. This meant that for
-symlink files it was trying to shove the real file contents into the symlink
-body on s3.
-----------
-
-2006-10-14
-Version 1.0.9
-Woops, I was using "max-entries" for some reason but the proper header is
-"max-keys".  Not a big deal.
-Broke out the S3try stuff into a separate file so I could re-use it for s3cmd.rb
-----------
-
-2006-10-16
-Added a couple debug lines; not even enough to call it a version revision.
-----------
-
-2006-10-25
-Version 1.0.10
-UTF-8 fixes.
-Catching a couple more retry-able errors in s3try (instead of aborting the
-program).
-----------
-
-2006-10-26
-Version 1.0.11
-Revamped some details of the generators and comparator so that directories are
-handled in a more exact and uniform fashion across local and S3.
-----------
-
-2006-11-28
-Version 1.0.12
-Added a couple more error catches to s3try.
-----------
-
-2007-01-08
-Version 1.0.13
-Numerous small changes to slash and path handling, in order to catch several
-cases where "root" directory nodes were not being created on S3.
-This makes restores work a lot more intuitively in many cases.
-----------
-
-2007-01-25
-Version 1.0.14
-Peter Fales' marker fix.
-Also, markers should be decoded into native charset (because that's what s3
-expects to see).
-----------
-
-2007-02-19
-Version 1.1.0
-*WARNING* Lots of path-handling changes. *PLEASE* test safely before you just
-swap this in for your working 1.0.x version.
-
-- Adding --exclude (and there was much rejoicing).
-- Found Yet Another Leading Slash Bug with respect to local nodes. It was always
-"recursing" into the first folder even if there was no trailing slash and -r
-wasn't specified. What it should have done in this case is simply create a node
-for the directory itself, then stop (not check the dir's contents).
-- Local node canonicalization was (potentially) stripping the trailing slash,
-which we need in order to make some decisios in the local generator.
-- Fixed problem where it would prepend a "/" to s3 key names even with blank
-prefix.
-- Fixed S3->local when there's no "/" in the source so it doesn't try to create
-a folder with the bucket name.
-- Updated s3try and s3_s3ranger_mod to allow SSL_CERT_FILE
-----------
-
-2007-02-22
-Version 1.1.1
-Fixed dumb regression bug caused by the S3->local bucket name fix in 1.1.0
-----------
-
-2007-02-25
-Version 1.1.2
-Added --progress
-----------
-
-2007-06-02
-Version 1.1.3
-IMPORTANT!
-Pursuant to http://s3sync.net/forum/index.php?topic=49.0 , the tar.gz now
-expands into its own sub-directory named "s3ranger" instead of dumping all the
-files into the current directory.
-
-In the case of commands of the form:
-	s3ranger -r somedir somebucket:
-The root directory node in s3 was being stored as "somedir/" instead of "somedir"
-which caused restores to mess up when you say:
-	s3ranger -r somebucket: restoredir
-The fix to this, by coincidence, actually makes s3fox work even *less* well with
-s3ranger.  I really need to build my own xul+javascript s3 GUI some day.
-
-Also fixed some of the NoMethodError stuff for when --progress is used
-and caught Errno::ETIMEDOUT
-----------
-
-2007-07-12
-Version 1.1.4
-Added Alastair Brunton's yaml config code.
-----------
-
-2007-11-17
-Version 1.2.1
-Compatibility for S3 API revisions.
-When retries are exhausted, emit an error.
-Don't ever try to delete the 'root' local dir.
-----------
-
-2007-11-20
-Version 1.2.2
-Handle EU bucket 307 redirects (in s3try.rb)
---make-dirs added
-----------
-
-2007-11-20
-Version 1.2.3
-Fix SSL verification settings that broke in new S3 API.
-----------
-
-2008-01-06
-Version 1.2.4
-Run from any dir (search "here" for includes).
-Search out s3config.yml in some likely places.
-Reset connection (properly) on retry-able non-50x errors.
-Fix calling format bug preventing it from working from yml.
-Added http proxy support.
-----------
-
-2008-05-11
-Version 1.2.5
-Added option --no-md5
-----------
-
-2008-06-16
-Version 1.2.6
-Catch connect errors and retry.
-----------
-
-FNORD

data/doc/old/README_s3cmd.txt

@@ -1,172 +0,0 @@
-Welcome to s3cmd.rb
--------------------
-This is a ruby program that wraps S3 operations into a simple command-line tool.
-It is inspired by things like rsh3ll, #sh3ll, etc., but shares no code from
-them. It's meant as a companion utility to s3ranger.rb but could be used on its
-own (provided you have read the other readme file and know how to use s3ranger in
-theory).
-
-I made this even though lots of other "shell"s exist, because I wanted a
-single-operation utility, instead of a shell "environment".  This lends itself
-more to scripting, etc.  Also the delete operation on rsh3ll seems to be borken
-at the moment? =(
-
-Users not yet familiar with s3ranger should read about that first, since s3cmd and
-s3ranger share a tremendous amount of conventions and syntax. Particularly you
-have to set up environment variables prior to calling s3cmd, and s3cmd also uses
-the "bucket:key" syntax popularized by s3ranger. Many of the options are the same
-too. Really, go read the other readme first if you haven't used s3ranger yet.
-Otherwise you will become confused.  It's OK, I'll wait.
-
-....
-
-In general, s3ranger and s3cmd complement each other. s3ranger is useful to perform
-serious synchronization operations, and s3cmd allows you to do simple things
-such as bucket management, listing, transferring single files, and the like.
-
-Here is the usage, with examples to follow.
-
-s3cmd.rb [options] <command> [arg(s)]           version 1.0.0
-  --help    -h        --verbose     -v     --dryrun    -n
-  --ssl     -s        --debug       -d
-
-Commands:
-s3cmd.rb  listbuckets  [headers]
-s3cmd.rb  createbucket|deletebucket  <bucket>  [headers]
-s3cmd.rb  list  <bucket>[:prefix]  [max/page]  [delimiter]  [headers]
-s3cmd.rb  delete  <bucket>:key  [headers]
-s3cmd.rb  deleteall  <bucket>[:prefix]  [headers]
-s3cmd.rb  get|put  <bucket>:key  <file>  [headers]
-
-
-A note about [headers]
-----------------------
-For some S3 operations, such as "put", you might want to specify certain headers
-to the request such as Cache-Control, Expires, x-amz-acl, etc.  Rather than
-supporting a load of separate command-line options for these, I just allow
-header specification.  So to upload a file with public-read access you could
-say:
-	s3cmd.rb  put  MyBucket:TheFile.txt  x-amz-acl:public-read
-
-If you don't need to add any particular headers then you can just ignore this
-whole [headers] thing and pretend it's not there.  This is somewhat of an
-advanced option.
-
-
-Examples
---------
-List all the buckets your account owns:
-	s3cmd.rb listbuckets
-
-Create a new bucket:
-	s3cmd.rb createbucket BucketName
-
-Create a new bucket in the EU:
-	s3cmd.rb createbucket BucketName EU
-
-Find out the location constraint of a bucket:
-   s3cmd.rb location BucketName
-
-Delete an old bucket you don't want any more:
-	s3cmd.rb deletebucket BucketName
-
-Find out what's in a bucket, 10 lines at a time:
-	s3cmd.rb list BucketName 10
-
-Only look in a particular prefix:
-	s3cmd.rb list BucketName:startsWithThis
-
-Look in the virtual "directory" named foo;
-lists sub-"directories" and keys that are at this level.
-Note that if you specify a delimiter you must specify a max before it.
-(until I make the options parsing smarter)
-	s3cmd.rb list BucketName:foo/  10  /
-
-Delete a key:
-	s3cmd.rb delete BucketName:AKey
-
-Delete all keys that match (like a combo between list and delete):
-	s3cmd.rb deleteall BucketName:SomePrefix
-
-Only pretend you're going to delete all keys that match, but list them:
-	s3cmd.rb  --dryrun  deleteall  BucketName:SomePrefix
-
-Delete all keys in a bucket (leaving the bucket):
-	s3cmd.rb deleteall BucketName
-
-Get a file from S3 and store it to a local file
-	s3cmd.rb get BucketName:TheFileOnS3.txt  ALocalFile.txt
-
-Put a local file up to S3
-Note we don't automatically set mime type, etc.
-NOTE that the order of the options doesn't change. S3 stays first!
-	s3cmd.rb put BucketName:TheFileOnS3.txt ALocalFile.txt
-
-
-Change Log:
------------
-2006-10-14:
-Created.
------------
-
-2006-10-16
-Version 1.0.1
-Force content length to a string value since some ruby's don't convert it right.
------------
-
-2006-10-25
-UTF-8 fixes.
------------
-
-2006-11-28
-Version 1.0.3
-Added a couple more error catches to s3try.
-----------
-
-2007-01-25
-Version 1.0.4
-Peter Fales' marker fix.
-Also, markers should be decoded into native charset (because that's what s3
-expects to see).
-----------
-
-2007-02-19
-- Updated s3try and s3_s3ranger_mod to allow SSL_CERT_FILE
-----------
-
-2007-2-25
-Added --progress
-----------
-
-2007-07-12
-Version 1.0.6
-Added Alastair Brunton's yaml config code.
-----------
-
-2007-11-17
-Version 1.2.1
-Compatibility for S3 API revisions.
-When retries are exhausted, emit an error.
-----------
-
-2007-11-20
-Version 1.2.2
-Handle EU bucket 307 redirects (in s3try.rb)
-----------
-
-2007-11-20
-Version 1.2.3
-Fix SSL verification settings that broke in new S3 API.
-----------
-
-2008-01-06
-Version 1.2.4
-Run from any dir (search "here" for includes).
-Search out s3config.yml in some likely places.
-Reset connection (properly) on retry-able non-50x errors.
-Fix calling format bug preventing it from working from yml.
-Added http proxy support.
-----------
-
-
-FNORD

data/s3ranger.gemspec

@@ -1,32 +0,0 @@
-# -*- mode: ruby; coding: utf-8; -*-
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 's3ranger/version'
-
-Gem::Specification.new do |spec|
-  spec.name          = "s3ranger"
-  spec.version       = S3Ranger::VERSION
-  spec.authors       = ["Lincoln de Sousa"]
-  spec.email         = ["lincoln@comum.org"]
-  spec.description   = 'Tool belt for managing your S3 buckets'
-  spec.summary       = 's3ranger is a library that aggregates a good range of features for managing your Amazon S3 buckets. It also provides basic interactive client'
-
-  spec.homepage      = "http://github.com/clarete/s3ranger"
-  spec.license       = "MIT"
-
-  spec.files         = `git ls-files`.split($/)
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
-
-  # Library requirements
-  spec.add_dependency "aws-sdk"
-  spec.add_dependency "cmdparse"
-
-  # Development requirements
-  spec.add_development_dependency "debugger"
-  spec.add_development_dependency "simplecov"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
-end

data/s3ranger.yml.example

@@ -1,2 +0,0 @@
-AWS_ACCESS_KEY_ID: 11111111111111111111111
-AWS_SECRET_ACCESS_KEY: 222222222222222222222

data/spec/fixtures/nodes/1.txt

@@ -1 +0,0 @@
-s3ranger

data/spec/local_source_spec.rb

@@ -1,55 +0,0 @@
-require 'spec_helper.rb'
-require 's3ranger/sync'
-
-include S3Ranger
-
-
-describe "Local file system IO" do
-
-  it "should list local files" do
-
-    # Given that I have remote source and a local destination with a couple
-    # files
-    source = "mybucket:path"
-    destination = directory "directory2"
-    file destination, "file1.txt", "First file"
-    file destination, "file2.txt", "Second file"
-
-    # When I create a new local directory based on that path
-    local = LocalDirectory.new destination
-
-    # Then I see that the directory nodes contain both their parent paths and
-    # their names
-    local.list_files.should be_eql [
-      Node.new(fixture("directory2"), "file1.txt", 10),
-      Node.new(fixture("directory2"), "file2.txt", 11),
-    ]
-
-    rm destination
-  end
-
-  it "should skip local folders while listing files" do
-    # Given that I have remote source and a local destination with files
-    source = "mybucket:path"
-    destination = directory "directory2"
-    file destination, "file1.txt", "First file"
-    file destination, "file2.txt", "Second file"
-
-    # And with a sub-directory
-    subdir = directory "directory2/subd"
-    file subdir, "sub1.txt", "Sub content"
-
-    # When I create a new local directory based on that path
-    local = LocalDirectory.new destination
-
-    # Then I see that the directory nodes contain both their parent paths and
-    # their names
-    local.list_files.should be_eql [
-      Node.new(fixture("directory2"), "file1.txt", 10),
-      Node.new(fixture("directory2"), "file2.txt", 11),
-      Node.new(fixture("directory2"), "subd/sub1.txt", 11),
-    ]
-
-    rm destination
-  end
-end

data/spec/main_spec.rb

@@ -1,140 +0,0 @@
-require 'spec_helper.rb'
-require 's3ranger/cli'
-require 's3ranger/config'
-require 's3ranger/sync'
-
-include S3Ranger
-
-describe "Parsing command line arguments" do
-
-  describe "Processing the final destination based on how the user expressed the source" do
-
-    it "Put the local etc directory itself into S3" do
-      source = "/etc"
-      destination = "mybucket:pre"
-
-      # This will yield S3 keys named  pre/etc/...
-      SyncCommand.process_destination(source, destination).should be_eql ["/etc", ["pre/etc/", "mybucket"]]
-    end
-
-    it "Put the contents of the local /etc dir into S3, rename dir" do
-      source = "/etc/"
-      destination = "mybucket:pre/etcbackup"
-
-      # This will yield S3 keys named  pre/etcbackup/...
-      SyncCommand.process_destination(source, destination).should be_eql ["/etc/", ["pre/etcbackup/", "mybucket"]]
-    end
-
-    it "Put contents of S3 \"directory\" etc into local dir" do
-      source = "mybucket:pre/etc/"
-      destination = "/root/etcrestore"
-
-      # This will yield local files at  /root/etcrestore/...
-      SyncCommand.process_destination(source, destination).should be_eql [["pre/etc/", "mybucket"], "/root/etcrestore/"]
-    end
-
-    it "Put the contents of S3 \"directory\" etc into a local dir named etc" do
-      source = "mybucket:pre/etc"
-      destination = "/root"
-
-      # This will yield local files at  /root/etc/...
-      SyncCommand.process_destination(source, destination).should be_eql [["pre/etc", "mybucket"], "/root/etc/"]
-    end
-
-    it "Put S3 nodes under the key pre/etc/ to the local dir etcrestore" do
-      source = "mybucket:pre/etc/"
-      destination = "/root/etcrestore"
-
-      # This will yield local files at  /root/etcrestore/...
-      SyncCommand.process_destination(source, destination).should be_eql [["pre/etc/", "mybucket"], "/root/etcrestore/"]
-    end
-
-    it "Put S3 nodes under an empty key (root) to the local dir /tmp/lib" do
-      source = "mybucket:"
-      destination = "/tmp/lib"
-
-      # This will yield local files at  /root/etcrestore/...
-      SyncCommand.process_destination(source, destination).should be_eql [["", "mybucket"], "/tmp/lib/"]
-    end
-  end
-
-  it "Should calculate the right destination for each path" do
-    file = "pre/etc/sub/path/blah.txt"   # This is how it comes from s3
-    source = "mybucket:pre/etc/"
-    destination = "/root/etcrestore"
-
-    SyncCommand.process_file_destination(source, destination, file).should be_eql "/root/etcrestore/sub/path/blah.txt"
-  end
-
-  it "Put S3 files under an empty key (root) to the local dir /tmp/lib" do
-    source = "mybucket:"
-    destination = "/tmp/lib"
-    file = "myfile.rb"
-
-    # This will yield local files at  /tmp/lib/...
-    SyncCommand.process_file_destination(source, destination, file).should be_eql "/tmp/lib/myfile.rb"
-  end
-
-  it "Returning locations based on the parsed destination" do
-    source = "/etc"
-    destination = "mybucket:pre"
-
-    # When I parse the above arguments using the SyncCommand
-    src_location, dst_location = SyncCommand.parse_params [source, destination]
-
-    # Then I see I got the locations with the right params
-    src_location.should be_eql S3Ranger::Location.new("/etc")
-    dst_location.should be_eql S3Ranger::Location.new("pre/etc/", "mybucket")
-  end
-
-  it "Location should be parsed when it is remote with no path" do
-    source = "/etc"
-    destination = "mybucket:"
-
-    # When I parse the above arguments using the SyncCommand
-    src_location, dst_location = SyncCommand.parse_params [source, destination]
-
-    # Then I see I got the locations with the right params
-    src_location.should be_eql S3Ranger::Location.new("/etc")
-    dst_location.should be_eql S3Ranger::Location.new("etc/", "mybucket")
-  end
-
-  it "should be possible to detect if a location is remote" do
-    SyncCommand.remote_prefix?("bucket:prefix").should be_true
-    SyncCommand.remote_prefix?("path").should be_false
-    SyncCommand.remote_prefix?("C://blah").should be_false  # We support windows, LOL
-  end
-end
-
-describe "Comparing file lists" do
-
-  it "should be possible to describe nodes with their paths and size" do
-
-    # Full test
-    node = Node.new "path//to", "file1", 10
-    node.path.should be_eql "file1"
-    node.full.should be_eql "path/to/file1"
-    node.size.should be_eql 10
-
-    # Alternative constructor scenarios
-    node = Node.new "", "file1", 10
-    node.path.should be_eql "file1"
-  end
-
-  it "should be possible to compare two lists of files" do
-
-    # Given that I have two lists of Nodes to compare
-    list1 = [Node.new("", "file1", 10), Node.new("", "file2", 12), Node.new("", "file3", 12)]
-    list2 = [Node.new("", "file1", 10), Node.new("", "file2", 22), Node.new("", "file4", 22),]
-
-    # When I compare those two file lists
-    same_in_both, to_be_added_to_list2, to_be_removed_from_list2 = SyncCommand.cmp list1, list2
-
-    # Then I see that the three lists that I requested were returned with the
-    # right content
-    same_in_both.should == [Node.new("", "file1", 10)]     # Just testing our == operator
-    same_in_both.should be_eql [Node.new("", "file1", 10)]
-    to_be_added_to_list2.should be_eql [Node.new("", "file2", 12), Node.new("", "file3", 12)]
-    to_be_removed_from_list2.should be_eql [Node.new("", "file4", 22)]
-  end
-end

data/spec/spec_helper.rb

@@ -1,25 +0,0 @@
-require 'fileutils'
-require 'simplecov'
-SimpleCov.start
-
-def fixture *args
-  File.join File.dirname(__FILE__), "fixtures", *args
-end
-
-def directory path
-  full = fixture(path)
-  FileUtils::mkdir_p full
-  return full
-end
-
-def file *args
-  file = File.join *args[0..-2]
-  directory File.dirname(file)
-  File.open(file, 'w') {|f| f.write args[-1] }
-  return file
-end
-
-
-def rm path
-  FileUtils::rm_rf path
-end