s3cp 1.1.16 → 1.1.17

data/History.txt

@@ -1,3 +1,7 @@
+=== 1.1.17 (2013-02-05)
+
+* Fixed:    Speed-up `s3du` (same dealio as s3ls -l from version 1.1.14)
+
 === 1.1.16 (2013-01-31)
 
 * Fixed:    s3ls paging (with > 1000 keys) was broken (bug introduced in s3cp 1.1.14)

data/lib/s3cp/#Untitled-5#

@@ -0,0 +1,5 @@
+
+1TB / 3GB
+
+300 splits
+

data/lib/s3cp/s3cp.rb

@@ -478,7 +478,7 @@ def copy(from, to, options)
         end
       end
     else
-      key_to += File.basename(key_from) if key_to[-1..-1] == "/"
+      key_to += File.basename(from) if key_to[-1..-1] == "/"
       local_to_s3(bucket_to, key_to, File.expand_path(from), options) unless !options[:overwrite] && s3_exist?(bucket_to, key_to)
     end
   when :s3_to_local

data/lib/s3cp/s3du.rb

@@ -95,27 +95,39 @@ def print(key, size)
 end
 
 begin
-  @s3.buckets[@bucket].objects.with_prefix(@prefix).each do |entry|
-    key  = entry.key
-    size = entry.content_length
-
-    if options[:regex].nil? || options[:regex].match(key)
-      current_key = if actual_depth
-        pos = nth_occurrence(key, "/", actual_depth)
-        (pos != -1) ? key[0..pos-1] : key
+  s3_options = Hash.new
+  s3_options[:bucket_name] = @bucket
+  s3_options[:prefix] = @prefix
+
+  begin
+    response = @s3.client.list_objects(s3_options)
+    response[:contents].each do |object|
+
+      key  = object[:key]
+      size = object[:size].to_i
+
+      if options[:regex].nil? || options[:regex].match(key)
+        current_key = if actual_depth
+          pos = nth_occurrence(key, "/", actual_depth)
+          (pos != -1) ? key[0..pos-1] : key
+        end
+
+        if (last_key && last_key != current_key)
+          print(last_key, subtotal_size)
+          subtotal_size = size
+        else
+          subtotal_size += size
+        end
+
+        last_key = current_key
+        total_size += size
       end
+    end
 
-      if (last_key && last_key != current_key)
-        print(last_key, subtotal_size)
-        subtotal_size = size
-      else
-        subtotal_size += size
-      end
+    break if response[:contents].empty?
 
-      last_key = current_key
-      total_size += size
-    end
-  end
+    s3_options.merge!(:marker => response[:contents].last[:key])
+  end while response[:truncated]
 
   if last_key != nil
     print(last_key, subtotal_size)

data/lib/s3cp/version.rb

@@ -16,5 +16,5 @@
 # the License.
 
 module S3CP
-  VERSION = "1.1.16"
+  VERSION = "1.1.17"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: s3cp
 version: !ruby/object:Gem::Version 
-  hash: 51
+  hash: 49
   prerelease: 
   segments: 
   - 1
   - 1
-  - 16
-  version: 1.1.16
+  - 17
+  version: 1.1.17
 platform: ruby
 authors: 
 - Alex Boisvert
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-01-31 00:00:00 Z
+date: 2013-02-05 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement 
@@ -138,6 +138,7 @@ extra_rdoc_files:
 files: 
 - lib/s3cp/s3cat.rb
 - lib/s3cp/s3tree.rb
+- lib/s3cp/#Untitled-5#
 - lib/s3cp/s3ls.rb
 - lib/s3cp/s3du.rb
 - lib/s3cp/s3mod.rb