s3cp 1.1.22 → 1.1.23

data/History.txt

@@ -1,3 +1,22 @@
+=== 1.1.23 (2013-03-15)
+
+* Added:   s3cat now accepts --head, --tail and --range arguments.
+
+              # Display only first 8KB of file
+              % s3cat --head 8k bucket:path/to/file
+
+              # Display only last 4096 bytes of file
+              % s3cat --tail 4096 bucket:path/to/file
+
+              # Download file content between positions 4096 - 12288 (exclusive)
+              % s3cat --range 4k-12k bucket:path/to/file > file_fragment
+
+              # Download content after position 12288 (inclusive)
+              % s3cat --range 12k- bucket:path/to/file > file_fragment
+
+              # Download content before position 12288 (exclusive)
+              % s3cat --range -12k bucket:path/to/file > file_fragment
+
 === 1.1.22 (2013-03-07)
 
 * Changed: s3cp now adds a new "md5" custom metadata to any uploaded file

data/lib/s3cp/s3cat.rb

@@ -39,6 +39,21 @@ op = OptionParser.new do |opts|
     options[:edit] = edit
   end
 
+  opts.on("--head SIZE", "Download only 'head' part of the file mode, e.g. 4KB, 1024B, 2GB") do |size|
+    options[:head] = S3CP.size_in_bytes(size)
+  end
+
+  opts.on("--tail SIZE", "Download only 'tail' part of the file mode, e.g. 4KB, 1024B, 2GB") do |size|
+    options[:tail] = S3CP.size_in_bytes(size)
+  end
+
+  opts.on("--range START-END", "Download only range (in absolute positions) portion, e.g. 1GB-2GB, 16K-128k, 128k-, -128k") do |range|
+    range_start, range_end = range.split("-")
+    options[:range_start] = S3CP.size_in_bytes(range_start) if range_start && (!range_start.empty?)
+    options[:range_end] = S3CP.size_in_bytes(range_end) - 1 if range_end && (!range_end.empty?)
+    range_start, range_end = range.split("-")
+  end
+
   opts.on_tail("-h", "--help", "Show this message") do
     puts op
     exit
@@ -63,8 +78,20 @@ fail "Your URL looks funny, doesn't it?" unless @bucket
 
 S3CP.load_config()
 
+read_options = {}
+read_options[:debug] = options[:debug]
+read_options[:head] = options[:head] if options[:head]
+read_options[:tail] = options[:tail] if options[:tail]
+read_options[:range_start] = options[:range_start] if options[:range_start]
+read_options[:range_end] = options[:range_end] if options[:range_end]
+
 @s3 = S3CP.connect().buckets[@bucket]
 
+
+if options[:edit] && (options[:head] || options[:tail] || options[:range_start] || options[:range_end])
+  fail "--edit option is not intended to be used with --head, --tail nor --range"
+end
+
 if options[:tty] || options[:edit]
   # store contents to file to display with PAGER
   size = @s3.objects[@prefix].content_length
@@ -76,7 +103,7 @@ if options[:tty] || options[:edit]
   file = Tempfile.new(File.basename(@prefix) + '_')
   out = File.new(file.path, "wb")
   begin
-    @s3.objects[@prefix].read_as_stream do |chunk|
+    @s3.objects[@prefix].read_as_stream(read_options) do |chunk|
       out.write(chunk)
       progress_bar.inc chunk.size
     end
@@ -104,7 +131,7 @@ if options[:tty] || options[:edit]
   end
   file.delete()
 else
-  @s3.objects[@prefix].read_as_stream do |chunk|
+  @s3.objects[@prefix].read_as_stream(read_options) do |chunk|
     begin
       STDOUT.print(chunk)
     rescue Errno::EPIPE

data/lib/s3cp/utils.rb

@@ -165,6 +165,23 @@ module S3CP
     end
   end
 
+  def size_in_bytes(size)
+    case size
+    when /^(\d+)$/ # bytes by default
+      $1.to_i
+    when /^(\d+)b$/i
+      $1.to_i
+    when /^(\d+)kb?$/i
+      $1.to_i * 1024
+    when /^(\d+)mb?$/i
+      $1.to_i * 1024 * 1024
+    when /^(\d+)gb?$/i
+      $1.to_i * 1024 * 1024 * 1024
+    else
+      fail("Invalid size value: #{size}.")
+    end
+  end
+
   def set_header_options(options, headers)
     return options unless headers
 
@@ -245,11 +262,19 @@ module AWS
     class S3Object
       def read_as_stream(options = nil, &blk)
         options ||= {}
-        chunk_size = options[:chunk] || DEFAULT_STREAMING_CHUNK_SIZE
         size = content_length
-        byte_offset = 0
-        while byte_offset < size
-          range = "bytes=#{byte_offset}-#{byte_offset + chunk_size - 1}"
+        chunk_size = options[:chunk] || DEFAULT_STREAMING_CHUNK_SIZE
+        if options[:range_start] || options[:range_end]
+          byte_offset = options[:range_start] || 0
+          max_offset = options[:range_end] || size
+        else
+          byte_offset = options[:tail] ? (size - options[:tail]) : 0
+          max_offset = options[:head] ? (options[:head] - 1) : size
+        end
+        $stderr.print "Downloading byte range #{byte_offset}-#{max_offset}\n" if options[:debug]
+        while byte_offset < max_offset
+          range = "bytes=#{byte_offset}-#{[byte_offset + chunk_size - 1, max_offset].min}"
+          $stderr.print range + "\n" if options[:debug]
           yield read(:range => range)
           byte_offset += chunk_size
         end

data/lib/s3cp/version.rb

@@ -16,5 +16,5 @@
 # the License.
 
 module S3CP
-  VERSION = "1.1.22"
+  VERSION = "1.1.23"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: s3cp
 version: !ruby/object:Gem::Version 
-  hash: 63
+  hash: 61
   prerelease: 
   segments: 
   - 1
   - 1
-  - 22
-  version: 1.1.22
+  - 23
+  version: 1.1.23
 platform: ruby
 authors: 
 - Alex Boisvert
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-03-08 00:00:00 Z
+date: 2013-03-15 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement 
@@ -171,6 +171,7 @@ files:
 - bin/s3lifecycle
 - bin/s3rm
 - bin/s3stat
+- bin/tmp
 homepage: 
 licenses: []