raca 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 708a253f4ed2a4a4698ad10a9bdf324d7ce10621
-  data.tar.gz: 4a2b677bfd024e010b5c65f3d4e33e910f60c843
+  metadata.gz: afe99d603e6ba54f7b49a0647a5b41fc200a48cd
+  data.tar.gz: e27360551bbe68b0e280956b6b5120043ede60ed
 SHA512:
-  metadata.gz: 9bb93f27145cc61c1c07978b412015b6258ed1134c6fe21f8b61bb2a5f036b8fd3f9a96e3323c85ebaefe2460d30aca8c8f1ad72fa8ae1ab21b3d82f91a35c99
-  data.tar.gz: c5ec178d8f6468a1020d6478ebc73e5611e193eae83fe1c2e27f022f4e70058501fe32f5197402e659215a11ebc37e03920fc2475f3293108540b5f788f6cd25
+  metadata.gz: 24db05708c37b8fa0d2c0a49bfd19d1a75c8609a9c3c5e7fad889fbee0ccec8bfac8bcc5bdf1cae60be8bb28db4d3e9997afaeeada507e894fbc7492e7902aa8
+  data.tar.gz: ede623eb08d1fcb150d23231f853eee503c5401dedbbd7ae8cbbc813f8f3461c044d3ca6313cd8b71c8d86ddfc320b54b6a34bd2f289be87e517068f14d4ddc0

data/CHANGELOG

@@ -1,3 +1,7 @@
+v0.4.3 (12th September 2015)
+* Use less memory when uploading large files (> 5Gb) to cloud files
+* Allow a pre-computed ETag header to be provided when uploading to cloud files
+
 v0.4.2 (19th July 2015)
 * Large files (> 5Gb) that are uploaded to cloud files are broken into 4Gb
   chunks instead of 100Mb.

data/lib/raca/container.rb

@@ -1,6 +1,7 @@
 require 'digest/md5'
 require 'openssl'
 require 'uri'
+require 'raca/windowed_io'
 
 module Raca
 
@@ -32,15 +33,14 @@ module Raca
     # manually specify content type, content disposition, CORS headers, etc.
     #
     def upload(key, data_or_path, headers = {})
-      case data_or_path
-      when StringIO, File
+      if data_or_path.respond_to?(:read) && data_or_path.respond_to?(:size)
         upload_io(key, data_or_path, data_or_path.size, headers)
-      when String
-        File.open(data_or_path, "rb") do |io|
+      elsif !File.file?(data_or_path.to_s)
+        raise ArgumentError, "data_or_path must be an IO with data or filename string"
+      else
+        File.open(data_or_path.to_s, "rb") do |io|
           upload_io(key, io, io.stat.size, headers)
         end
-      else
-        raise ArgumentError, "data_or_path must be an IO with data or filename string"
       end
     end
 
@@ -276,7 +276,7 @@ module Raca
       if io.respond_to?(:path)
         headers['Content-Type'] ||= extension_content_type(io.path)
       end
-      headers['ETag']           = md5_io(io)
+      headers['ETag']           ||= md5_io(io)
       headers['Content-Type']   ||= "application/octet-stream"
       if content_type_needs_cors(key)
         headers['Access-Control-Allow-Origin'] = "*"
@@ -292,9 +292,8 @@ module Raca
       while segments.size < segment_count
         start_pos = 0 + (LARGE_FILE_SEGMENT_SIZE * segments.size)
         segment_key = "%s.%03d" % [key, segments.size]
-        io.seek(start_pos)
-        segment_io = StringIO.new(io.read(LARGE_FILE_SEGMENT_SIZE))
-        etag = upload_io_standard(segment_key, segment_io, segment_io.size, headers)
+        segment_io = WindowedIO.new(io, start_pos, LARGE_FILE_SEGMENT_SIZE)
+        etag = upload_io_standard(segment_key, segment_io, segment_io.size, headers.reject { |k,_v| k == "ETag"})
         segments << {path: "#{@container_name}/#{segment_key}", etag: etag, size_bytes: segment_io.size}
       end
       full_path = File.join(container_path, Raca::Util.url_encode(key)) + "?multipart-manifest=put"

data/lib/raca/windowed_io.rb

@@ -0,0 +1,72 @@
+module Raca
+
+  # Wrap an IO object and expose only a partial subset of the underlying data
+  # in an IO-ish interface. Calling code will have access to the window of data
+  # starting at 'offset' and the following 'length' bytes.
+  #
+  # This doesn't implement the entire IO contract, just enough to make it work
+  # when handed to Net::HTTP as a request body.
+  #
+  class WindowedIO
+    def initialize(io, offset, length)
+      @io = io
+      @offset = offset
+      if @offset + length > @io.size
+        @length = @io.size - offset
+      else
+        @length = length
+      end
+      @io.seek(@offset)
+    end
+
+    def eof?
+      @io.pos >= @offset + @length
+    end
+
+    def pos
+      @io.pos - @offset
+    end
+
+    def seek(to)
+      if to <= 0
+        @io.seek(@offset)
+      elsif to >= @length
+        @io.seek(@offset + @length)
+      else
+        @io.seek(@offset + to)
+      end
+    end
+
+    def size
+      @length
+    end
+
+    def each(bytes, &block)
+      loop do
+        line = read(bytes)
+        break if line.nil? || line == ""
+        yield line
+      end
+    end
+
+    def read(bytes = 1024)
+      @io.read(capped_bytes_to_read(bytes))
+    end
+
+    def readpartial(bytes = 1024, outbuf = nil)
+      raise EOFError.new("end of file reached") if eof?
+      bytes = capped_bytes_to_read(bytes)
+      @io.readpartial(bytes, outbuf)
+    end
+
+    private
+
+    def capped_bytes_to_read(bytes)
+      if pos + bytes > @length
+        size - pos
+      else
+        bytes
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: raca
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - James Healy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-19 00:00:00.000000000 Z
+date: 2015-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -42,16 +42,16 @@ dependencies:
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.20'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.20'
 - !ruby/object:Gem::Dependency
   name: ir_b
   requirement: !ruby/object:Gem::Requirement
@@ -102,6 +102,7 @@ files:
 - lib/raca/user.rb
 - lib/raca/users.rb
 - lib/raca/util.rb
+- lib/raca/windowed_io.rb
 homepage: http://github.com/conversation/raca
 licenses:
 - MIT
@@ -125,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: A simple wrapper for the Rackspace Cloud API with no dependencies