down 4.2.0 → 4.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 443491ec05f03e75593726f54314b04587637bf2
-  data.tar.gz: a04c39fd140b7266f7ea80bfee2da4f649c079c4
+  metadata.gz: 16efde61a43b30f68aa001eb0f06bdc8403d2e7c
+  data.tar.gz: 0e56e5adc2bb0bc0387f7198af7b0063ab577aef
 SHA512:
-  metadata.gz: 1a79349a849954e2e68b5bcf187e79825255c9a1c3fd5f1ea81dc12f2d97e4d20a5419443a6e3d3942e82b3af8615fcce24cfcdbd4ded091048d7f02e17d5039
-  data.tar.gz: 2fb8ed137234217f44790f35a767dead06eecff2174fba0960711ebde04d90ff4ee76f20dced2ab2bd6a48b7e5b31c335556504d6892d9603f6c89092cc7512b
+  metadata.gz: b2f9108523aea427699de88ffc65f8e6d114d75850317f07bce6c3c7019e7f5add1b52ee660e57c109b526d1a8ea42f81c0944564c17ba6f6b18d537f51f4a69
+  data.tar.gz: 535dbb6812caba261e692ec713bf25df4aa57ec97027a1b74474aef060b2c9c182267a0d8008210e760d66826e7c978b898efdf4dc0d66355507c39755d84944

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.2.1 (2018-01-29)
+
+* Reduce memory allocation in `Down::ChunkedIO` by 10x when buffer string is used (@janko-m)
+
+* Reduce memory allocation in `Down::Http.download` by 10x.
+
 ## 4.2.0 (2017-12-22)
 
 * Handle `:max_redirects` in `Down::NetHttp#open` and follow up to 2 redirects by default (@janko-m)

data/README.md

@@ -247,10 +247,7 @@ some of open-uri's undesired behaviours:
 * allows you to limit maximum number of redirects
 
 On the other hand `Down::NetHttp.open` is implemented using Net::HTTP directly,
-as open-uri
-
-Since open-uri doesn't expose support for partial downloads,
-`Down::NetHttp.open` is implemented using `Net::HTTP` directly.
+as open-uri doesn't support downloading on-demand.
 
 #### Redirects
 
@@ -426,7 +423,7 @@ wget.open("http://nature.com/forest.jpg")
 You can run tests with
 
 ```
-$ rake test
+$ bundle exec rake test
 ```
 
 The test suite pulls and runs [kennethreitz/httpbin] as a Docker container, so

data/down.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "minitest", "~> 5.8"
   spec.add_development_dependency "mocha"
+  spec.add_development_dependency "rake"
   spec.add_development_dependency "http", "~> 3.0"
   spec.add_development_dependency "posix-spawn" unless RUBY_ENGINE == "jruby"
   spec.add_development_dependency "http_parser.rb"

data/lib/down/chunked_io.rb

@@ -148,7 +148,7 @@ module Down
     def readpartial(length = nil, outbuf = nil)
       fail IOError, "closed stream" if closed?
 
-      data = outbuf.replace("").force_encoding(@encoding) if outbuf
+      data = outbuf.clear.force_encoding(@encoding) if outbuf
 
       if cache && !cache.eof?
         data = cache.read(length, outbuf)
@@ -182,6 +182,8 @@ module Down
         else
           @buffer = nil
         end
+
+        buffered_data.clear unless buffered_data.equal?(data)
       end
 
       @position += data.bytesize

data/lib/down/http.rb

@@ -32,7 +32,7 @@ module Down
       tempfile = Tempfile.new(["down-http", extname], binmode: true)
 
       until io.eof?
-        tempfile.write(io.readpartial)
+        tempfile.write(io.readpartial(nil, buffer ||= String.new))
 
         progress_proc.call(tempfile.size) if progress_proc
 

data/lib/down/net_http.rb

@@ -66,7 +66,8 @@ module Down
 
       open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects)
 
-      tempfile = ensure_tempfile(open_uri_file)
+      tempfile = ensure_tempfile(open_uri_file, File.extname(open_uri_file.base_uri.path))
+      OpenURI::Meta.init tempfile, open_uri_file # add back open-uri methods
       tempfile.extend Down::NetHttp::DownloadedFile
 
       tempfile
@@ -132,25 +133,23 @@ module Down
       request_error!(exception)
     end
 
-    # Converts the open-uri result file into a Tempfile if it isn't already,
-    # and makes sure the Tempfile has the correct file extension.
-    def ensure_tempfile(open_uri_file)
-      extension = File.extname(open_uri_file.base_uri.path)
-      tempfile  = Tempfile.new(["down-net_http", extension], binmode: true)
+    # Converts the given IO into a Tempfile if it isn't one already (open-uri
+    # returns a StringIO when there is less than 10KB of content), and gives
+    # it the specified file extension.
+    def ensure_tempfile(io, extension)
+      tempfile = Tempfile.new(["down-net_http", extension], binmode: true)
 
-      if open_uri_file.is_a?(Tempfile)
+      if io.is_a?(Tempfile)
         # Windows requires file descriptors to be closed before files are moved
-        open_uri_file.close
+        io.close
         tempfile.close
-        FileUtils.mv open_uri_file.path, tempfile.path
-      else # open-uri returns a StringIO when there is less than 10KB of content
-        IO.copy_stream(open_uri_file, tempfile)
-        open_uri_file.close
+        FileUtils.mv io.path, tempfile.path
+      else
+        IO.copy_stream(io, tempfile)
+        io.close
       end
 
       tempfile.open
-      OpenURI::Meta.init tempfile, open_uri_file # adds open-uri methods
-
       tempfile
     end
 

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "4.2.0"
+  VERSION = "4.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 4.2.0
+  version: 4.2.1
 platform: ruby
 authors:
 - Janko Marohnić
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-22 00:00:00.000000000 Z
+date: 2018-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement