microget 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: de1316815435cc4a99b3d2a7c6f13b0e43015d87
-  data.tar.gz: 13e92c78deb579b3fd98ff7ccc48bf6c659595b0
+  metadata.gz: 10904aa7598a374362dd5086d3b033a8f5a296a1
+  data.tar.gz: f61403282080b0722189d571436a8c6dfa89189a
 SHA512:
-  metadata.gz: 220d1cada56fdded21fc72b5970505b93bb92f582ffda2eeca4a2f99e583ed1e2abbf02074f73182632ad560f15a4f6ebf00998f8ee97749da2f160089c50d15
-  data.tar.gz: 8d09e070e936bf79bdb53ec97af71b31eaeba3f3b41dfa4cffa2267fb77a5efe9146412d35edd52cceb4db5b509b5c0194f1e1e8984c217de2556b225b027d59
+  metadata.gz: 141b9588147336dc6656f8c49ab6c3c438ea551baaec13cfd2edab42509406ba4ca60fd6aaf37e8b0f37722c220daf4d7b6a04b4bae567a1bba70bf19caec793
+  data.tar.gz: 8a2b5cd9b07e850f23ed17f5990a20054193c9effcee6ec66983c6758f615dc5b9c96d6534aa165f7d9a9cfc1dd52816c816160a4177406a19c7c73e2ae2c895

data/Gemfile

@@ -8,7 +8,7 @@ source "http://rubygems.org"
 group :development do
   gem 'puma'
   gem 'yard'
-  gem "rspec", "~> 3.3.0"
+  gem "rspec", "~> 3.2.0", "< 3.3"
   gem "rdoc", "~> 3.12"
   gem "bundler", "~> 1.0"
   gem "jeweler", "~> 2.0.1"

data/lib/microget.rb

@@ -3,7 +3,9 @@ require 'socket'
 
 # An no-nonsense, pedal-to-the-metal unbuffered HTTP streaming client for doing GETs of large bodies, fast.
 module Microget
-  VERSION = '1.0.0'
+  autoload :ServerRunner, File.dirname(__FILE__) + '/microget/server_runner'
+  
+  VERSION = '1.1.0'
   
   extend self
 
@@ -26,6 +28,8 @@ module Microget
     raise ('Only plain HTTP is supported (%s)' % uri) unless uri.scheme == 'http'
     raise "Unknown host" unless uri.host
     
+    # Some reading on what might be usable here:
+    # http://www.mikeperham.com/2009/03/15/socket-timeouts-in-ruby/
     socket = TCPSocket.open(uri.host, uri.port || 80)
     socket.write("GET #{uri.request_uri} HTTP/1.1\r\n")
     
@@ -65,24 +69,25 @@ module Microget
   def perform_get(uri, request_headers: {}, chunk_size: 1024 * 1024 * 5)
     status_code, header_hash, socket = get_status_headers_and_body_socket(uri, request_headers: request_headers)
     body_bytes_received = 0
+    
+    # Yield the status and headers once with an empty response
+    # so that the client can bail out of the request even before the body
+    # starts to arrive
+    return body_bytes_received unless yield(status_code, header_hash, '')
+    
     # ...and then just read the body, without any buffering, using a non-blocking read
     while !socket.eof?
       begin
         data = socket.read_nonblock(chunk_size)
         body_bytes_received += data.bytesize
-        return unless yield(status_code, header_hash, data)
+        continue_reading = yield(status_code, header_hash, data)
+        return body_bytes_received unless continue_reading 
       rescue IO::WaitReadable
         IO.select([socket], [], SOCKET_TIMEOUT)
         retry
       end
     end
     
-    # Yield the status and headers once with an empty response
-    # so that the client gets at least something.
-    if body_bytes_received.zero?
-      yield(status_code, header_hash, '')
-    end
-    
     body_bytes_received
   ensure
     socket.close if socket && !socket.closed?

data/lib/microget/server_runner.rb

@@ -0,0 +1,60 @@
+require 'net/http'
+
+# A simplistic runner for external web servers within a separate process.
+class Microget::ServerRunner < Struct.new(:name, :command, :port, :rackup_file_path)
+  SHOULD_CONNECT_WITHIN = 2
+  
+  def command
+    super % [port, rackup_file_path]
+  end
+   
+  def start!
+    # Boot Puma in a forked process
+    @pid = fork do
+      $stderr.puts "Spinning up with #{command.inspect}"
+      
+      # Do not pollute the test suite output with the Puma logs,
+      # save the stuff to logfiles instead
+      $stdout.reopen(File.open('server_runner_%s_stdout.log' % name, 'a'))
+      $stderr.reopen(File.open('server_runner_%s_stderr.log' % name, 'a'))
+      
+      # Since we have to do with timing tolerances, having the output drip in ASAP is useful
+      $stdout.sync = true
+      $stderr.sync = true
+      exec(command)
+    end
+    
+    Thread.abort_on_exception = true
+    
+    t = Time.now
+    # Wait for Puma to be online, poll the alive URL until it stops responding
+    loop do
+      sleep 0.5
+      begin
+        alive_check_url = "http://0.0.0.0:%d/" % port
+        response = Net::HTTP.get_response(URI(alive_check_url))
+        @running = true
+        break
+      rescue Errno::ECONNREFUSED
+        if (Time.now - t) > SHOULD_CONNECT_WITHIN # Timeout when starting
+          raise "Could not get the server started in 2 seconds, something might be misconfigured"
+        end
+      end
+    end
+    
+    trap("TERM") { stop! }
+  end
+  
+  def running?
+    !!@running
+  end
+  
+  def stop!
+    return unless @pid
+    
+    # Tell the webserver to quit, twice (we do not care if there are running responses)
+    %W( TERM TERM KILL ).each {|sig| Process.kill(sig, @pid); sleep 0.5 }
+    @pid = nil
+    @running = false
+  end
+end

data/microget.gemspec

@@ -0,0 +1,72 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+# stub: microget 1.1.0 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "microget"
+  s.version = "1.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
+  s.authors = ["Julik Tarkhanov"]
+  s.date = "2015-11-04"
+  s.description = "Pedal-to-the-metal HTTP client for GET requests"
+  s.email = "me@julik.nl"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    ".yardopts",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "lib/microget.rb",
+    "lib/microget/server_runner.rb",
+    "microget.gemspec",
+    "spec/helper.rb",
+    "spec/microget_spec.rb",
+    "spec/microget_with_real_server_spec.rb",
+    "spec/streaming_app.ru"
+  ]
+  s.homepage = "http://github.com/julik/microget"
+  s.licenses = ["MIT"]
+  s.rubygems_version = "2.2.2"
+  s.summary = "Designed for slow and/or large response bodies"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<puma>, [">= 0"])
+      s.add_development_dependency(%q<yard>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, ["< 3.3", "~> 3.2.0"])
+      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 2.0.1"])
+      s.add_development_dependency(%q<simplecov>, [">= 0"])
+    else
+      s.add_dependency(%q<puma>, [">= 0"])
+      s.add_dependency(%q<yard>, [">= 0"])
+      s.add_dependency(%q<rspec>, ["< 3.3", "~> 3.2.0"])
+      s.add_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+      s.add_dependency(%q<simplecov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<puma>, [">= 0"])
+    s.add_dependency(%q<yard>, [">= 0"])
+    s.add_dependency(%q<rspec>, ["< 3.3", "~> 3.2.0"])
+    s.add_dependency(%q<rdoc>, ["~> 3.12"])
+    s.add_dependency(%q<bundler>, ["~> 1.0"])
+    s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+    s.add_dependency(%q<simplecov>, [">= 0"])
+  end
+end
+

data/spec/helper.rb

@@ -3,4 +3,3 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
 
 require 'rspec'
 require 'microget'
-require_relative 'server_runner'

data/spec/microget_with_real_server_spec.rb

@@ -3,9 +3,8 @@ require_relative 'helper'
 describe "Microget running against a real server" do
   before :all do
     rack_app = File.expand_path(File.join(__dir__, 'streaming_app.ru'))
-    @server = ServerRunner.new("bundle exec puma --port %d %s", 9393, rack_app)
+    @server = Microget::ServerRunner.new(:puma, "bundle exec puma --port %d %s", 9393, rack_app)
     @server.start!
-    sleep 0.5 until @server.running?
   end
   
   after :all do
@@ -49,6 +48,9 @@ describe "Microget running against a real server" do
         t = Time.now
       end
       
+      first_chunk_and_delta = time_deltas_and_chunks.shift
+      expect(first_chunk_and_delta[1]).to be_empty # First chunk is empty to allow header/status checks
+      
       time_deltas_and_chunks.each do |(delta, chunk_contents)|
         expect(chunk_contents).to include('Message number ')
         expect(delta).to be_within(0.2).of(1.0) # The server "drips" down one message every second, approximately

data/spec/streaming_app.ru

@@ -61,6 +61,10 @@ map '/with-content-length' do
   run StreamerWithLength
 end
 
+map '/' do
+  run ->(env) { [200, {}, ['Yes']]}
+end
+
 map '/alive' do
   run ->(env) { [200, {}, ['Yes']]}
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: microget
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-01 00:00:00.000000000 Z
+date: 2015-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: puma
@@ -42,16 +42,22 @@ dependencies:
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.3'
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.0
+        version: 3.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.3'
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.0
+        version: 3.2.0
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -124,11 +130,11 @@ files:
 - README.md
 - Rakefile
 - lib/microget.rb
+- lib/microget/server_runner.rb
+- microget.gemspec
 - spec/helper.rb
 - spec/microget_spec.rb
 - spec/microget_with_real_server_spec.rb
-- spec/server_runner.rb
-- spec/spec_helper.rb
 - spec/streaming_app.ru
 homepage: http://github.com/julik/microget
 licenses:

data/spec/server_runner.rb

@@ -1,57 +0,0 @@
-require 'net/http'
-
-class ServerRunner < Struct.new(:command, :port, :rackup_file_path)
-  def command
-    super % [port, rackup_file_path]
-  end
-   
-  def start!
-    # Boot Puma in a forked process
-    @pid = fork do
-      $stderr.puts "Spinning up with #{command.inspect}"
-      # Do not pollute the RSpec output with the Puma logs,
-      # save the stuff to logfiles instead
-#      $stdout.reopen(File.open('server_runner_stdout.log' % name, 'a'))
-#      $stderr.reopen(File.open('server_runner_stderr.log' % name, 'a'))
-      
-      # Since we have to do with timing tolerances, having the output drip in ASAP is useful
-      $stdout.sync = true
-      $stderr.sync = true
-      exec(command)
-    end
-    
-    Thread.abort_on_exception = true
-    
-    Thread.new do
-      t = Time.now
-      # Wait for Puma to be online, poll the alive URL until it stops responding
-      loop do
-        sleep 0.5
-        begin
-          alive_check_url = "http://0.0.0.0:%d/alive" % port
-          response = Net::HTTP.get_response(URI(alive_check_url))
-          @running = true
-          break
-        rescue Errno::ECONNREFUSED
-          if Time.now - t > 2 # Timeout when starting
-            raise "Could not get the server started in 2 seconds, something might be misconfigured"
-          end
-        end
-      end
-    end
-    
-    trap("TERM") { stop! }
-  end
-  
-  def running?
-    !!@running
-  end
-  
-  def stop!
-    return unless @pid
-    
-    # Tell the webserver to quit, twice (we do not care if there are running responses)
-    %W( TERM TERM KILL ).each {|sig| Process.kill(sig, @pid); sleep 0.5 }
-    @pid = nil
-  end
-end

data/spec/spec_helper.rb

@@ -1,6 +0,0 @@
-require 'simplecov'
-
-
-RSpec.configure do |config|
-
-end