http_streaming_client 0.8.5 → 0.8.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d0bb65b843e7e815666b6ad0469c5012fc0bd74f
-  data.tar.gz: dd6dba01afc281a4466d597d1a133c0cc712bc89
+  metadata.gz: 607b852759a7216b85fa6d820d2bc70a2d50b475
+  data.tar.gz: 49b554701c78b290e3b7e337a1b8fbcc12f0f726
 SHA512:
-  metadata.gz: d3760c04566e74423422331002fc97caefcf630b148a6972693ca318d4f991528bcd4df9f210ac64f004dad1664031ecdaf7e89fbc65b4487f149b6506528e6c
-  data.tar.gz: d44f9f0ecb13336fcf84d47f33d4b2a1185ac2e18b2f0b235185aa9f57716447171c15dd2af23e9171a4ac34a101c0e57c99d8b78f1068e81a66575ca0220c77
+  metadata.gz: c1d07b0e483dc0e81d83800d05d6170d28f00dad305f6e6f3644ffe96b9bac00ff203a7bd34fadf697245b10520cdb1e303b79442dc0bd6bd3a477bb52b99f63
+  data.tar.gz: a26cf37f5791d4701ca0f0bf74cdedb009fb1a6d1ee58934f15dfad8a12ffb8c3755a3854c95b496fc2ca22bae69c70e8a04d40c7ada2efd54f41c5ead6ca2d2

data/README.md

@@ -43,10 +43,6 @@ Or install it yourself as:
 
     $ gem install http_streaming_client
 
-## Version
-
-Current release version: 0.8.1 (see <a href="https://github.com/adobe-research/http_streaming_client/releases">RELEASES</a>)
-
 ## Logging
 
 HTTP protocol trace logging is available as :debug level logging. The gem supports configurable logging to both STDOUT and a log file, and includes a Railtie to use Rails.logger when the gem is included in a Rails application.

data/lib/http_streaming_client/client.rb

@@ -97,6 +97,7 @@ module HttpStreamingClient
     end
 
     def request(method, uri, opts = {}, &block)
+
       logger.debug("Client::request:#{method}:#{uri}:#{opts}")
 
       if uri.is_a?(String)
@@ -166,6 +167,15 @@ module HttpStreamingClient
 
       logger.debug "response headers:#{response_head[:headers]}"
 
+      if response_head[:code] == 301 then
+	location = response_head[:headers]["Location"]
+	raise InvalidRedirect, "Unable to find Location header for HTTP 301 response" if location.nil?
+	logger.debug "Received HTTP 301 redirect to #{location}, following..."
+	socket.close if !socket.nil? and !socket.closed?
+	opts.delete(:socket)
+	return request(method, location, opts, &block)
+      end
+
       content_length = response_head[:headers]["Content-Length"].to_i
       logger.debug "content-length: #{content_length}"
 
@@ -200,11 +210,11 @@ module HttpStreamingClient
 	  logger.debug "response compression detected"
 	  if block_given? then
 	    decoder = HttpStreamingClient::Decoders::GZip.new { |line|
-	      logger.debug "read #{line.size} uncompressed bytes"
+	      logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
 	      block.call(line) unless @interrupted }
 	  else
 	    decoder = HttpStreamingClient::Decoders::GZip.new { |line|
-	      logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total"
+	      logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
 	      response << line unless @interrupted }
 	  end
 	end
@@ -237,7 +247,7 @@ module HttpStreamingClient
 	  return if @interrupted
 
 	  if response_compression then
-	      decoder << partial
+	    decoder << partial
 	  else
 	    if block_given? then
 	      yield partial
@@ -274,11 +284,11 @@ module HttpStreamingClient
 
 	  if block_given? then
 	    decoder = HttpStreamingClient::Decoders::GZip.new { |line|
-	      logger.debug "read #{line.size} uncompressed bytes"
+	      logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
 	      block.call(line) unless @interrupted }
 	  else
 	    decoder = HttpStreamingClient::Decoders::GZip.new { |line|
-	      logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total"
+	      logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
 	      response << line unless @interrupted }
 	  end
 

data/lib/http_streaming_client/decoders/gzip.rb

@@ -44,10 +44,41 @@ module HttpStreamingClient
 	@packet_callback = packet_callback
       end
 
+      def nonblock_readline(io)
+	@line_buffer ||= ""
+	ch = nil
+	begin
+	  while ch = io.getc
+	    @line_buffer += ch
+	    if ch == "\n" then
+	      result = @line_buffer
+	      @line_buffer = ""
+	      return result
+	    end
+	  end
+	rescue Zlib::GzipFile::Error
+	  # this is raised on EOF by ZLib, return nil to indicate EOF and leave partial line in the buffer
+	  return nil
+	end
+      end
+
       def <<(compressed_packet)
 	return unless compressed_packet && compressed_packet.size > 0
-	decompressed_packet = decompress(compressed_packet)
-	process_decompressed_packet(decompressed_packet)
+	@buf ||= GZipBufferIO.new
+	@buf << compressed_packet
+
+	# pass at least 2k bytes to GzipReader to avoid zlib EOF
+	if @buf.size > 2048 then
+
+	  @gzip ||= Zlib::GzipReader.new @buf
+
+	  while true do
+	    decompressed_packet = nonblock_readline(@gzip)
+	    #logger.debug "decompressed_packet:#{decompressed_packet}"
+	    break if decompressed_packet.nil?
+	    process_decompressed_packet(decompressed_packet)
+	  end
+	end
       end
 
       def close
@@ -55,26 +86,25 @@ module HttpStreamingClient
 	decompressed_packet = ""
 	begin
 	  @gzip ||= Zlib::GzipReader.new @buf
-	  decompressed_packet = @gzip.readline
+
+	  while true do
+	    decompressed_packet = nonblock_readline(@gzip)
+	    #logger.debug "decompressed_packet:#{decompressed_packet}"
+	    break if decompressed_packet.nil?
+	    process_decompressed_packet(decompressed_packet)
+	  end
+
 	rescue Zlib::Error
 	  raise DecoderError
 	end
-	process_decompressed_packet(decompressed_packet)
       end
 
-      protected
-
-      def decompress(compressed_packet)
-	@buf ||= GZipBufferIO.new
-	@buf << compressed_packet
-
-	# pass at least 2k bytes to GzipReader to avoid zlib EOF
-	if @buf.size > 2048
-	  @gzip ||= Zlib::GzipReader.new @buf
-	  @gzip.readline
-	end
+      def size
+	@buf.size
       end
 
+      protected
+
       class GZipBufferIO
 
 	def logger
@@ -92,21 +122,38 @@ module HttpStreamingClient
 
 	# called by GzipReader
 	def readpartial(length=nil, buffer=nil)
-	  logger.debug "GZipBufferIO:read:packet_stream:#{@packet_stream.nil? ? 'nil' : 'not nil'}"
+	  logger.debug "GZipBufferIO:readpartial:length:#{length}:@packet_stream:#{@packet_stream.nil? ? 'nil' : 'not null'}"
 	  buffer ||= ""
-	  length ||= 0
+
+	  raise EOFError "" if @packet_stream.size == 0
+
+	  length ||= @packet_stream.size # read all if a fraction is specified
+	  length = [ length, @packet_stream.size ].min # read length or @packet_stream.size, whichever is smaller
+
+	  #logger.debug "GZipBufferIO:readpartial:before:psize:#{@packet_stream.size}:bsize:#{buffer.size}:length:#{length}"
+
 	  buffer << @packet_stream[0..(length-1)]
-	  @packet_stream = @packet_stream[length..-1]
+
+	  if length == @packet_stream.size then
+	    @packet_stream = ""
+	  else
+	    @packet_stream = @packet_stream[length..-1]
+	  end
+
+	  #logger.debug "GZipBufferIO:readpartial:after:psize:#{@packet_stream.size}:bsize:#{buffer.size}"
 	  buffer
 	end
-	
+
 	# called by GzipReader
 	def read(length=nil, buffer=nil)
+	  logger.debug "read:length:#{length}"
+	  return nil if @packet_stream.size == 0
 	  readpartial(length, buffer)
 	end
 
 	# called by GzipReader
 	def size
+	  logger.debug "size():#{@packet_stream.size}"
 	  @packet_stream.size
 	end
       end
@@ -116,7 +163,7 @@ module HttpStreamingClient
       def process_decompressed_packet(decompressed_packet)
 	logger.debug "GZipBufferIO:process_decompressed_packet:size:#{decompressed_packet.nil? ? "nil" : decompressed_packet.size}"
 	if decompressed_packet && decompressed_packet.size > 0
-	  @packet_callback.call(decompressed_packet)
+	  @packet_callback.call(decompressed_packet) unless @packet_callback.nil?
 	end
       end
 

data/lib/http_streaming_client/errors.rb

@@ -30,6 +30,8 @@
 module HttpStreamingClient
 
   class InvalidContentType < Exception; end
+  
+  class InvalidRedirect < Exception; end
 
   class HttpTimeOut < StandardError; end
 

data/lib/http_streaming_client/version.rb

@@ -28,5 +28,5 @@
 ###########################################################################
 
 module HttpStreamingClient
-  VERSION = "0.8.5"
+  VERSION = "0.8.6"
 end

data/spec/client_spec.rb

@@ -9,9 +9,17 @@ describe HttpStreamingClient do
     it { should_not be_nil}
   end
 
-  describe "client instance get test" do
+  describe "client instance get test, no chunked transfer encosing and GZIP compression" do
     client = HttpStreamingClient::Client.new
-    response = client.get "http://www.google.com/"
+    response = client.get "http://screamradius.com/"
+    logger.debug "response: #{response}"
+    subject { response }
+    it { should_not be_nil}
+  end
+
+  describe "client instance get test, test 301 redirect" do
+    client = HttpStreamingClient::Client.new(compression: true)
+    response = client.get("https://www.ebay.com/")
     logger.debug "response: #{response}"
     subject { response }
     it { should_not be_nil}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: http_streaming_client
 version: !ruby/object:Gem::Version
-  version: 0.8.5
+  version: 0.8.6
 platform: ruby
 authors:
 - David Tompkins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-10 00:00:00.000000000 Z
+date: 2014-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler