iudex-jetty-httpclient 1.1.0-java

data/History.rdoc

@@ -0,0 +1,2 @@
+=== 1.1.0 (2011-11-13)
+* Initial release with Iudex 1.1.x.

data/Manifest.txt

@@ -0,0 +1,10 @@
+History.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+pom.xml
+lib/iudex-jetty-httpclient/base.rb
+lib/iudex-jetty-httpclient.rb
+test/setup.rb
+test/test_httpclient.rb
+lib/iudex-jetty-httpclient/iudex-jetty-httpclient-1.1.0.jar

data/README.rdoc

@@ -0,0 +1,25 @@
+= iudex-jetty-httpclient
+
+* http://github.com/dekellum/iudex
+
+== Description
+
+Iudex is a general purpose web crawler and feed processor in
+ruby/java. This gem is a Jetty HTTP Client based implementation of the
+iudex-http interfaces.
+
+== License
+
+Copyright (c) 2011 David Kellum
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License.  You
+may obtain a copy of the License at:
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.

data/Rakefile

@@ -0,0 +1,43 @@
+# -*- ruby -*-
+
+$LOAD_PATH << './lib'
+require 'iudex-jetty-httpclient/base'
+
+require 'rubygems'
+gem     'rjack-tarpit', '~> 1.4'
+require 'rjack-tarpit'
+
+t = RJack::TarPit.new( 'iudex-jetty-httpclient',
+                       Iudex::JettyHTTPClient::VERSION,
+                       :no_assembly, :java_platform )
+
+t.specify do |h|
+  h.developer( "David Kellum", "dek-oss@gravitext.com" )
+
+  h.extra_deps += [ [ 'iudex-http',             '~> 1.1.0' ],
+                    [ 'rjack-jetty',            '~> 7.5.2' ],
+                    [ 'hooker',                 '~> 1.0.0' ] ]
+
+  h.testlib = :minitest
+  h.extra_dev_deps += [ [ 'minitest',        '~> 2.3' ],
+                        [ 'iudex-http-test', '~> 1.1.0' ],
+                        [ 'rjack-logback',   '~> 1.0' ] ]
+end
+
+file 'Manifest.txt' => "lib/#{t.name}/base.rb"
+
+task :check_pom_version do
+  t.test_line_match( 'pom.xml', /<version>/, /#{t.version}/ )
+end
+task :check_history_version do
+  t.test_line_match( 'History.rdoc', /^==/, / #{t.version} / )
+end
+task :check_history_date do
+  t.test_line_match( 'History.rdoc', /^==/, /\([0-9\-]+\)$/ )
+end
+
+task :gem  => [ :check_pom_version, :check_history_version                      ]
+task :tag  => [ :check_pom_version, :check_history_version, :check_history_date ]
+task :push => [                                             :check_history_date ]
+
+t.define_tasks

data/lib/iudex-jetty-httpclient.rb

@@ -0,0 +1,73 @@
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-http'
+
+require 'rjack-jetty'
+require 'rjack-jetty/client'
+
+require 'hooker'
+
+require 'iudex-jetty-httpclient/base'
+
+require 'java'
+
+module Iudex
+
+  module JettyHTTPClient
+    require "#{LIB_DIR}/iudex-jetty-httpclient-#{VERSION}.jar"
+
+    import 'iudex.jettyhttpclient.Client'
+
+    include RJack::Jetty
+
+    def self.create_jetty_client( opts = {} )
+      cfg = { :timeout                     => 6_000,
+              :so_timeout                  => 5_000,
+              :connect_timeout             => 3_000,
+              :idle_timeout                => 6_000,
+              :max_retries                 => 1,
+              :max_redirects               => 6,
+              :max_connections_per_address => 2,
+              :max_queue_size_per_address  => 100,
+              :connect_blocking            => false,
+              :handle_redirects_internal   => false }
+
+      cfg = cfg.merge( opts )
+      cfg = Hooker.merge( [ :iudex, :jetty_httpclient ], cfg )
+
+      jclient = HttpClient.new
+
+      redir_listen = cfg.delete( :handle_redirects_internal )
+
+      cfg.each do |key,value|
+        jclient.__send__( "set_#{key}", value )
+      end
+
+      if redir_listen
+        jclient.register_listener( 'org.eclipse.jetty.client.RedirectListener' )
+      end
+
+      jclient
+    end
+
+    def self.create_client( opts = {} )
+      Client.new( create_jetty_client( opts ) )
+    end
+
+  end
+
+end

data/lib/iudex-jetty-httpclient/base.rb

@@ -0,0 +1,23 @@
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module Iudex
+  module JettyHTTPClient
+    VERSION = '1.1.0'
+
+    LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
+  end
+end

data/pom.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>iudex</groupId>
+  <artifactId>iudex-jetty-httpclient</artifactId>
+  <packaging>jar</packaging>
+  <version>1.1.0</version>
+  <name>Iudex Jetty HTTP Client Adaptor</name>
+
+  <parent>
+    <groupId>iudex</groupId>
+    <artifactId>iudex-parent</artifactId>
+    <version>1.1</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <repositories>
+    <repository>
+      <id>Sonatype</id>
+      <name>Sonatype Release</name>
+      <url>http://oss.sonatype.org/content/repositories/releases </url>
+    </repository>
+  </repositories>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>iudex</groupId>
+      <artifactId>iudex-http</artifactId>
+      <version>[1.1,1.2)</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-client</artifactId>
+      <version>[7.5.2,7.5.9999)</version>
+    </dependency>
+
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <!-- Parent settings -->
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <!-- Parent settings -->
+        <artifactId>maven-source-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

data/test/setup.rb

@@ -0,0 +1,46 @@
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+#### General test setup: LOAD_PATH, logging, console output ####
+
+ldir = File.join( File.dirname( __FILE__ ), "..", "lib" )
+$LOAD_PATH.unshift( ldir ) unless $LOAD_PATH.include?( ldir )
+
+require 'rubygems'
+require 'rjack-logback'
+require 'minitest/unit'
+require 'minitest/autorun'
+
+module TestSetup
+  include RJack
+  Logback.config_console( :stderr => true, :thread => true )
+
+  if ( ARGV & %w[ -v --verbose --debug ] ).empty?
+    # Make test output logging compatible: no partial lines.
+    class TestOut
+      def print( *a ); $stdout.puts( *a ); end
+      def puts( *a );  $stdout.puts( *a ); end
+    end
+    MiniTest::Unit.output = TestOut.new
+    Logback[ 'org.eclipse.jetty.http.ssl.SslContextFactory' ].level =
+      Logback::WARN
+  else
+    Logback.root.level = Logback::DEBUG
+  end
+
+  ARGV.delete( '--debug' )
+
+end

data/test/test_httpclient.rb

@@ -0,0 +1,602 @@
+#!/usr/bin/env jruby
+
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+
+require 'iudex-http-test/helper'
+require 'iudex-http-test/broken_server'
+
+require 'iudex-jetty-httpclient'
+require 'thread'
+require 'cgi'
+require 'uri'
+
+class TestHTTPClient < MiniTest::Unit::TestCase
+  include Iudex
+  include Iudex::HTTP
+  include Iudex::HTTP::Test
+  include Helper
+
+  import 'java.util.concurrent.TimeoutException'
+  import 'java.net.ConnectException'
+  import 'java.net.UnknownHostException'
+  import 'java.net.URISyntaxException'
+  import 'java.io.IOException'
+  import 'java.lang.IllegalStateException'
+  import 'java.nio.channels.UnresolvedAddressException'
+
+  CustomUnit.register
+
+  def setup
+    @rlock = Mutex.new
+    server # make sure jetty starts, for cosmetic log output
+  end
+
+  def test_default_config
+    client = JettyHTTPClient.create_client
+    client.close
+    pass
+  end
+
+  import 'java.util.concurrent.ThreadPoolExecutor'
+  import 'java.util.concurrent.ArrayBlockingQueue'
+  import 'java.util.concurrent.TimeUnit'
+  import 'org.eclipse.jetty.util.thread.ExecutorThreadPool'
+
+  def test_custom_executor
+    #FIXME: A bit shaky, fails under 3 threads?
+    executor = ThreadPoolExecutor.new( 3, 10,
+                                       10, TimeUnit::SECONDS,
+                                       ArrayBlockingQueue.new( 10 ) )
+    pool = ExecutorThreadPool.new( executor )
+
+    with_new_client( :thread_pool => pool ) do |client|
+      with_session_handler( client, "/index" ) do |s,x|
+        assert_equal( 200, s.status_code )
+      end
+    end
+
+    pool.stop
+  end
+
+  def test_200
+    with_new_client do |client|
+
+      with_session_handler( client, "/index" ) do |s,x|
+        output_bomb( s ) unless s.status_code == 200
+        assert_equal( 200, s.status_code, "see bomb.out" )
+        assert_match( /Test Index Page/, s.response_stream.to_io.read )
+      end
+
+      with_session_handler( client, "/atom.xml" ) do |s,x|
+        output_bomb( s ) unless s.status_code == 200
+        assert_equal( 200, s.status_code, "see bomb.out" )
+        cl = find_header( s.response_headers, "Content-Length" )
+        assert_operator( cl.to_i, :>, 10_000 )
+      end
+
+    end
+  end
+
+  def test_correct_type
+    with_new_client do |client|
+      client.accepted_content_types = ContentTypeSet.new( [ "text/html" ] )
+      with_session_handler( client, "/index" ) do |s,x|
+        assert_equal( 200, s.status_code )
+        assert_nil( x )
+        assert_match( /^text\/html/,
+                      find_header( s.response_headers, 'Content-Type' ) )
+      end
+    end
+  end
+
+  def test_headers
+    req,rsp = nil
+    with_new_client do |client|
+      with_session_handler( client,
+                            "/echo/header/Accept?noop=3",
+                            true,
+                            { 'Accept' => 'text/plain;moo' } ) do |s,x|
+        assert_equal( 200, s.status_code )
+        assert_equal( 'GET /echo/header/Accept?noop=3',
+                      find_header( s.request_headers, "Request-Line" ) )
+        assert_equal( 'text/plain;moo',
+                      find_header( s.request_headers, 'Accept' ) )
+        assert_equal( 'localhost:19292',
+                      find_header( s.request_headers, 'Host' ) )
+
+        assert_match( /^text\/plain/,
+                      find_header( s.response_headers, 'Content-Type' ) )
+        assert_match( /^text\/plain;moo$/, s.response_stream.to_io.read )
+      end
+    end
+  end
+
+  def test_unknown_host
+    with_new_client( :timeout         => 12_000,
+                     :connect_timeout => 10_000,
+                     :so_timeout      => 10_000,
+                     :idle_timeout    => 10_000 ) do |client|
+      with_session_handler( client,
+                            "http://9xa9.a7v6a7lop-9m9q-w12.com" ) do |s,x|
+        assert_equal( HTTPSession::UNRESOLVED, s.status_code )
+        assert_includes( [ UnresolvedAddressException,
+                           UnknownHostException ], x.class )
+      end
+    end
+  end
+
+  def test_local_connection_refused
+    with_new_client do |client|
+      with_session_handler( client,
+                            "http://localhost:54929/" ) do |s,x|
+        assert_instance_of( ConnectException, x )
+      end
+    end
+  end
+
+  def test_connection_timeout
+    bs = BrokenServer.new
+    bs.start
+
+    #FIXME: Looks like request_timeout is used as this timeout as well.
+    with_new_client( :short => true ) do |client|
+      with_session_handler( client,
+                            "http://localhost:19293/" ) do |s,x|
+        assert_includes( (-42..-40), s.status_code )
+        assert_kind_of( TimeoutException, x )
+      end
+    end
+  ensure
+    bs.stop
+  end
+
+  def test_404
+    with_new_client do |client|
+      with_session_handler( client, "/not-found" ) do |s,x|
+        assert_equal( 404, s.status_code )
+      end
+    end
+  end
+
+  def test_304
+    with_new_client do |client|
+      client.accepted_content_types = ContentTypeSet.new( [ "text/html" ] )
+      with_session_handler( client, "/304" ) do |s,x|
+        assert_equal( 304, s.status_code )
+      end
+    end
+  end
+
+  def test_timeout
+    with_new_client( :short => true ) do |client|
+      with_session_handler( client, "/index?sleep=1.0" ) do |s,x|
+        assert_includes( (-42..-40), s.status_code )
+        assert_kind_of( TimeoutException, x )
+      end
+    end
+    sleep 0.70 # FIXME: Account for test server delay. Should be
+               # joined instead.
+  end
+
+  def test_redirect
+    with_new_client( :handle_redirects_internal => true ) do |client|
+      with_session_handler( client, "/" ) do |s,x|
+        assert_equal( 200, s.status_code )
+        assert_equal( 'http://localhost:19292/index', s.url )
+      end
+    end
+  end
+
+  def test_redirect_with_query_string
+    with_new_client( :handle_redirects_internal => true ) do |client|
+      with_session_handler( client, "/redirects/multi/2?sleep=0" ) do |s,x|
+        assert_equal( 200, s.status_code )
+        assert_equal( 'http://localhost:19292/redirects/multi/1?sleep=0',
+                      s.url )
+        assert_equal( 'GET /redirects/multi/1?sleep=0',
+                      find_header( s.request_headers, "Request-Line" ) )
+      end
+    end
+  end
+
+  def test_redirect_multi_host
+    with_new_client( :handle_redirects_internal => true ) do |client|
+      rurl = 'http://127.0.0.1:19292/index'
+      rurl_e = CGI.escape( rurl )
+      with_session_handler( client, "/redirect?loc=#{rurl_e}" ) do |s,x|
+        assert_equal( 200, s.status_code )
+        assert_equal( rurl, s.url )
+      end
+    end
+  end
+
+  def test_redirect_multi_host_bad
+    skip( "Error: -1 java.lang.NumberFormatException" )
+    with_new_client( :handle_redirects_internal => true ) do |client|
+      rurl = 'http://localhost:19292/index'
+      url = "http://127.0.0.1:19292?redirect?loc=" + CGI.escape( rurl )
+      # Note >?<redirect? above
+      url = "/redirect?loc=" + CGI.escape( url )
+
+      with_session_handler( client, url ) do |s,x|
+        assert_equal( HTTPSession::INVALID_REDIRECT_URL, s.status_code )
+        assert_instance_of( URISyntaxException, x )
+      end
+    end
+  end
+
+  def test_redirect_multi_host_3
+    with_new_client( :handle_redirects_internal => true ) do |client|
+      rurl = 'http://localhost:19292/index'
+      url = "http://127.0.0.1:19292/redirect?loc=" + CGI.escape( rurl )
+      url = "/redirect?loc=" + CGI.escape( url )
+
+      with_session_handler( client, url ) do |s,x|
+        assert_equal( 200, s.status_code )
+        assert_equal( rurl, s.url )
+      end
+    end
+  end
+
+  def test_redirect_multi_host_fragment
+    with_new_client( :handle_redirects_internal => true ) do |client|
+      rurl = '/index#!foo'
+      url = "/redirect?loc=" + CGI.escape( rurl )
+
+      with_session_handler( client, url ) do |s,x|
+        assert_equal( 200, s.status_code )
+        assert_equal( 'http://localhost:19292' + rurl, s.url )
+      end
+    end
+  end
+
+  def test_redirect_bad_host
+    with_new_client( :handle_redirects_internal => true ) do |client|
+      rurl = CGI.escape( 'http://\bad.com/' )
+      with_session_handler( client, "/redirect?loc=#{ rurl }" ) do |s,x|
+        assert_equal( HTTPSession::INVALID_REDIRECT_URL, s.status_code )
+        assert_instance_of( URISyntaxException, x )
+      end
+    end
+  end
+
+  def test_multi_redirect
+    with_new_client( :handle_redirects_internal => true,
+                     :max_redirects => 8 ) do |client|
+      with_session_handler( client, "/redirects/multi/6" ) do |s,x|
+        assert_equal( 200, s.status_code )
+        assert_nil x
+      end
+    end
+  end
+
+  def test_unfollowed_301_redirect
+    with_new_client do |client|
+      with_session_handler( client, "/301" ) do |s,x|
+        assert_equal( 301, s.status_code )
+        lh = find_header( s.response_headers, "Location" )
+        assert_match( %r{/index$}, lh )
+      end
+    end
+  end
+
+  def test_too_many_redirects
+    with_new_client( :handle_redirects_internal => true,
+                     :max_redirects => 18 ) do |client|
+      #FIXME: One redirect off somewhere? 19 fails.
+      with_session_handler( client, "/redirects/multi/20" ) do |s,x|
+        assert_equal( 302, s.status_code, x )
+      end
+    end
+  end
+
+  def test_redirect_timeout
+    skip( "Unreliable timeout with redirects, timing dependent" )
+    with_new_client( :handle_redirects_internal => true,
+                     :short => true ) do |client|
+      with_session_handler( client, "/redirects/multi/3?sleep=0.40" ) do |s,x|
+        assert_instance_of( TimeoutException, x )
+      end
+      sleep 0.80
+    end
+  end
+
+  def test_bad_server_response
+    bs = BrokenServer.new
+    bs.start
+
+    sthread = Thread.new do
+      bs.accept { |sock| sock.write "FU Stinky\r\n" }
+    end
+
+    #FIXME: IllegalStateException on bad HTTP response line?
+    with_new_client do |client|
+      with_session_handler( client, "http://localhost:19293/" ) do |s,x|
+        assert_instance_of( IllegalStateException, x )
+      end
+    end
+
+    sthread.join
+
+  ensure
+    bs.stop
+  end
+
+  def test_empty_server_response
+    bs = BrokenServer.new
+    bs.start
+
+    sthread = Thread.new do
+      bs.accept { |sock| sock.close }
+    end
+
+    with_new_client do |client|
+      with_session_handler( client, "http://localhost:19293/" ) do |s,x|
+        assert_match( /EofException/i, x.class.name )
+      end
+    end
+
+    sthread.join
+
+  ensure
+    bs.stop
+  end
+
+  def test_early_close
+    bs = BrokenServer.new
+    bs.start
+
+    sthread = Thread.new do
+      bs.accept do |sock|
+        sock.write "HTTP/1.1 200 OK\r\n"
+        sock.write "Content-Type: text/plain\r\n"
+        sock.write "Transfer-Encoding: chunked\r\n"
+        sock.write "\r\n"
+        sock.write "FF3DF\r\n"
+        sock.write "An incomplete chunk"
+        sock.write "An incomplete chunk"
+        sock.write "An incomplete chunk"
+        sock.close
+      end
+    end
+
+    with_new_client do |client|
+      with_session_handler( client, "http://localhost:19293/" ) do |s,x|
+        assert_match( /EofException/i, x.class.name )
+      end
+    end
+
+    sthread.join
+
+  ensure
+    bs.stop
+  end
+
+  def test_redirect_early_close
+    bs = BrokenServer.new
+    bs.start
+
+    sthread = Thread.new do
+      bs.accept do |sock|
+        sock.write "HTTP/1.1 302 Found\r\n"
+        sock.write "Location: http://localhost:54929/no-exist\r\n"
+        sock.write "Content-Type: text/plain\r\n"
+        sock.write "Transfer-Encoding: chunked\r\n"
+        sock.write "\r\n"
+        sock.write "FF3DF\r\n"
+        sock.write "An incomplete chunk"
+        sock.write "An incomplete chunk"
+        sock.write "An incomplete chunk"
+        sock.close
+      end
+    end
+
+    with_new_client do |client|
+      with_session_handler( client, "http://localhost:19293/" ) do |s,x|
+        assert_match( /EofException/i, x.class.name )
+      end
+    end
+
+    sthread.join
+
+  ensure
+    bs.stop
+  end
+
+  def test_concurrent
+    with_new_client( :timeout         => 18_000,
+                     :connect_timeout => 15_000,
+                     :so_timeout      => 12_000,
+                     :idle_timeout    => 12_000,
+                     :max_connections_per_address => 4 ) do |client|
+
+      resps = []
+      sessions = (1..19).map do |i|
+        with_session_handler( client, "/index?sleep=0.05&i=#{i}",
+                              false ) do |s,x|
+          sync do
+            resps << [ s.status_code, x ]
+            output_bomb( s ) if s.status_code != 200
+          end
+        end
+      end
+
+      sessions.each { |s| s.wait_for_completion }
+
+      assert_equal( [ [ 200, nil ] ] * 19, resps )
+    end
+  end
+
+  def test_maximum_connections_per_address
+    with_new_client( :timeout         => 12_000,
+                     :connect_timeout => 10_000,
+                     :so_timeout      => 10_000,
+                     :idle_timeout    => 10_000,
+                     :max_connections_per_address => 2 ) do |client|
+
+      resps = []
+      sessions = (1..7).map do |i|
+        with_session_handler( client, "/index?sleep=0.1&con=2&i=#{i}",
+                              false ) do |s,x|
+          sync do
+            resps << [ s.status_code, x ]
+          end
+        end
+      end
+
+      sessions.each { |s| s.wait_for_completion }
+
+      assert_equal( [ [ 200, nil ] ] * 7, resps )
+    end
+  end
+
+  def test_abort_when_too_large
+    with_new_client do |client|
+      with_session_handler( client, "/giant" ) do |s,x|
+        assert_nil( x )
+        assert_equal( HTTPSession::TOO_LARGE, s.status_code )
+      end
+    end
+  end
+
+  def test_abort_when_too_large_length
+    with_new_client do |client|
+      client.max_content_length = 1
+      with_session_handler( client, "/atom.xml" ) do |s,x|
+        assert_nil( x )
+        assert_equal( HTTPSession::TOO_LARGE_LENGTH, s.status_code )
+      end
+    end
+  end
+
+  def test_abort_when_wrong_type
+    with_new_client do |client|
+      client.accepted_content_types = ContentTypeSet.new( [ "gold/*" ] )
+      with_session_handler( client, "/giant" ) do |s,x|
+        assert_nil( x )
+        assert_equal( HTTPSession::NOT_ACCEPTED, s.status_code )
+      end
+    end
+  end
+
+  def sync( &block )
+    @rlock.synchronize( &block )
+  end
+
+  def output_bomb( s )
+    File.open( "bomb.out", "w" ) do |fout|
+      st = s && s.response_stream
+      if st
+        fout.puts st.to_io.read
+      else
+        fout.puts st.to_s
+      end
+    end
+    "See bomb.out"
+  end
+
+  def with_session_handler( client, uri, wait = true, headers = {}, &block )
+    session = client.create_session
+    uri = "http://localhost:#{server.port}#{uri}" unless uri =~ /^http:/
+    session.url = uri
+    headers.each do |k,v|
+      session.add_request_header( Java::iudex.http.Header.new( k, v ) )
+    end
+    handler = TestHandler.new( &block )
+    client.request( session, handler )
+    if wait
+      session.wait_for_completion
+      session.close
+      assert( handler.called?, "Handler should have been called!" )
+    end
+    session
+  end
+
+  def with_new_client( opts = {} )
+    o = if opts.delete( :short )
+          { :timeout          => 400,
+            :so_timeout       => 200,
+            :connect_timeout  => 200,
+            :idle_timeout     => 200 }
+        else
+          { :timeout          => 5000,
+            :so_timeout       => 4000,
+            :connect_timeout  => 3000,
+            :idle_timeout     => 2000 }
+        end
+
+    o = o.merge( { :max_retries      => 0,
+                   :connect_blocking => false } )
+    o = o.merge( opts )
+
+    client = JettyHTTPClient.create_client( o )
+    client.start
+
+    begin
+      yield client
+    ensure
+      client.close
+    end
+
+  end
+
+  class TestHandler < BaseResponseHandler
+
+    def initialize( &block )
+      @block = block
+      @failure = nil
+    end
+
+    def sessionCompleted( session )
+      forward( session, session.error )
+    end
+
+    def called?
+      raise @failure if @failure
+      @block.nil?
+    end
+
+    def forward( s, x = nil )
+      b, @block = @block, nil
+      if b
+        b.call( s, x )
+      else
+        flunk "Handler called twice!"
+      end
+    rescue NativeException => x
+      @failure = x.cause
+    rescue Exception => x
+      @failure = x
+    end
+
+  end
+
+  def find_header( headers, name )
+    cl = headers.find { |h| h.name.to_s == name }
+    cl && cl.value.to_s
+  end
+
+end
+
+if ARGV.delete( '--loop' )
+  loop do
+    failed = MiniTest::Unit.new.run( ARGV )
+    exit!( 1 ) if failed > 0
+  end
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification 
+name: iudex-jetty-httpclient
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 1.1.0
+platform: java
+authors: 
+  - David Kellum
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-11-13 00:00:00 Z
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: iudex-http
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.1.0
+    type: :runtime
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: rjack-jetty
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 7.5.2
+    type: :runtime
+    version_requirements: *id002
+  - !ruby/object:Gem::Dependency 
+    name: hooker
+    prerelease: false
+    requirement: &id003 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.0.0
+    type: :runtime
+    version_requirements: *id003
+  - !ruby/object:Gem::Dependency 
+    name: minitest
+    prerelease: false
+    requirement: &id004 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: "2.3"
+    type: :development
+    version_requirements: *id004
+  - !ruby/object:Gem::Dependency 
+    name: iudex-http-test
+    prerelease: false
+    requirement: &id005 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.1.0
+    type: :development
+    version_requirements: *id005
+  - !ruby/object:Gem::Dependency 
+    name: rjack-logback
+    prerelease: false
+    requirement: &id006 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: "1.0"
+    type: :development
+    version_requirements: *id006
+  - !ruby/object:Gem::Dependency 
+    name: rjack-tarpit
+    prerelease: false
+    requirement: &id007 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.4.0
+    type: :development
+    version_requirements: *id007
+description: |-
+  Iudex is a general purpose web crawler and feed processor in
+  ruby/java. This gem is a Jetty HTTP Client based implementation of the
+  iudex-http interfaces.
+email: 
+  - dek-oss@gravitext.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+  - Manifest.txt
+  - History.rdoc
+  - README.rdoc
+files: 
+  - History.rdoc
+  - Manifest.txt
+  - README.rdoc
+  - Rakefile
+  - pom.xml
+  - lib/iudex-jetty-httpclient/base.rb
+  - lib/iudex-jetty-httpclient.rb
+  - test/setup.rb
+  - test/test_httpclient.rb
+  - lib/iudex-jetty-httpclient/iudex-jetty-httpclient-1.1.0.jar
+  - .gemtest
+homepage: http://github.com/dekellum/iudex
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+  - --main
+  - README.rdoc
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: iudex-jetty-httpclient
+rubygems_version: 1.8.9
+signing_key: 
+specification_version: 3
+summary: Iudex is a general purpose web crawler and feed processor in ruby/java
+test_files: 
+  - test/test_httpclient.rb