manticore 0.1.0-java → 0.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3adc56dba6cc75385969e1b3654322a1c23c71f9
-  data.tar.gz: d695c2db318e5eaec73f6cdbd5641df4fd51af07
+  metadata.gz: e0ee8e57988bc31f65a09326d862683aed668e2a
+  data.tar.gz: 7d7247a8d5087e9630b2ffed540abd16f4930bbd
 SHA512:
-  metadata.gz: 9d09b50c6d313f09bd09cd0948905033079badb23f972430b29edfb6275312c750dc72d8274f76860d093747fd717de98436b7eddd4fb971df9998da4cc68907
-  data.tar.gz: 4d290e644a3a9426608f378aef1deaf625c1fdba81c36d303e63211c884e55391a6aab445eaddc2618df3a261a28bc0cc819af09ac8b44ca9ee5f9d620f2806e
+  metadata.gz: 0728f990ebb06a7251267eb121d1be4bf108a3b13fb742a62289eea6d41f4d376125b79878d2ccb8bed83158783fba1170f826f14fd2b443f31d52c64cff5580
+  data.tar.gz: 3b32c8017107ce22d1da41b56faa3fab5e3a57749cd47814c86ed2632d771c7d923637e1c359b0f28cf0855089f3343ba921d47b0a0b79d8336c85f63b5999be

data/Gemfile

@@ -6,4 +6,5 @@ gemspec
 gem "net-http-server"
 gem "rspec"
 gem "httpclient"
-gem "rack"
+gem "rack"
+gem "rake-compiler"

data/README.md

@@ -22,6 +22,10 @@ Or install it yourself as:
 
   Documentation is available [at rubydoc.info](http://rubydoc.info/github/cheald/manticore/master/frames).
 
+## Performance
+
+  Manticore is [very fast](https://github.com/cheald/manticore/wiki/Performance).
+
 ## Major Features
 
   As it's built on the Apache Commons HTTP components, Manticore is very rich. It includes support for:
@@ -89,7 +93,7 @@ Manticore can perform multiple concurrent execution of requests.
     # These aren't actually executed until #execute! is called.
     # You can define response handlers in a block when you queue the request:
     client.async_get("http://www.google.com") {|req|
-      req.on_success do |response, request|
+      req.on_success do |response|
         puts response.body
       end
 
@@ -100,13 +104,13 @@ Manticore can perform multiple concurrent execution of requests.
 
     # ...or by invoking the method on the queued response returned:
     response = client.async_get("http://www.yahoo.com")
-    response.on_success do |response, request|
+    response.on_success do |response|
       puts "The length of the Yahoo! homepage is #{response.body.length}"
     end
 
     # ...or even by chaining them onto the call
     client.async_get("http://bing.com").
-      on_success {|r, _| puts r.code }.
+      on_success {|r| puts r.code }.
       on_failure {|e| puts "on noes!"}
 
     client.execute!

data/Rakefile

@@ -5,4 +5,17 @@ RSpec::Core::RakeTask.new(:spec) do |spec|
   spec.pattern = 'spec/**/*_spec.rb'
   spec.rspec_opts = ['--tty --color --format documentation']
 end
-task :default => :spec
+task :default => :spec
+
+require 'rake/javaextensiontask'
+
+# Dependency jars for the Kerrigan ext build
+jars = [
+  "#{ENV['MY_RUBY_HOME']}/lib/jruby.jar",
+  "lib/jar/httpcore-4.3.1.jar"
+]
+Rake::JavaExtensionTask.new do |ext|
+  ext.name = "manticore-ext"
+  ext.lib_dir = "lib/jar"
+  ext.classpath = jars.map {|x| File.expand_path x}.join ':'
+end

data/ext/manticore/org/manticore/Manticore.java

@@ -0,0 +1,96 @@
+package org.manticore;
+
+import java.io.IOException;
+import java.io.InputStream;
+import org.jruby.Ruby;
+import org.jruby.RubyClass;
+import org.jruby.RubyString;
+import org.jruby.RubyObject;
+import org.jruby.RubyModule;
+import org.jruby.util.ByteList;
+import org.jruby.anno.JRubyMethod;
+import org.jruby.runtime.Block;
+import org.jruby.runtime.ThreadContext;
+import org.jruby.runtime.load.Library;
+import org.jruby.runtime.ObjectAllocator;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.apache.http.HttpEntity;
+import org.apache.http.util.EntityUtils;
+import org.apache.http.protocol.HTTP;
+import org.jcodings.Encoding;
+
+public class Manticore implements Library {
+
+  public void load(final Ruby ruby, boolean _) {
+    RubyModule manticore = ruby.defineModule("Manticore");
+    RubyClass converter = ruby.defineClassUnder("EntityConverter", ruby.getObject(), new ObjectAllocator() {
+        public IRubyObject allocate(Ruby ruby, RubyClass rc) {
+            return new EntityConverter(ruby, rc);
+        }
+    }, manticore);
+    converter.defineAnnotatedMethods(EntityConverter.class);
+  }
+
+  public class EntityConverter extends RubyObject {
+    public EntityConverter(Ruby ruby, RubyClass rubyClass) {
+      super(ruby, rubyClass);
+    }
+
+    // @JRubyMethod
+    // public IRubyObject initialize(ThreadContext context) {
+    //   return context.nil;
+    // }
+
+    @JRubyMethod(name = "read_entity")
+    public IRubyObject readEntity(ThreadContext context, IRubyObject rEntity, Block block) throws IOException {
+      HttpEntity entity = (HttpEntity)rEntity.toJava(HttpEntity.class);
+      String charset = EntityUtils.getContentCharSet(entity);
+      if(charset == null) { charset = HTTP.DEFAULT_CONTENT_CHARSET; }
+      Encoding encoding;
+      try {
+        encoding = context.getRuntime().getEncodingService().getEncodingFromString(charset);
+      } catch(Throwable e) {
+        encoding = context.getRuntime().getEncodingService().getEncodingFromString(HTTP.DEFAULT_CONTENT_CHARSET);
+      }
+
+      if(block.isGiven()) {
+        return streamEntity(context, entity, encoding, block);
+      } else {
+        return readWholeEntity(context, entity, encoding);
+      }
+    }
+
+    // @JRubyMethod(name = "read_entity")
+    private IRubyObject readWholeEntity(ThreadContext context, HttpEntity entity, Encoding encoding) throws IOException {
+      ByteList bl = new ByteList(EntityUtils.toByteArray(entity), false);
+      return RubyString.newStringShared(context.getRuntime(), bl, encoding);
+    }
+
+    // @JRubyMethod(name = "stream_entity")
+    private IRubyObject streamEntity(ThreadContext context, HttpEntity entity, Encoding encoding, Block block) throws IOException {
+      InputStream instream = entity.getContent();
+      if (instream == null) { return null; }
+      String charset = EntityUtils.getContentCharSet(entity);
+      if(charset == null) { charset = HTTP.DEFAULT_CONTENT_CHARSET; }
+
+      int i = (int)entity.getContentLength();
+      if (i < 0) { i = 4096; }
+
+      if (charset == null) {
+        charset = HTTP.DEFAULT_CONTENT_CHARSET;
+      }
+
+      try {
+        byte[] tmp = new byte[4096];
+        int l;
+        while((l = instream.read(tmp)) != -1) {
+          // String str = new String(tmp, charset);
+          block.call( context, RubyString.newString(context.getRuntime(), new ByteList(tmp, true), encoding) );
+        }
+      } finally {
+        instream.close();
+      }
+      return context.nil;
+    }
+  }
+}

data/lib/jar/lazy_decompressing_stream.patch

@@ -0,0 +1,17 @@
+diff --git a/httpclient/src/main/java/org/apache/http/client/entity/LazyDecompressingInputStream.java b/httpclient/src/main/java/org/apache/http/client/entity/LazyDecompressingInputStream.java
+index 26e4981..ef07caf 100644
+--- a/httpclient/src/main/java/org/apache/http/client/entity/LazyDecompressingInputStream.java
++++ b/httpclient/src/main/java/org/apache/http/client/entity/LazyDecompressingInputStream.java
+@@ -57,6 +57,12 @@ class LazyDecompressingInputStream extends InputStream {
+     }
+ 
+     @Override
++    public int read(final byte[] b, final int off, final int len) throws IOException {
++        initWrapper();
++        return wrapperStream.read(b, off, len);
++    }
++
++    @Override
+     public int available() throws IOException {
+         initWrapper();
+         return wrapperStream.available();

data/lib/manticore.rb

@@ -1,7 +1,12 @@
 require 'java'
+# 4.3.x
 require_relative "./jar/httpcore-4.3.1"
-require_relative "./jar/httpclient-4.3.2"
+require_relative "./jar/httpclient-4.3.2-patched"
 require_relative "./jar/commons-logging-1.1.3"
+require_relative "./jar/manticore-ext"
+
+org.manticore.Manticore.new.load(JRuby.runtime, false)
+
 require_relative "./manticore/version"
 require "addressable/uri"
 

data/lib/manticore/async_response.rb

@@ -10,9 +10,9 @@ module Manticore
     def initialize(client, request, context, body_handler_block)
       @client = client
       @handlers = {
-        success:   ->(request){},
-        failure:   ->(ex){},
-        cancelled: ->{}
+        success:   Proc.new {|resp| resp.body },
+        failure:   Proc.new {},
+        cancelled: Proc.new {}
       }
       body_handler_block.call(self) if body_handler_block
       super request, context, nil
@@ -23,6 +23,7 @@ module Manticore
     def call
       begin
         @client.execute @request, self, @context
+        self
       rescue Java::JavaNet::SocketTimeoutException, Java::OrgApacheHttpConn::ConnectTimeoutException, Java::OrgApacheHttp::NoHttpResponseException => e
         @handlers[:failure].call( Manticore::Timeout.new(e.get_cause) )
       rescue Java::OrgApacheHttpClient::ClientProtocolException => e
@@ -71,7 +72,8 @@ module Manticore
         @response = response
         @code     = response.get_status_line.get_status_code
         @headers  = Hash[* response.get_all_headers.flat_map {|h| [h.get_name.downcase, h.get_value]} ]
-        @handlers[:success].call(self, @request)
+        @callback_result = @handlers[:success].call(self)
+        nil
       rescue => e
         @exception = e
       end

data/lib/manticore/client.rb

@@ -23,17 +23,17 @@ module Manticore
     java_import 'java.util.concurrent.LinkedBlockingQueue'
 
     # The default maximum pool size for requests
-    DEFAULT_MAX_POOL_SIZE      = 50
+    DEFAULT_MAX_POOL_SIZE   = 50
 
     # The default maximum number of threads per route that will be permitted
-    DEFAULT_MAX_PER_ROUTE      = 2
+    DEFAULT_MAX_PER_ROUTE   = 10
 
     DEFAULT_REQUEST_TIMEOUT = 60
-    DEFAULT_SOCKET_TIMEOUT = 10
+    DEFAULT_SOCKET_TIMEOUT  = 10
     DEFAULT_CONNECT_TIMEOUT = 10
-    DEFAULT_MAX_REDIRECTS = 5
+    DEFAULT_MAX_REDIRECTS   = 5
     DEFAULT_EXPECT_CONTINUE = false
-    DEFAULT_STALE_CHECK = false
+    DEFAULT_STALE_CHECK     = false
 
     # Create a new HTTP client with a backing request pool. if you pass a block to the initializer, the underlying
     # {http://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/client/HttpClientBuilder.html HttpClientBuilder}
@@ -269,21 +269,11 @@ module Manticore
 
     # Execute all queued async requests
     #
-    # @return [Array] An array of the results of the on_success bodies for the requests executed.
+    # @return [Array] An array of the responses from the requests executed.
     def execute!
-      tasks = @async_requests.map do |request, response|
-        task = FutureTask.new(response)
-        @executor.submit task
-        task
-      end
+      result = @executor.invoke_all(@async_requests).map(&:get)
       @async_requests.clear
-      tasks.map do |task|
-        begin
-          response = task.get
-        rescue => e
-          raise e.getCause
-        end
-      end
+      result
     end
 
     protected
@@ -330,7 +320,7 @@ module Manticore
     def async_request(request, &block)
       create_executor_if_needed
       response = AsyncResponse.new(@client, request, BasicHttpContext.new, block)
-      @async_requests << [request, response]
+      @async_requests << response
       response
     end
 
@@ -338,7 +328,6 @@ module Manticore
       response = Response.new(request, BasicHttpContext.new, block)
       begin
         @client.execute request, response, response.context
-        response
       rescue Java::JavaNet::SocketTimeoutException, Java::OrgApacheHttpConn::ConnectTimeoutException, Java::OrgApacheHttp::NoHttpResponseException => e
         raise Manticore::Timeout.new(e.get_cause)
       rescue Java::OrgApacheHttpClient::ClientProtocolException => e

data/lib/manticore/response.rb

@@ -8,13 +8,16 @@ module Manticore
   #   @return [Integer] Response code from this response
   # @!attribute [r] context
   #   @return [HttpContext] Context associated with this request/response
+  # @!attribute [r] callback_result
+  #   @return Value returned from any given on_success/response block
   class Response
     include_package "org.apache.http.client"
     include_package "org.apache.http.util"
     include_package "org.apache.http.protocol"
+    # java_import "org.manticore.EntityConverter"
     include ResponseHandler
 
-    attr_reader :headers, :code, :context
+    attr_reader :headers, :code, :context, :request, :callback_result
 
     # Creates a new Response
     #
@@ -34,12 +37,11 @@ module Manticore
       @code     = response.get_status_line.get_status_code
       @headers  = Hash[* response.get_all_headers.flat_map {|h| [h.get_name.downcase, h.get_value]} ]
       if @handler_block
-        @handler_block.call(self)
+        @callback_result = @handler_block.call(self)
       else
         read_body
       end
-    # ensure
-    #   @request.release_connection
+      self
     end
 
     # Fetch the final resolved URL for this response
@@ -53,14 +55,27 @@ module Manticore
       URI.join(host, url.to_s)
     end
 
-    # Fetch the body content of this response
+    # Fetch the body content of this response.
+    # This fetches the input stream in Ruby; this isn't optimal, but it's faster than
+    # fetching the whole thing in Java then UTF-8 encoding it all into a giant Ruby string.
+    #
+    # This permits for streaming response bodies, as well.
+    #
+    # @example Streaming response
+    #
+    #     client.get("http://example.com/resource").on_success do |response|
+    #       response.body do |chunk|
+    #         # Do something with chunk, which is a parsed portion of the returned body
+    #       end
+    #     end
     #
     # @return [String] Reponse body
-    def read_body
+    def read_body(&block)
       @body ||= begin
-       entity = @response.get_entity
-       entity && EntityUtils.to_string(entity)
-      rescue Java::JavaIo::IOException, Java::JavaNet::SocketException => e
+        if entity = @response.get_entity
+          EntityConverter.new.read_entity(entity, &block)
+        end
+      rescue Java::JavaIo::IOException, Java::JavaNet::SocketException, IOError => e
         raise StreamClosedException.new("Could not read from stream: #{e.message} (Did you forget to read #body from your block?)")
       end
     end
@@ -72,5 +87,18 @@ module Manticore
     def length
       (@headers["content-length"] || -1).to_i
     end
+
+    private
+
+    def encode(string, charset)
+      return string if charset.nil?
+      begin
+        string.encode(charset)
+      rescue Encoding::ConverterNotFoundError
+        string.encode("utf-8")
+      rescue
+        string
+      end
+    end
   end
 end

data/lib/manticore/version.rb

@@ -1,3 +1,3 @@
 module Manticore
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/spec/manticore/client_spec.rb

@@ -40,6 +40,30 @@ describe Manticore::Client do
     end
   end
 
+  context "when no response charset is specified" do
+    let(:content_type) { "text/plain" }
+
+    it "should decode response bodies according to the content-type header" do
+      client.get(local_server, headers: {"X-Content-Type" => content_type}).body.encoding.name.should == "ISO-8859-1"
+    end
+  end
+
+  context "when an invalid response charset is specified" do
+    let(:content_type) { "text/plain; charset=bogus" }
+
+    it "should decode the content as UTF-8" do
+      client.get(local_server, headers: {"X-Content-Type" => content_type}).body.encoding.name.should == "ISO-8859-1"
+    end
+  end
+
+  context "when the response charset is UTF-8" do
+    let(:content_type) { "text/plain; charset=utf-8" }
+
+    it "should decode response bodies according to the content-type header" do
+      client.get(local_server, headers: {"X-Content-Type" => content_type}).body.encoding.name.should == "UTF-8"
+    end
+  end
+
   describe "#get" do
     it "should work" do
       response = client.get(local_server)
@@ -136,7 +160,7 @@ describe Manticore::Client do
 
     it "can chain handlers" do
       client.async_get("http://localhost:55441/").on_success {|r| r.code }
-      client.execute!.should == [200]
+      client.execute!.map(&:callback_result).should == [200]
     end
   end
 
@@ -162,7 +186,7 @@ describe Manticore::Client do
         end
       end
 
-      client.execute!.should == ["Result", "Result"]
+      client.execute!.map(&:callback_result).should == ["Result", "Result"]
     end
   end
 

data/spec/spec_helper.rb

@@ -41,16 +41,17 @@ def start_server(port = PORT)
       if request[:headers]["X-Redirect"] && request[:uri][:path] != request[:headers]["X-Redirect"]
         [301, {"Location" => local_server( request[:headers]["X-Redirect"] )}, [""]]
       else
+        content_type = request[:headers]["X-Content-Type"] || "text/plain"
         if request[:headers]["Accept-Encoding"] && request[:headers]["Accept-Encoding"].match("gzip")
           out = StringIO.new('', "w")
           io = Zlib::GzipWriter.new(out, 2)
           io.write JSON.dump(request)
           io.close
           payload = out.string
-          [200, {'Content-Type' => "text/plain", 'Content-Encoding' => "gzip", "Content-Length" => payload.length}, [payload]]
+          [200, {'Content-Type' => content_type, 'Content-Encoding' => "gzip", "Content-Length" => payload.length}, [payload]]
         else
           payload = JSON.dump(request)
-          [200, {'Content-Type' => "text/plain", "Content-Length" => payload.length}, [payload]]
+          [200, {'Content-Type' => content_type, "Content-Length" => payload.length}, [payload]]
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: manticore
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: java
 authors:
 - Chris Heald
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-02-17 00:00:00.000000000 Z
+date: 2014-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -66,9 +66,12 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- ext/manticore/org/manticore/Manticore.java
 - lib/jar/commons-logging-1.1.3.jar
-- lib/jar/httpclient-4.3.2.jar
+- lib/jar/httpclient-4.3.2-patched.jar
 - lib/jar/httpcore-4.3.1.jar
+- lib/jar/lazy_decompressing_stream.patch
+- lib/jar/manticore-ext.jar
 - lib/manticore.rb
 - lib/manticore/async_response.rb
 - lib/manticore/client.rb