manticore 0.6.0-java → 0.7.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitlab-ci.yml +42 -0
- data/.travis.yml +8 -10
- data/CHANGELOG.md +18 -1
- data/Gemfile +4 -2
- data/README.md +3 -1
- data/Rakefile +14 -12
- data/ext/manticore/org/manticore/Manticore.java +13 -4
- data/lib/faraday/adapter/manticore.rb +11 -15
- data/lib/manticore.rb +10 -10
- data/lib/manticore/client.rb +102 -76
- data/lib/manticore/client/proxies.rb +3 -1
- data/lib/manticore/cookie.rb +12 -12
- data/lib/manticore/facade.rb +2 -2
- data/lib/manticore/java_extensions.rb +1 -1
- data/lib/manticore/response.rb +48 -30
- data/lib/manticore/stubbed_response.rb +6 -5
- data/lib/manticore/version.rb +1 -1
- data/lib/manticore_jars.rb +6 -6
- data/lib/org/manticore/manticore-ext.jar +0 -0
- data/manticore.gemspec +4 -2
- data/spec/manticore/client_proxy_spec.rb +5 -4
- data/spec/manticore/client_spec.rb +177 -69
- data/spec/manticore/cookie_spec.rb +9 -10
- data/spec/manticore/facade_spec.rb +6 -6
- data/spec/manticore/response_spec.rb +22 -11
- data/spec/manticore/stubbed_response_spec.rb +5 -5
- data/spec/spec_helper.rb +51 -28
- metadata +28 -14
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 508b2d2c3ce9fc25ca2ac915954807783cc8266aac23936a6896d103121a2f0c
|
4
|
+
data.tar.gz: a8ac97bd7fb22df23ac164d5f06f3a35c926d9eac57ae7bfb8e84c0c8d9e7907
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 56a60c747c187ddd1255e5da7cbc2ad78570c2ebf5390841298f45366133b48c857df8ac7b7b8bdbbb5194546e3b3112a8c08b5109c66a1299f6fe787ae890fc
|
7
|
+
data.tar.gz: 78e29e767486b820f1d1a280ae4db04b1ad33eccb7da7538bae9e48116305b44d069dc13ee322484146f9cce28b6edf71e868d73be8e075957404c50a99416eb
|
data/.gitlab-ci.yml
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
.default: &default
|
2
|
+
variables:
|
3
|
+
TERM: xterm-256color
|
4
|
+
JRUBY_OPTS: --debug
|
5
|
+
cache:
|
6
|
+
paths:
|
7
|
+
- bundler --path vendor/bundle
|
8
|
+
- $HOME/.m2
|
9
|
+
before_script:
|
10
|
+
- apt update && apt install -y git
|
11
|
+
- gem install ruby-maven
|
12
|
+
- bundle install --path vendor/bundle
|
13
|
+
script:
|
14
|
+
- bundle exec rake
|
15
|
+
|
16
|
+
test jruby-9.2:
|
17
|
+
<<: *default
|
18
|
+
image: jruby:9.2
|
19
|
+
artifacts:
|
20
|
+
expire_in: 3 days
|
21
|
+
paths:
|
22
|
+
- coverage
|
23
|
+
|
24
|
+
test jruby-9.1:
|
25
|
+
<<: *default
|
26
|
+
image: jruby:9.1
|
27
|
+
|
28
|
+
test jruby-1.7:
|
29
|
+
<<: *default
|
30
|
+
image: jruby:1.7
|
31
|
+
|
32
|
+
pages:
|
33
|
+
stage: deploy
|
34
|
+
only:
|
35
|
+
- master
|
36
|
+
artifacts:
|
37
|
+
expire_in: 3 days
|
38
|
+
paths:
|
39
|
+
- public
|
40
|
+
script:
|
41
|
+
- mkdir -p public
|
42
|
+
- mv coverage/ public/coverage/
|
data/.travis.yml
CHANGED
@@ -1,22 +1,20 @@
|
|
1
|
+
dist: xenial
|
1
2
|
language: ruby
|
2
|
-
sudo: false
|
3
3
|
cache:
|
4
4
|
- bundler
|
5
5
|
- directories:
|
6
6
|
- $HOME/.m2
|
7
7
|
rvm:
|
8
|
-
- jruby-1.
|
9
|
-
- jruby-9.0.5
|
8
|
+
- jruby-9.1.17.0 # Ruby 2.3
|
9
|
+
- jruby-9.2.13.0 # Ruby 2.5
|
10
10
|
jdk:
|
11
|
-
-
|
12
|
-
-
|
13
|
-
- openjdk7
|
11
|
+
- openjdk9
|
12
|
+
- openjdk11
|
14
13
|
before_install:
|
15
|
-
- gem install
|
16
|
-
- bundle install
|
14
|
+
- gem install bundler -v 1.17.3
|
17
15
|
matrix:
|
18
16
|
include:
|
19
17
|
- rvm: jruby-head
|
20
|
-
jdk:
|
18
|
+
jdk: openjdk10
|
21
19
|
allow_failures:
|
22
|
-
- rvm: jruby-head
|
20
|
+
- rvm: jruby-head
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,23 @@
|
|
1
1
|
## v0.6
|
2
2
|
|
3
|
-
### v0.
|
3
|
+
### v0.7.0
|
4
|
+
|
5
|
+
* Drop support for JRuby 1.7. It probably still works, but we don't test against it anymore
|
6
|
+
* Fix a thread safety issue with regards to adding requests to the parallel execution queue while the client is already processing a queue (#80)
|
7
|
+
|
8
|
+
### v0.6.4
|
9
|
+
|
10
|
+
* client_cert and client_key now take the literal keys as strings, OpenSSL::X509::Certificate/OpenSSL::PKey::Pkey instances, or key file paths. (#77)
|
11
|
+
* Reduced unnecessary string copying (!78 - thanks @kares)
|
12
|
+
|
13
|
+
### v0.6.2-v0.6.3
|
14
|
+
|
15
|
+
* Fixed the use of authentication information in proxy URLs (#71)
|
16
|
+
* Changed the default encoding to UTF-8 when a response MIME is application/json (#70)
|
17
|
+
|
18
|
+
### v0.6.1
|
19
|
+
|
20
|
+
* Manticore will accept a URI object (which it calls #to_s on) as an alternate to a String for the URL in client#get(url)
|
4
21
|
|
5
22
|
### v0.6.0
|
6
23
|
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
# Manticore
|
2
2
|
|
3
|
-
|
3
|
+
**Note**: While I'll continue to maintain the library here, I've moved the canonical copy to Gitlab at https://gitlab.com/cheald/manticore - it is preferred that you submit issues and PRs there.
|
4
|
+
|
5
|
+
[](https://travis-ci.org/cheald/manticore)
|
4
6
|
|
5
7
|
Manticore is a fast, robust HTTP client built on the Apache HTTPClient libraries. It is only compatible with JRuby.
|
6
8
|
|
data/Rakefile
CHANGED
@@ -1,27 +1,27 @@
|
|
1
1
|
require "bundler/gem_tasks"
|
2
2
|
|
3
|
-
require
|
3
|
+
require "rspec/core/rake_task"
|
4
4
|
RSpec::Core::RakeTask.new(:spec) do |spec|
|
5
|
-
spec.pattern =
|
6
|
-
spec.rspec_opts = [
|
5
|
+
spec.pattern = "spec/**/*_spec.rb"
|
6
|
+
spec.rspec_opts = ["--tty --color --format documentation"]
|
7
7
|
end
|
8
8
|
task :default => [:generate_certs, :spec]
|
9
9
|
|
10
10
|
# Download and vendor the jars needed
|
11
|
-
require
|
11
|
+
require "jars/installer"
|
12
12
|
task :install_jars do
|
13
13
|
Jars::Installer.vendor_jars!
|
14
14
|
end
|
15
15
|
|
16
16
|
## Build the Manticore extensions into a jar. You may need to install_jars first
|
17
17
|
# Dependency jars for the Manticore ext build
|
18
|
-
require
|
19
|
-
jars = ["#{ENV[
|
20
|
-
jars.reject! {|j| j.match("manticore-ext") }
|
18
|
+
require "rake/javaextensiontask"
|
19
|
+
jars = ["#{ENV["MY_RUBY_HOME"]}/lib/jruby.jar"] + Dir.glob("lib/**/*.jar")
|
20
|
+
jars.reject! { |j| j.match("manticore-ext") }
|
21
21
|
Rake::JavaExtensionTask.new do |ext|
|
22
22
|
ext.name = "manticore-ext"
|
23
23
|
ext.lib_dir = "lib/org/manticore"
|
24
|
-
ext.classpath = jars.map {|x| File.expand_path x}.join
|
24
|
+
ext.classpath = jars.map { |x| File.expand_path x }.join ":"
|
25
25
|
end
|
26
26
|
|
27
27
|
# Generate all the stuff we need for a full test run
|
@@ -30,10 +30,10 @@ task :generate_certs do
|
|
30
30
|
openssl = `which openssl`.strip
|
31
31
|
keytool = `which keytool`.strip
|
32
32
|
|
33
|
-
Dir.glob("#{root}/*").each {|f| File.unlink f }
|
33
|
+
Dir.glob("#{root}/*").each { |f| File.unlink f }
|
34
34
|
|
35
|
-
# Create the CA
|
36
35
|
cmds = [
|
36
|
+
# Create the CA
|
37
37
|
"#{openssl} genrsa 4096 | #{openssl} pkcs8 -topk8 -nocrypt -out #{root}/root-ca.key",
|
38
38
|
"#{openssl} req -sha256 -x509 -newkey rsa:4096 -nodes -key #{root}/root-ca.key -sha256 -days 365 -out #{root}/root-ca.crt -subj \"/C=US/ST=The Internet/L=The Internet/O=Manticore CA/OU=Manticore/CN=localhost\"",
|
39
39
|
|
@@ -41,15 +41,17 @@ task :generate_certs do
|
|
41
41
|
"#{openssl} genrsa 4096 | #{openssl} pkcs8 -topk8 -nocrypt -out #{root}/client.key",
|
42
42
|
"#{openssl} req -sha256 -key #{root}/client.key -newkey rsa:4096 -out #{root}/client.csr -subj \"/C=US/ST=The Internet/L=The Internet/O=Manticore Client/OU=Manticore/CN=localhost\"",
|
43
43
|
"#{openssl} x509 -req -in #{root}/client.csr -CA #{root}/root-ca.crt -CAkey #{root}/root-ca.key -CAcreateserial -out #{root}/client.crt -sha256 -days 1",
|
44
|
+
"#{openssl} x509 -req -in #{root}/client.csr -CA #{root}/root-ca.crt -CAkey #{root}/root-ca.key -CAcreateserial -out #{root}/client-expired.crt -sha256 -days -7",
|
44
45
|
|
45
46
|
# Create the server cert
|
46
47
|
"#{openssl} genrsa 4096 | #{openssl} pkcs8 -topk8 -nocrypt -out #{root}/host.key",
|
47
48
|
"#{openssl} req -sha256 -key #{root}/host.key -newkey rsa:4096 -out #{root}/host.csr -subj \"/C=US/ST=The Internet/L=The Internet/O=Manticore Host/OU=Manticore/CN=localhost\"",
|
48
49
|
"#{openssl} x509 -req -in #{root}/host.csr -CA #{root}/root-ca.crt -CAkey #{root}/root-ca.key -CAcreateserial -out #{root}/host.crt -sha256 -days 1",
|
50
|
+
"#{openssl} x509 -req -in #{root}/host.csr -CA #{root}/root-ca.crt -CAkey #{root}/root-ca.key -CAcreateserial -out #{root}/host-expired.crt -sha256 -days -7",
|
49
51
|
|
50
52
|
"#{keytool} -import -file #{root}/root-ca.crt -alias rootCA -keystore #{root}/truststore.jks -noprompt -storepass test123",
|
51
53
|
"#{openssl} pkcs12 -export -clcerts -out #{root}/client.p12 -inkey #{root}/client.key -in #{root}/client.crt -certfile #{root}/root-ca.crt -password pass:test123",
|
52
54
|
]
|
53
55
|
|
54
|
-
cmds.each.with_index {|cmd, index| puts "#{index}. #{cmd}"; system cmd }
|
55
|
-
end
|
56
|
+
cmds.each.with_index { |cmd, index| puts "#{index}. #{cmd}"; system cmd }
|
57
|
+
end
|
@@ -39,8 +39,17 @@ public class Manticore implements Library {
|
|
39
39
|
@JRubyMethod(name = "read_entity")
|
40
40
|
public IRubyObject readEntity(ThreadContext context, IRubyObject rEntity, Block block) throws IOException {
|
41
41
|
HttpEntity entity = (HttpEntity)rEntity.toJava(HttpEntity.class);
|
42
|
+
|
42
43
|
String charset = EntityUtils.getContentCharSet(entity);
|
43
|
-
if(charset == null) {
|
44
|
+
if (charset == null) {
|
45
|
+
String mimeType = EntityUtils.getContentMimeType(entity);
|
46
|
+
if ( mimeType != null && mimeType.startsWith("application/json") ) {
|
47
|
+
charset = "UTF-8";
|
48
|
+
} else {
|
49
|
+
charset = HTTP.DEFAULT_CONTENT_CHARSET;
|
50
|
+
}
|
51
|
+
}
|
52
|
+
|
44
53
|
Encoding encoding;
|
45
54
|
try {
|
46
55
|
encoding = context.getRuntime().getEncodingService().getEncodingFromString(charset);
|
@@ -57,7 +66,7 @@ public class Manticore implements Library {
|
|
57
66
|
|
58
67
|
private IRubyObject readWholeEntity(ThreadContext context, HttpEntity entity, Encoding encoding) throws IOException {
|
59
68
|
ByteList bl = new ByteList(EntityUtils.toByteArray(entity), false);
|
60
|
-
return RubyString.
|
69
|
+
return RubyString.newString(context.getRuntime(), bl, encoding);
|
61
70
|
}
|
62
71
|
|
63
72
|
private IRubyObject streamEntity(ThreadContext context, HttpEntity entity, Encoding encoding, Block block) throws IOException {
|
@@ -77,7 +86,7 @@ public class Manticore implements Library {
|
|
77
86
|
byte[] tmp = new byte[4096];
|
78
87
|
int l;
|
79
88
|
while((l = instream.read(tmp)) != -1) {
|
80
|
-
block.call( context, RubyString.
|
89
|
+
block.call( context, RubyString.newStringShared(context.getRuntime(), new ByteList(tmp, 0, l, false), encoding) );
|
81
90
|
}
|
82
91
|
} finally {
|
83
92
|
instream.close();
|
@@ -85,4 +94,4 @@ public class Manticore implements Library {
|
|
85
94
|
return context.nil;
|
86
95
|
}
|
87
96
|
}
|
88
|
-
}
|
97
|
+
}
|
@@ -1,9 +1,9 @@
|
|
1
|
-
require
|
1
|
+
require "faraday"
|
2
2
|
|
3
3
|
module Faraday
|
4
4
|
class Adapter
|
5
5
|
class Manticore < Faraday::Adapter
|
6
|
-
dependency { require
|
6
|
+
dependency { require "manticore" }
|
7
7
|
|
8
8
|
class ParallelManager
|
9
9
|
def client=(client)
|
@@ -20,20 +20,15 @@ module Faraday
|
|
20
20
|
ParallelManager.new
|
21
21
|
end
|
22
22
|
|
23
|
-
def initialize(app, connection_options = {})
|
24
|
-
@connection_options = connection_options
|
25
|
-
super(app)
|
26
|
-
end
|
27
|
-
|
28
23
|
def client(env)
|
29
24
|
@client ||= begin
|
30
25
|
opts = {}
|
31
26
|
if ssl = env[:ssl].to_hash
|
32
27
|
opts[:ssl] = {}
|
33
|
-
opts[:ssl][:verify]
|
34
|
-
opts[:ssl][:ca_file]
|
28
|
+
opts[:ssl][:verify] = ssl[:verify] unless ssl[:verify].nil?
|
29
|
+
opts[:ssl][:ca_file] = ssl[:ca_file]
|
35
30
|
opts[:ssl][:client_cert] = ssl[:client_cert]
|
36
|
-
opts[:ssl][:client_key]
|
31
|
+
opts[:ssl][:client_key] = ssl[:client_key]
|
37
32
|
end
|
38
33
|
conn_opts = @connection_options.dup
|
39
34
|
if conn_opts.key?(:ssl)
|
@@ -50,7 +45,7 @@ module Faraday
|
|
50
45
|
opts = {}
|
51
46
|
if env.key? :request_headers
|
52
47
|
opts[:headers] = env[:request_headers]
|
53
|
-
opts[:headers].reject! {|k, _| k.downcase == "content-length" } # Manticore computes Content-Length
|
48
|
+
opts[:headers].reject! { |k, _| k.downcase == "content-length" } # Manticore computes Content-Length
|
54
49
|
end
|
55
50
|
body = read_body(env)
|
56
51
|
opts[:body] = body if body
|
@@ -60,9 +55,9 @@ module Faraday
|
|
60
55
|
opts[:connect_timeout] = req[:open_timeout] if req.key?(:open_timeout)
|
61
56
|
if prx = req[:proxy]
|
62
57
|
opts[:proxy] = {
|
63
|
-
:url
|
64
|
-
:user
|
65
|
-
:password => prx[:password]
|
58
|
+
:url => prx[:uri].to_s,
|
59
|
+
:user => prx[:user],
|
60
|
+
:password => prx[:password],
|
66
61
|
}
|
67
62
|
end
|
68
63
|
end
|
@@ -106,6 +101,7 @@ module Faraday
|
|
106
101
|
env[:body].respond_to?(:read) ? env[:body].read : env[:body]
|
107
102
|
end
|
108
103
|
end
|
104
|
+
|
109
105
|
register_middleware nil, :manticore => :Manticore
|
110
106
|
end
|
111
|
-
end
|
107
|
+
end
|
data/lib/manticore.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require "java"
|
2
|
+
require "uri"
|
3
|
+
require "cgi"
|
4
4
|
|
5
5
|
require_relative "./manticore_jars.rb"
|
6
6
|
require_relative "./org/manticore/manticore-ext"
|
@@ -42,12 +42,12 @@ module Manticore
|
|
42
42
|
class UnknownException < ManticoreException; end
|
43
43
|
|
44
44
|
require_relative "./manticore/java_extensions"
|
45
|
-
require_relative
|
46
|
-
require_relative
|
47
|
-
require_relative
|
48
|
-
require_relative
|
49
|
-
require_relative
|
50
|
-
require_relative
|
45
|
+
require_relative "./manticore/client/proxies"
|
46
|
+
require_relative "./manticore/client"
|
47
|
+
require_relative "./manticore/response"
|
48
|
+
require_relative "./manticore/stubbed_response"
|
49
|
+
require_relative "./manticore/cookie"
|
50
|
+
require_relative "./manticore/facade"
|
51
51
|
|
52
52
|
include Facade
|
53
53
|
include_http_client
|
@@ -57,4 +57,4 @@ module Manticore
|
|
57
57
|
props.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.SimpleLog")
|
58
58
|
props.setProperty("org.apache.commons.logging.simplelog.log.org.apache.http", "error")
|
59
59
|
end
|
60
|
-
end
|
60
|
+
end
|
data/lib/manticore/client.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require "thread"
|
2
|
+
require "base64"
|
3
|
+
require "weakref"
|
4
|
+
require "openssl_pkcs8_pure"
|
4
5
|
|
5
6
|
module Manticore
|
6
7
|
# @!macro [new] http_method_shared
|
@@ -13,9 +14,9 @@ module Manticore
|
|
13
14
|
# @option options [String] proxy Proxy host in form: http://proxy.org:1234
|
14
15
|
# @option options [Hash] proxy Proxy host in form: {host: 'proxy.org'[, port: 80[, scheme: 'http']]}
|
15
16
|
# @option options [URI] proxy Proxy host as a URI object
|
16
|
-
# @option options [
|
17
|
-
# @option options [
|
18
|
-
# @option options [
|
17
|
+
# @option options [Float] connect_timeout Request-specific connect timeout (in seconds)
|
18
|
+
# @option options [Float] socket_timeout Request-specific socket timeout (in seconds)
|
19
|
+
# @option options [Float] request_timeout Request-specific request timeout (in seconds)
|
19
20
|
# @option options [Integer] max_redirects Request-specific maximum redirect limit
|
20
21
|
# @option options [Boolean] follow_redirects Specify whether this request should follow redirects
|
21
22
|
# @option options [Hash] auth Specify authentication for the request
|
@@ -79,7 +80,7 @@ module Manticore
|
|
79
80
|
include_package "org.apache.http.auth"
|
80
81
|
include_package "java.util.concurrent"
|
81
82
|
include_package "org.apache.http.client.protocol"
|
82
|
-
include_package
|
83
|
+
include_package "org.apache.http.conn.ssl"
|
83
84
|
include_package "java.security.cert"
|
84
85
|
include_package "java.security.spec"
|
85
86
|
include_package "java.security"
|
@@ -105,14 +106,14 @@ module Manticore
|
|
105
106
|
include ProxiesInterface
|
106
107
|
|
107
108
|
# The default maximum pool size for requests
|
108
|
-
DEFAULT_MAX_POOL_SIZE
|
109
|
+
DEFAULT_MAX_POOL_SIZE = 50
|
109
110
|
|
110
111
|
DEFAULT_REQUEST_TIMEOUT = 60
|
111
|
-
DEFAULT_SOCKET_TIMEOUT
|
112
|
+
DEFAULT_SOCKET_TIMEOUT = 10
|
112
113
|
DEFAULT_CONNECT_TIMEOUT = 10
|
113
|
-
DEFAULT_MAX_REDIRECTS
|
114
|
+
DEFAULT_MAX_REDIRECTS = 5
|
114
115
|
DEFAULT_EXPECT_CONTINUE = false
|
115
|
-
DEFAULT_STALE_CHECK
|
116
|
+
DEFAULT_STALE_CHECK = false
|
116
117
|
|
117
118
|
attr_reader :client
|
118
119
|
|
@@ -169,15 +170,15 @@ module Manticore
|
|
169
170
|
# @option options [String] ssl[:keystore_password] (nil) Password used for decrypting the client auth key store
|
170
171
|
# @option options [String] ssl[:keystore_type] (nil) Format of the key store, ie "JKS" or "PKCS12". If left nil, the type will be inferred from the keystore filename.
|
171
172
|
# @option options [String] ssl[:ca_file] (nil) OpenSSL-style path to an X.509 certificate to use to validate SSL certificates
|
172
|
-
# @option options [String]
|
173
|
-
# @option options [String] ssl[:client_key] (nil)
|
173
|
+
# @option options [String|OpenSSL::X509::Certificate] ssl[:client_cert] (nil) A string containing a base64-encoded X.509 certificate, OR a path to an OpenSSL-style X.509 certificate, OR an instance of OpenSSL::X509::Certificate
|
174
|
+
# @option options [String|OpenSSL::PKey::Pkey] ssl[:client_key] (nil) A string containing a base64-encoded RSA key to use for client authentication, OR a path to an OpenSSL-style RSA key, OR an instance of OpenSSL::PKey::PKey
|
174
175
|
# @option options [boolean] ssl[:track_state] (false) Turn on or off connection state tracking. This helps prevent SSL information from leaking across threads, but means that connections
|
175
176
|
# can't be shared across those threads. This should generally be left off unless you know what you're doing.
|
176
177
|
def initialize(options = {})
|
177
178
|
@finalizers = []
|
178
179
|
self.class.shutdown_on_finalize self, @finalizers
|
179
180
|
|
180
|
-
builder
|
181
|
+
builder = client_builder
|
181
182
|
builder.set_user_agent options.fetch(:user_agent, "Manticore #{VERSION}")
|
182
183
|
@options = options
|
183
184
|
@use_cookies = options.fetch(:cookies, false)
|
@@ -195,24 +196,24 @@ module Manticore
|
|
195
196
|
|
196
197
|
@keepalive = options.fetch(:keepalive, true)
|
197
198
|
if @keepalive == false
|
198
|
-
builder.set_connection_reuse_strategy {|response, context| false }
|
199
|
+
builder.set_connection_reuse_strategy { |response, context| false }
|
199
200
|
else
|
200
201
|
builder.set_connection_reuse_strategy DefaultConnectionReuseStrategy.new
|
201
202
|
end
|
202
203
|
|
203
204
|
socket_config_builder = SocketConfig.custom
|
204
|
-
socket_config_builder.set_so_timeout(
|
205
|
-
socket_config_builder.set_tcp_no_delay(
|
205
|
+
socket_config_builder.set_so_timeout(options.fetch(:socket_timeout, DEFAULT_SOCKET_TIMEOUT) * 1000)
|
206
|
+
socket_config_builder.set_tcp_no_delay(options.fetch(:tcp_no_delay, true))
|
206
207
|
builder.set_default_socket_config socket_config_builder.build
|
207
208
|
|
208
209
|
builder.set_connection_manager pool(options)
|
209
210
|
|
210
211
|
request_config = RequestConfig.custom
|
211
|
-
request_config.set_connection_request_timeout
|
212
|
-
request_config.set_connect_timeout
|
213
|
-
request_config.set_socket_timeout
|
214
|
-
request_config.set_max_redirects
|
215
|
-
request_config.set_expect_continue_enabled
|
212
|
+
request_config.set_connection_request_timeout options.fetch(:request_timeout, DEFAULT_REQUEST_TIMEOUT) * 1000
|
213
|
+
request_config.set_connect_timeout options.fetch(:connect_timeout, DEFAULT_CONNECT_TIMEOUT) * 1000
|
214
|
+
request_config.set_socket_timeout options.fetch(:socket_timeout, DEFAULT_SOCKET_TIMEOUT) * 1000
|
215
|
+
request_config.set_max_redirects options.fetch(:max_redirects, DEFAULT_MAX_REDIRECTS)
|
216
|
+
request_config.set_expect_continue_enabled options.fetch(:expect_continue, DEFAULT_EXPECT_CONTINUE)
|
216
217
|
request_config.set_stale_connection_check_enabled options.fetch(:stale_check, DEFAULT_STALE_CHECK)
|
217
218
|
request_config.set_circular_redirects_allowed false
|
218
219
|
|
@@ -222,7 +223,7 @@ module Manticore
|
|
222
223
|
@client = builder.build
|
223
224
|
finalize @client, :close
|
224
225
|
@options = options
|
225
|
-
@async_requests =
|
226
|
+
@async_requests = Queue.new
|
226
227
|
@stubs = {}
|
227
228
|
end
|
228
229
|
|
@@ -233,7 +234,7 @@ module Manticore
|
|
233
234
|
max: stats.get_max,
|
234
235
|
leased: stats.get_leased,
|
235
236
|
pending: stats.get_pending,
|
236
|
-
available: stats.get_available
|
237
|
+
available: stats.get_available,
|
237
238
|
}
|
238
239
|
end
|
239
240
|
|
@@ -333,8 +334,9 @@ module Manticore
|
|
333
334
|
# @return [Array] An array of the responses from the requests executed.
|
334
335
|
def execute!
|
335
336
|
method = executor.java_method(:submit, [java.util.concurrent.Callable.java_class])
|
336
|
-
|
337
|
-
|
337
|
+
|
338
|
+
result = []
|
339
|
+
result << method.call(@async_requests.pop) until @async_requests.empty?
|
338
340
|
result.map do |future|
|
339
341
|
begin
|
340
342
|
future.get
|
@@ -357,8 +359,8 @@ module Manticore
|
|
357
359
|
|
358
360
|
def self.shutdown_on_finalize(client, objs)
|
359
361
|
ObjectSpace.define_finalizer client, -> {
|
360
|
-
|
361
|
-
|
362
|
+
objs.each { |obj, args| obj.send(*args) rescue nil }
|
363
|
+
}
|
362
364
|
end
|
363
365
|
|
364
366
|
protected
|
@@ -383,13 +385,15 @@ module Manticore
|
|
383
385
|
end
|
384
386
|
|
385
387
|
def pool_builder(options)
|
386
|
-
http_sf
|
388
|
+
http_sf = PlainConnectionSocketFactory.new
|
387
389
|
|
390
|
+
# :nocov:
|
388
391
|
if options[:ignore_ssl_validation]
|
389
|
-
$stderr.puts
|
392
|
+
$stderr.puts "The options[:ignore_ssl_validation] setting is deprecated in favor of options[:ssl][:verify]"
|
390
393
|
options[:ssl] ||= {}
|
391
|
-
options[:ssl]
|
394
|
+
options[:ssl] = {:verify => !options.delete(:ignore_ssl_validation)}.merge(options[:ssl])
|
392
395
|
end
|
396
|
+
# :nocov:
|
393
397
|
|
394
398
|
https_sf = ssl_socket_factory_from_options options.fetch(:ssl, {})
|
395
399
|
registry = RegistryBuilder.create.register("http", http_sf).register("https", https_sf).build
|
@@ -416,10 +420,10 @@ module Manticore
|
|
416
420
|
|
417
421
|
def request(klass, url, options, &block)
|
418
422
|
req, context = request_from_options(klass, url, options)
|
419
|
-
async
|
420
|
-
background
|
423
|
+
async = options.delete(:async)
|
424
|
+
background = options.delete(:async_background)
|
421
425
|
create_executor_if_needed if (background || async)
|
422
|
-
response
|
426
|
+
response = response_object_for(req, context, &block)
|
423
427
|
|
424
428
|
if async
|
425
429
|
@async_requests << response
|
@@ -439,13 +443,14 @@ module Manticore
|
|
439
443
|
|
440
444
|
match_key = @stubs.keys.find { |k| request_uri.match(k) }
|
441
445
|
if match_key
|
442
|
-
StubbedResponse.new(self, request, context, &block).stub(
|
446
|
+
StubbedResponse.new(self, request, context, &block).stub(@stubs[match_key])
|
443
447
|
else
|
444
448
|
Response.new(self, request, context, &block)
|
445
449
|
end
|
446
450
|
end
|
447
451
|
|
448
452
|
def uri_from_url_and_options(url, options)
|
453
|
+
url = url.to_s if url.is_a?(URI)
|
449
454
|
builder = URIBuilder.new(url)
|
450
455
|
pairs = struct_to_name_value_pairs(options[:query])
|
451
456
|
builder.add_parameters pairs unless pairs.empty?
|
@@ -455,7 +460,7 @@ module Manticore
|
|
455
460
|
def request_from_options(klass, url, options)
|
456
461
|
req = klass.new uri_from_url_and_options(url, options).to_s
|
457
462
|
|
458
|
-
if (
|
463
|
+
if (options[:params] || options[:body] || options[:entity]) && req.kind_of?(HttpEntityEnclosingRequestBase)
|
459
464
|
if options[:params]
|
460
465
|
pairs = struct_to_name_value_pairs(options[:params])
|
461
466
|
encoding = minimum_encoding_for options[:params].to_s
|
@@ -478,23 +483,30 @@ module Manticore
|
|
478
483
|
if req_options[:proxy]
|
479
484
|
config.set_proxy get_proxy_host(req_options[:proxy])
|
480
485
|
end
|
481
|
-
config.set_max_redirects req_options[:max_redirects]
|
482
|
-
config.set_redirects_enabled !!req_options[:follow_redirects]
|
483
|
-
config.set_connect_timeout req_options[:connect_timeout] * 1000
|
484
|
-
config.set_socket_timeout req_options[:socket_timeout] * 1000
|
486
|
+
config.set_max_redirects req_options[:max_redirects] if req_options[:max_redirects]
|
487
|
+
config.set_redirects_enabled !!req_options[:follow_redirects] if req_options.fetch(:follow_redirects, nil) != nil
|
488
|
+
config.set_connect_timeout req_options[:connect_timeout] * 1000 if req_options[:connect_timeout]
|
489
|
+
config.set_socket_timeout req_options[:socket_timeout] * 1000 if req_options[:socket_timeout]
|
485
490
|
config.set_connection_request_timeout req_options[:request_timeout] * 1000 if req_options[:request_timeout]
|
486
491
|
req.set_config config.build
|
487
492
|
end
|
488
493
|
|
489
|
-
|
490
|
-
options[:headers].each {|k, v| req[k] = v } if options.key?(:headers)
|
491
|
-
|
494
|
+
headers = []
|
492
495
|
# Support keepalive on HTTP/1.0 connections
|
493
|
-
|
496
|
+
headers.push BasicHeader.new("Connection", "Keep-Alive") if @keepalive
|
497
|
+
|
498
|
+
if options.key?(:headers)
|
499
|
+
options[:headers].each do |k, v|
|
500
|
+
Array(v).each do |_v|
|
501
|
+
headers.push BasicHeader.new(k, _v)
|
502
|
+
end
|
503
|
+
end
|
504
|
+
end
|
505
|
+
req.set_headers headers.to_java(BasicHeader) unless headers.empty?
|
494
506
|
|
495
507
|
context = HttpClientContext.new
|
496
508
|
proxy_user = req_options[:proxy].is_a?(Hash) && (req_options[:proxy][:user] || req_options[:proxy][:username])
|
497
|
-
auth_from_options(req, req_options, context)
|
509
|
+
auth_from_options(req, req_options, context)
|
498
510
|
|
499
511
|
if @use_cookies == :per_request
|
500
512
|
store = BasicCookieStore.new
|
@@ -529,7 +541,16 @@ module Manticore
|
|
529
541
|
|
530
542
|
def auth_from_options(req, options, context)
|
531
543
|
proxy = options.fetch(:proxy, {})
|
532
|
-
|
544
|
+
|
545
|
+
proxy_user, proxy_pass = if proxy.is_a?(String)
|
546
|
+
proxy_uri = URI.parse(proxy)
|
547
|
+
[proxy_uri.user, proxy_uri.password]
|
548
|
+
else
|
549
|
+
[(proxy[:user] || proxy[:username]),
|
550
|
+
(proxy[:pass] || proxy[:password])]
|
551
|
+
end
|
552
|
+
|
553
|
+
if options[:auth] || proxy_user
|
533
554
|
provider = BasicCredentialsProvider.new
|
534
555
|
if options[:auth]
|
535
556
|
username = options[:auth][:user] || options[:auth][:username]
|
@@ -547,10 +568,8 @@ module Manticore
|
|
547
568
|
end
|
548
569
|
end
|
549
570
|
|
550
|
-
if
|
551
|
-
|
552
|
-
password = proxy[:pass] || proxy[:password]
|
553
|
-
provider.set_credentials AuthScope.new(get_proxy_host(proxy)), UsernamePasswordCredentials.new(username, password)
|
571
|
+
if proxy_user
|
572
|
+
provider.set_credentials AuthScope.new(get_proxy_host(proxy)), UsernamePasswordCredentials.new(proxy_user, proxy_pass)
|
554
573
|
end
|
555
574
|
context.set_credentials_provider(provider)
|
556
575
|
end
|
@@ -561,9 +580,9 @@ module Manticore
|
|
561
580
|
when nil
|
562
581
|
[]
|
563
582
|
when Hash
|
564
|
-
value.flat_map {|key, val| struct_to_name_value_pairs val, namespace ? "#{namespace}[#{key}]" : key }
|
583
|
+
value.flat_map { |key, val| struct_to_name_value_pairs val, namespace ? "#{namespace}[#{key}]" : key }
|
565
584
|
when Array
|
566
|
-
value.flat_map {|val| struct_to_name_value_pairs val, namespace }
|
585
|
+
value.flat_map { |val| struct_to_name_value_pairs val, namespace }
|
567
586
|
else
|
568
587
|
BasicNameValuePair.new(namespace, value.to_s)
|
569
588
|
end
|
@@ -572,6 +591,7 @@ module Manticore
|
|
572
591
|
# Apache HTTP assumes ISO_8859_1 for StringEntities; we'll try to be nice and pass that when possible
|
573
592
|
# so that it doesn't have to any multibyte work.
|
574
593
|
ISO_8859_1 = "ISO-8859-1".freeze
|
594
|
+
|
575
595
|
def minimum_encoding_for(string)
|
576
596
|
if string.ascii_only?
|
577
597
|
ISO_8859_1
|
@@ -623,44 +643,50 @@ module Manticore
|
|
623
643
|
end
|
624
644
|
|
625
645
|
KEY_EXTRACTION_REGEXP = /(?:^-----BEGIN(.* )PRIVATE KEY-----\n)(.*?)(?:-----END\1PRIVATE KEY.*$)/m
|
646
|
+
|
626
647
|
def setup_key_store(ssl_options, context)
|
627
648
|
key_store = get_store(:keystore, ssl_options) if ssl_options.key?(:keystore)
|
628
649
|
keystore_password = (ssl_options[:keystore_password] || "").to_java.toCharArray
|
629
650
|
|
630
651
|
# Support OpenSSL-style bare X.509 certs with an RSA key
|
631
|
-
# This is really dumb - we have to b64-decode the key ourselves, and we can only support PKCS8
|
632
652
|
if ssl_options[:client_cert] && ssl_options[:client_key]
|
633
653
|
key_store ||= blank_keystore
|
634
654
|
certs, key = nil, nil
|
635
|
-
|
636
|
-
|
637
|
-
|
655
|
+
|
656
|
+
cert_str = if ssl_options[:client_cert].is_a?(OpenSSL::X509::Certificate)
|
657
|
+
ssl_options[:client_cert].to_s
|
658
|
+
elsif ssl_options[:client_cert].is_a?(String) && File.exists?(ssl_options[:client_cert])
|
659
|
+
File.read(ssl_options[:client_cert])
|
660
|
+
else
|
661
|
+
ssl_options[:client_cert].to_s
|
662
|
+
end
|
663
|
+
|
664
|
+
cert_stream = java.io.ByteArrayInputStream.new(cert_str.strip.to_java_bytes)
|
665
|
+
certs = CertificateFactory.get_instance("X509").generate_certificates(cert_stream).to_array([].to_java(Certificate))
|
666
|
+
|
667
|
+
key_str = if ssl_options[:client_key].is_a?(OpenSSL::PKey::PKey)
|
668
|
+
ssl_options[:client_key].to_pem_pkcs8
|
669
|
+
elsif ssl_options[:client_key].is_a?(String) && File.exists?(ssl_options[:client_key])
|
670
|
+
File.read(ssl_options[:client_key])
|
671
|
+
else
|
672
|
+
ssl_options[:client_key].to_s
|
673
|
+
end
|
638
674
|
|
639
675
|
# Add each of the keys in the given keyfile into the keystore.
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
key_store.set_key_entry("key-#{Digest::SHA1.hexdigest(body)}", key, keystore_password, certs)
|
649
|
-
end
|
676
|
+
key_parts = key_str.scan(KEY_EXTRACTION_REGEXP)
|
677
|
+
key_parts.each do |type, b64key|
|
678
|
+
body = Base64.decode64 b64key
|
679
|
+
spec = PKCS8EncodedKeySpec.new(body.strip.to_java_bytes)
|
680
|
+
type = type.strip
|
681
|
+
type = "RSA" if type == ""
|
682
|
+
key = KeyFactory.getInstance(type).generatePrivate(spec)
|
683
|
+
key_store.set_key_entry("key-#{Digest::SHA1.hexdigest(body)}", key, keystore_password, certs)
|
650
684
|
end
|
651
685
|
end
|
652
686
|
|
653
687
|
context.load_key_material(key_store, keystore_password) if key_store
|
654
688
|
end
|
655
689
|
|
656
|
-
def get_trust_store(options)
|
657
|
-
get_store :truststore, options
|
658
|
-
end
|
659
|
-
|
660
|
-
def get_key_store(options)
|
661
|
-
get_store :keystore, options
|
662
|
-
end
|
663
|
-
|
664
690
|
def get_store(prefix, options)
|
665
691
|
KeyStore.get_instance(options[:"#{prefix}_type"] || guess_store_type(options[prefix])).tap do |store|
|
666
692
|
instream = open(options[prefix], "rb").to_inputstream
|
@@ -669,7 +695,7 @@ module Manticore
|
|
669
695
|
end
|
670
696
|
|
671
697
|
def blank_keystore
|
672
|
-
KeyStore.get_instance(KeyStore.get_default_type).tap {|k| k.load(nil, nil) }
|
698
|
+
KeyStore.get_instance(KeyStore.get_default_type).tap { |k| k.load(nil, nil) }
|
673
699
|
end
|
674
700
|
|
675
701
|
def guess_store_type(filename)
|
@@ -682,7 +708,7 @@ module Manticore
|
|
682
708
|
|
683
709
|
def treat_params_as_query(options)
|
684
710
|
if options.key?(:params) && !options.key?(:query)
|
685
|
-
options.dup.tap {|o| o[:query] = o.delete(:params) }
|
711
|
+
options.dup.tap { |o| o[:query] = o.delete(:params) }
|
686
712
|
else
|
687
713
|
options
|
688
714
|
end
|