phantom_client 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +28 -0
- data/Rakefile +17 -0
- data/bin/phantom_client +25 -0
- data/lib/phantom_client/phantom_client.rb +71 -0
- data/lib/phantom_client/phantom_client_connection.rb +13 -0
- data/lib/phantom_client.rb +4 -0
- metadata +72 -0
data/README.rdoc
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
== Phantom Client
|
2
|
+
Phantom Client is a clientlibrary to Phantom Proxy
|
3
|
+
(see https://github.com/experteer/phantom_proxy).
|
4
|
+
It's a thin layer around Net::HTTP to fetch a page
|
5
|
+
through the Phantom Proxy with specific options.
|
6
|
+
|
7
|
+
== Installation
|
8
|
+
gem build phantom_client.gemspec
|
9
|
+
gem install phantom_client-*.gem
|
10
|
+
|
11
|
+
== Usage
|
12
|
+
|
13
|
+
Usage example:
|
14
|
+
|
15
|
+
require 'phantom_client'
|
16
|
+
|
17
|
+
client = PhantomJSProxy::PhantomJSClient.new([{:addr => "127.0.0.1", :port => 5000}])
|
18
|
+
dom = client.get("http://foo.de/", {:imageOnly => false, :withIframes => true})
|
19
|
+
|
20
|
+
or in the bash
|
21
|
+
|
22
|
+
phantom_client http://foo.de/ [proxy_ip:port []]
|
23
|
+
|
24
|
+
== TODO
|
25
|
+
* nicer API
|
26
|
+
* gemify
|
27
|
+
* namespace cleanup
|
28
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
|
3
|
+
task :build do
|
4
|
+
sh "gem build phclient.gemspec"
|
5
|
+
end
|
6
|
+
|
7
|
+
task :install do
|
8
|
+
sh "gem install phantom_client-*.gem"
|
9
|
+
end
|
10
|
+
|
11
|
+
task :uninstall do
|
12
|
+
sh "gem uninstall phantom_client"
|
13
|
+
end
|
14
|
+
|
15
|
+
task :clean do
|
16
|
+
sh "rm phantom_client-*.gem"
|
17
|
+
end
|
data/bin/phantom_client
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'phantom_client'
|
5
|
+
|
6
|
+
url = nil
|
7
|
+
proxy_list = []
|
8
|
+
|
9
|
+
ARGV.each{ |arg|
|
10
|
+
if url == nil
|
11
|
+
url = arg
|
12
|
+
else
|
13
|
+
newE = {}
|
14
|
+
newE[:addr] = arg.split(':')[0]
|
15
|
+
newE[:port] = arg.split(':')[1]
|
16
|
+
proxy_list.push newE
|
17
|
+
end
|
18
|
+
}
|
19
|
+
|
20
|
+
if url == nil
|
21
|
+
return
|
22
|
+
end
|
23
|
+
|
24
|
+
client = PhantomJSProxy::PhantomJSClient.new(proxy_list)
|
25
|
+
puts client.get(url, {:imageOnly => false, :withIframes => true})
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'uri'
|
3
|
+
|
4
|
+
module PhantomJSProxy
|
5
|
+
class PhantomJSClient
|
6
|
+
|
7
|
+
attr_accessor :proxy_addr
|
8
|
+
attr_accessor :proxy_port
|
9
|
+
attr_accessor :proxy_list
|
10
|
+
attr_accessor :connection
|
11
|
+
|
12
|
+
def initialize(addr_list=[], con = PhantomJSClientConnection.new)
|
13
|
+
@proxy_list = addr_list
|
14
|
+
@connection = con
|
15
|
+
end
|
16
|
+
|
17
|
+
def get(addr, options=nil)
|
18
|
+
url = URI.parse(addr)
|
19
|
+
req = Net::HTTP::Get.new(url.path)
|
20
|
+
req['User-Agent'] = "PhantomJSClient"
|
21
|
+
if /\?/.match(addr)
|
22
|
+
req.body = addr.split('?')[1]
|
23
|
+
end
|
24
|
+
|
25
|
+
if options && options['imageOnly']
|
26
|
+
req['Get-Page-As-Image'] = options['imageOnly']
|
27
|
+
puts "Do image only"
|
28
|
+
end
|
29
|
+
|
30
|
+
if options && options['withIframes']
|
31
|
+
req['HTTP_GET_PAGE_WITH_IFRAMES'] = options['withIframes']
|
32
|
+
puts "Do fetch iframes"
|
33
|
+
end
|
34
|
+
#::Proxy(@proxy_addr, @proxy_port)
|
35
|
+
|
36
|
+
#element = get_proxy()
|
37
|
+
#begin
|
38
|
+
# @connection.do_request(element, url, req)
|
39
|
+
#rescue
|
40
|
+
# return "Could not connect to proxy"
|
41
|
+
#end
|
42
|
+
do_get(url, req, 10)
|
43
|
+
end
|
44
|
+
|
45
|
+
def do_get(url, req, count)
|
46
|
+
element = get_proxy()
|
47
|
+
if element[:addr] && element[:port]
|
48
|
+
puts "try: "+element[:addr]+", "+element[:port]
|
49
|
+
else
|
50
|
+
puts "DUMMY TRY"
|
51
|
+
end
|
52
|
+
|
53
|
+
begin
|
54
|
+
@connection.do_request(element, url, req)
|
55
|
+
rescue
|
56
|
+
if count == 0
|
57
|
+
return "Could not connect to proxy"
|
58
|
+
end
|
59
|
+
do_get(url, req, count-1)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def get_proxy
|
64
|
+
element_n = rand(@proxy_list.count)
|
65
|
+
if @proxy_list.count > 0
|
66
|
+
return @proxy_list[element_n]
|
67
|
+
end
|
68
|
+
return {:addr => nil, :port => nil}
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'uri'
|
3
|
+
|
4
|
+
module PhantomJSProxy
|
5
|
+
class PhantomJSClientConnection
|
6
|
+
def do_request(proxy, url, req)
|
7
|
+
res = Net::HTTP::Proxy(proxy[:addr], proxy[:port]).start(url.host, url.port) {|http|
|
8
|
+
http.request(req)
|
9
|
+
}
|
10
|
+
return res.body
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
metadata
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: phantom_client
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 23
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
- 0
|
10
|
+
version: 1.0.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Daniel Sudmann
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2012-03-23 00:00:00 +01:00
|
19
|
+
default_executable:
|
20
|
+
dependencies: []
|
21
|
+
|
22
|
+
description: This is a phantomjs Proxy client
|
23
|
+
email: suddani@googlemail.com
|
24
|
+
executables:
|
25
|
+
- phantom_client
|
26
|
+
extensions: []
|
27
|
+
|
28
|
+
extra_rdoc_files: []
|
29
|
+
|
30
|
+
files:
|
31
|
+
- lib/phantom_client/phantom_client_connection.rb
|
32
|
+
- lib/phantom_client/phantom_client.rb
|
33
|
+
- lib/phantom_client.rb
|
34
|
+
- bin/phantom_client
|
35
|
+
- README.rdoc
|
36
|
+
- Rakefile
|
37
|
+
has_rdoc: true
|
38
|
+
homepage: http://experteer.com
|
39
|
+
licenses: []
|
40
|
+
|
41
|
+
post_install_message:
|
42
|
+
rdoc_options: []
|
43
|
+
|
44
|
+
require_paths:
|
45
|
+
- lib
|
46
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
47
|
+
none: false
|
48
|
+
requirements:
|
49
|
+
- - ">="
|
50
|
+
- !ruby/object:Gem::Version
|
51
|
+
hash: 3
|
52
|
+
segments:
|
53
|
+
- 0
|
54
|
+
version: "0"
|
55
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
56
|
+
none: false
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
hash: 3
|
61
|
+
segments:
|
62
|
+
- 0
|
63
|
+
version: "0"
|
64
|
+
requirements: []
|
65
|
+
|
66
|
+
rubyforge_project:
|
67
|
+
rubygems_version: 1.5.2
|
68
|
+
signing_key:
|
69
|
+
specification_version: 3
|
70
|
+
summary: This is a phantomjs Proxy client
|
71
|
+
test_files: []
|
72
|
+
|