pantopoda 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/pantopoda/version.rb +1 -1
- data/lib/pantopoda.rb +2 -2
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2942d25ee2525eb4cf2ee047da228a329e2fadd6
|
4
|
+
data.tar.gz: 9c92b461efece06cea336411b4dfabd780698c9a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7a7ac84b43458113b8a7c86ac76328d08affa7fc4da96ce2a05c97be8e5fcfa78f27ebdeb39dba7df6c47cabef5b0707ff504ccb74ed0ebe7d9dc808a748c5bc
|
7
|
+
data.tar.gz: 3ba1c4f9a5d35527adb4e69cd21d6a46127712be09e8e5e23848747083f449adbbf99a9bca1e5919a4dfd83cbb28e6473f4675398477f02c94ecca34987f8725
|
data/lib/pantopoda/version.rb
CHANGED
data/lib/pantopoda.rb
CHANGED
@@ -23,7 +23,7 @@ module Pantopoda
|
|
23
23
|
# Defaults to -1 so it will always keep running until it runs out of urls
|
24
24
|
max_urls = options[:max_urls] ? options[:max_urls] : nil
|
25
25
|
|
26
|
-
@hydra =
|
26
|
+
@hydra = Typhoeus::Hydra.new(:max_concurrency => threads)
|
27
27
|
@global_visited = BloomFilter::Native.new(:size => 1000000, :hashes => 5, :seed => 1, :bucket => 8, :raise => false)
|
28
28
|
@global_queue = []
|
29
29
|
|
@@ -35,7 +35,7 @@ module Pantopoda
|
|
35
35
|
begin
|
36
36
|
ip,port,user,pass = nil
|
37
37
|
|
38
|
-
request =
|
38
|
+
request = Typhoeus::Request.new(q, :timeout => 10000, :follow_location => true) if ip == nil
|
39
39
|
request.on_complete do |response|
|
40
40
|
yield response
|
41
41
|
links = Nokogiri::HTML.parse(response.body).xpath('.//a/@href')
|