benchmark-http 0.12.0 → 0.13.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/benchmark-http.gemspec +3 -1
- data/lib/benchmark/http/spider.rb +6 -3
- data/lib/benchmark/http/version.rb +1 -1
- metadata +30 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 13d218f8460468f4e36bfee9339ce124a4a850681c8fb5f007290cfadec71440
|
|
4
|
+
data.tar.gz: 67f837ffda547018fd5a15bb21e31fa5fb70c2b0506fa5ae0628bbfe7af65e1a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 93f1d330ac3e0eb517deb422a41dc31796b2018949e91a7c1a694c53710d895ffb9af23ac95f41523c03cb0dcda243fd59d3781e4545243c1f0ce3eb4019ab98
|
|
7
|
+
data.tar.gz: c88bb8bdd60d1a281381f24b30e84a09fc63aa5642a58a1c878dc35df33d7bdcc04c0bf2979fec219a14a10ea0eb40ad59153ac21d7ebbda56ac0b7b26d91daf
|
data/benchmark-http.gemspec
CHANGED
|
@@ -24,7 +24,9 @@ Gem::Specification.new do |spec|
|
|
|
24
24
|
spec.add_dependency("trenni-sanitize")
|
|
25
25
|
|
|
26
26
|
spec.add_dependency('samovar', "~> 2.0")
|
|
27
|
-
|
|
27
|
+
|
|
28
|
+
spec.add_development_dependency "covered"
|
|
29
|
+
spec.add_development_dependency "async-rspec"
|
|
28
30
|
spec.add_development_dependency "bundler"
|
|
29
31
|
spec.add_development_dependency "rake", "~> 10.0"
|
|
30
32
|
spec.add_development_dependency "rspec", "~> 3.0"
|
|
@@ -34,8 +34,9 @@ module Benchmark
|
|
|
34
34
|
class Spider
|
|
35
35
|
include Async::Await
|
|
36
36
|
|
|
37
|
-
def initialize(depth: nil)
|
|
37
|
+
def initialize(depth: nil, ignore: nil)
|
|
38
38
|
@depth = depth
|
|
39
|
+
@ignore = ignore
|
|
39
40
|
end
|
|
40
41
|
|
|
41
42
|
def extract_links(url, response)
|
|
@@ -71,11 +72,13 @@ module Benchmark
|
|
|
71
72
|
end
|
|
72
73
|
|
|
73
74
|
async def fetch(statistics, client, url, depth = @depth, fetched = Set.new, &block)
|
|
74
|
-
if depth
|
|
75
|
+
if depth&.zero?
|
|
75
76
|
Async.logger.warn(self) {"Exceeded depth while trying to visit #{url}!"}
|
|
76
77
|
return
|
|
77
78
|
elsif fetched.include?(url)
|
|
78
79
|
return
|
|
80
|
+
elsif @ignore&.match?(url.path)
|
|
81
|
+
return
|
|
79
82
|
end
|
|
80
83
|
|
|
81
84
|
fetched << url
|
|
@@ -90,7 +93,7 @@ module Benchmark
|
|
|
90
93
|
location = url + response.headers['location']
|
|
91
94
|
if location.host == url.host
|
|
92
95
|
Async.logger.debug(self) {"Following redirect to #{location}..."}
|
|
93
|
-
fetch(statistics, client, location, depth
|
|
96
|
+
fetch(statistics, client, location, depth&.-(1), fetched, &block).wait
|
|
94
97
|
return
|
|
95
98
|
else
|
|
96
99
|
Async.logger.debug(self) {"Ignoring redirect to #{location}."}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: benchmark-http
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.13.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Samuel Williams
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2020-
|
|
11
|
+
date: 2020-03-14 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: async-io
|
|
@@ -80,6 +80,34 @@ dependencies:
|
|
|
80
80
|
- - "~>"
|
|
81
81
|
- !ruby/object:Gem::Version
|
|
82
82
|
version: '2.0'
|
|
83
|
+
- !ruby/object:Gem::Dependency
|
|
84
|
+
name: covered
|
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
|
86
|
+
requirements:
|
|
87
|
+
- - ">="
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: '0'
|
|
90
|
+
type: :development
|
|
91
|
+
prerelease: false
|
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
93
|
+
requirements:
|
|
94
|
+
- - ">="
|
|
95
|
+
- !ruby/object:Gem::Version
|
|
96
|
+
version: '0'
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: async-rspec
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - ">="
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '0'
|
|
104
|
+
type: :development
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - ">="
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: '0'
|
|
83
111
|
- !ruby/object:Gem::Dependency
|
|
84
112
|
name: bundler
|
|
85
113
|
requirement: !ruby/object:Gem::Requirement
|