shelob 0.1.0.beta4 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +6 -0
- data/README.md +4 -0
- data/lib/link_result.rb +6 -2
- data/lib/resolver.rb +9 -3
- data/lib/shelob.rb +1 -1
- data/lib/shelob/version.rb +1 -1
- data/shelob.gemspec +10 -6
- data/test/test_link_result.rb +12 -2
- data/test/test_resolver.rb +6 -0
- metadata +33 -32
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 44d5f76a10c681da571b277d16cf64bb95f33bff
|
4
|
+
data.tar.gz: 9a0945b0e552a1733e4129a71954ce1e29d85047
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 368ac2e4de43a2b6898a95402a6b1113fa0384b3b2be5e245c28cf5903f2c5aa8cff189f3a5fe7ecf71db09444149aeac270d7e296fa485127f0668b39b6e54c
|
7
|
+
data.tar.gz: bbf3812392cccc5d8ab1921ca5129f3fa973c87ddd24f3245b8ab1ac1936599b59eea9f564cfaaedd0bee73fb98643e398f51e162b9e47f5cc0d4f36f9dc6b20
|
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# Shelob
|
2
2
|
|
3
|
+
[![Build Status](https://travis-ci.org/bmnick/shelob.png)](https://travis-ci.org/bmnick/shelob)
|
4
|
+
|
3
5
|
Shelob is a giant spider that starts on a given page, finds all links on the page, ensure they resolve, and recurses if the link is underneath the starting url. Intended primarily for double checking that your site has no horrible error pages to be exposed to the user by clicking on a link.
|
4
6
|
|
5
7
|
## Usage
|
@@ -13,6 +15,8 @@ You can also use the link resolver, extractor, or the spider itself programmatic
|
|
13
15
|
|
14
16
|
## Installation
|
15
17
|
|
18
|
+
Shelob requires Ruby 1.9.3 or newer. Hopefully this isn't a problem, but if so, check out RVM!
|
19
|
+
|
16
20
|
Add this line to your application's Gemfile:
|
17
21
|
|
18
22
|
gem 'shelob'
|
data/lib/link_result.rb
CHANGED
@@ -7,11 +7,15 @@ class LinkResult
|
|
7
7
|
@body = body
|
8
8
|
end
|
9
9
|
|
10
|
+
def pretty_status
|
11
|
+
@status == 900 ? 'Exception' : @status
|
12
|
+
end
|
13
|
+
|
10
14
|
def to_s
|
11
|
-
"#{
|
15
|
+
"#{pretty_status}: #{@url}"
|
12
16
|
end
|
13
17
|
|
14
|
-
def failed
|
18
|
+
def failed?
|
15
19
|
@status.to_i >= 400
|
16
20
|
end
|
17
21
|
end
|
data/lib/resolver.rb
CHANGED
@@ -4,13 +4,19 @@ require 'net/http'
|
|
4
4
|
module Shelob
|
5
5
|
class Resolver
|
6
6
|
def initialize url
|
7
|
-
@
|
7
|
+
@url = url
|
8
8
|
end
|
9
9
|
|
10
10
|
def resolve
|
11
|
-
|
11
|
+
begin
|
12
|
+
uri = URI(@url)
|
12
13
|
|
13
|
-
|
14
|
+
resp = Net::HTTP.get_response(uri)
|
15
|
+
|
16
|
+
LinkResult.new uri.to_s, resp.code.to_i, resp.body
|
17
|
+
rescue Exception => e
|
18
|
+
LinkResult.new @url, 900, e.message
|
19
|
+
end
|
14
20
|
end
|
15
21
|
end
|
16
22
|
end
|
data/lib/shelob.rb
CHANGED
data/lib/shelob/version.rb
CHANGED
data/shelob.gemspec
CHANGED
@@ -18,12 +18,16 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
19
|
spec.require_paths = ["lib"]
|
20
20
|
|
21
|
+
# Automation
|
21
22
|
spec.add_development_dependency "bundler", "~> 1.3"
|
22
|
-
spec.add_development_dependency "rake"
|
23
|
-
spec.add_development_dependency "
|
24
|
-
spec.add_development_dependency "
|
25
|
-
spec.add_development_dependency "guard"
|
26
|
-
spec.add_development_dependency "guard-minitest"
|
23
|
+
spec.add_development_dependency "rake", "~> 10.1"
|
24
|
+
spec.add_development_dependency "guard", "~> 2.2"
|
25
|
+
spec.add_development_dependency "guard-minitest", "~> 2.1"
|
27
26
|
|
28
|
-
|
27
|
+
# Testing tools
|
28
|
+
spec.add_development_dependency "minitest", "~> 5.2"
|
29
|
+
spec.add_development_dependency "webmock", "~> 1.16"
|
30
|
+
|
31
|
+
# Runtime dependencies
|
32
|
+
spec.add_runtime_dependency "nokogiri", "~> 1.6"
|
29
33
|
end
|
data/test/test_link_result.rb
CHANGED
@@ -27,9 +27,19 @@ describe LinkResult, "Link fetch result" do
|
|
27
27
|
end
|
28
28
|
|
29
29
|
it "should determine if a request is failed" do
|
30
|
-
@result.failed
|
31
|
-
@failed.failed
|
30
|
+
@result.failed?.must_equal false
|
31
|
+
@failed.failed?.must_equal true
|
32
32
|
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
describe "exception cases" do
|
37
|
+
|
38
|
+
it "should be aware of the 900 = Exception rule in to_s" do
|
39
|
+
res = LinkResult.new("http://google.com/bad uri", 900, 'undefined local variable or method `url` for blah')
|
40
|
+
res.to_s.must_equal 'Exception: http://google.com/bad uri'
|
41
|
+
end
|
42
|
+
|
33
43
|
end
|
34
44
|
end
|
35
45
|
|
data/test/test_resolver.rb
CHANGED
@@ -10,6 +10,12 @@ describe Shelob::Resolver, "Link fetching module" do
|
|
10
10
|
it "should be created with a url" do
|
11
11
|
Shelob::Resolver.new("http://bmnick.com/ruby-c-extensions")
|
12
12
|
end
|
13
|
+
|
14
|
+
it "should return an error result for bad URIs" do
|
15
|
+
resolver = Shelob::Resolver.new("http://bmnick.com/bad uri")
|
16
|
+
res = resolver.resolve
|
17
|
+
res.failed?
|
18
|
+
end
|
13
19
|
end
|
14
20
|
|
15
21
|
describe "when used" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shelob
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Benjamin Nicholas
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2014-01-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -28,86 +28,86 @@ dependencies:
|
|
28
28
|
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ~>
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '10.1'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ~>
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '10.1'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: guard
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - ~>
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '2.2'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ~>
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '2.2'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: guard-minitest
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - ~>
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '2.1'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - ~>
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
68
|
+
version: '2.1'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: minitest
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ~>
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
75
|
+
version: '5.2'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - ~>
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '
|
82
|
+
version: '5.2'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
84
|
+
name: webmock
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- -
|
87
|
+
- - ~>
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
89
|
+
version: '1.16'
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- -
|
94
|
+
- - ~>
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
96
|
+
version: '1.16'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: nokogiri
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- -
|
101
|
+
- - ~>
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: '
|
103
|
+
version: '1.6'
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- -
|
108
|
+
- - ~>
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version: '
|
110
|
+
version: '1.6'
|
111
111
|
description: A giant spider that starts on a given page, finds all links on the page,
|
112
112
|
ensure they resolve, and recurses if the link is underneath the starting url
|
113
113
|
email:
|
@@ -118,6 +118,7 @@ extensions: []
|
|
118
118
|
extra_rdoc_files: []
|
119
119
|
files:
|
120
120
|
- .gitignore
|
121
|
+
- .travis.yml
|
121
122
|
- Gemfile
|
122
123
|
- Guardfile
|
123
124
|
- LICENSE.txt
|
@@ -149,9 +150,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
149
150
|
version: '0'
|
150
151
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
151
152
|
requirements:
|
152
|
-
- - '
|
153
|
+
- - '>='
|
153
154
|
- !ruby/object:Gem::Version
|
154
|
-
version:
|
155
|
+
version: '0'
|
155
156
|
requirements: []
|
156
157
|
rubyforge_project:
|
157
158
|
rubygems_version: 2.0.3
|