robots 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +2 -0
- data/VERSION +1 -1
- data/lib/robots.rb +2 -2
- data/robots.gemspec +2 -2
- data/test/test_robots.rb +2 -2
- metadata +3 -3
data/CHANGELOG
CHANGED
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.
|
|
1
|
+
0.10.0
|
data/lib/robots.rb
CHANGED
|
@@ -12,7 +12,7 @@ class Robots
|
|
|
12
12
|
def initialize(uri, user_agent)
|
|
13
13
|
@last_accessed = Time.at(1)
|
|
14
14
|
|
|
15
|
-
io = Robots.get_robots_txt(uri)
|
|
15
|
+
io = Robots.get_robots_txt(uri, user_agent)
|
|
16
16
|
|
|
17
17
|
if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
|
|
18
18
|
io = StringIO.new("User-agent: *\nAllow: /\n")
|
|
@@ -98,7 +98,7 @@ class Robots
|
|
|
98
98
|
end
|
|
99
99
|
end
|
|
100
100
|
|
|
101
|
-
def self.get_robots_txt(uri)
|
|
101
|
+
def self.get_robots_txt(uri, user_agent)
|
|
102
102
|
begin
|
|
103
103
|
Timeout::timeout(Robots.timeout) do
|
|
104
104
|
io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
|
data/robots.gemspec
CHANGED
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
|
|
6
6
|
Gem::Specification.new do |s|
|
|
7
7
|
s.name = %q{robots}
|
|
8
|
-
s.version = "0.
|
|
8
|
+
s.version = "0.10.0"
|
|
9
9
|
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
11
11
|
s.authors = ["Kyle Maxwell"]
|
|
12
|
-
s.date = %q{2010-
|
|
12
|
+
s.date = %q{2010-06-23}
|
|
13
13
|
s.description = %q{It parses robots.txt files}
|
|
14
14
|
s.email = %q{kyle@kylemaxwell.com}
|
|
15
15
|
s.extra_rdoc_files = [
|
data/test/test_robots.rb
CHANGED
|
@@ -14,7 +14,7 @@ end
|
|
|
14
14
|
|
|
15
15
|
class TestRobots < Test::Unit::TestCase
|
|
16
16
|
def setup
|
|
17
|
-
def Robots.get_robots_txt(uri)
|
|
17
|
+
def Robots.get_robots_txt(uri, user_agent)
|
|
18
18
|
fixture_file = File.dirname(__FILE__) + "/fixtures/" + uri.host.split(".")[-2] + ".txt"
|
|
19
19
|
File.open(fixture_file).extend(FakeHttp)
|
|
20
20
|
end
|
|
@@ -23,7 +23,7 @@ class TestRobots < Test::Unit::TestCase
|
|
|
23
23
|
end
|
|
24
24
|
|
|
25
25
|
def test_allowed_if_no_robots
|
|
26
|
-
def Robots.get_robots_txt(uri)
|
|
26
|
+
def Robots.get_robots_txt(uri, user_agent)
|
|
27
27
|
return nil
|
|
28
28
|
end
|
|
29
29
|
|
metadata
CHANGED
|
@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
|
|
|
4
4
|
prerelease: false
|
|
5
5
|
segments:
|
|
6
6
|
- 0
|
|
7
|
-
-
|
|
7
|
+
- 10
|
|
8
8
|
- 0
|
|
9
|
-
version: 0.
|
|
9
|
+
version: 0.10.0
|
|
10
10
|
platform: ruby
|
|
11
11
|
authors:
|
|
12
12
|
- Kyle Maxwell
|
|
@@ -14,7 +14,7 @@ autorequire:
|
|
|
14
14
|
bindir: bin
|
|
15
15
|
cert_chain: []
|
|
16
16
|
|
|
17
|
-
date: 2010-
|
|
17
|
+
date: 2010-06-23 00:00:00 -07:00
|
|
18
18
|
default_executable:
|
|
19
19
|
dependencies:
|
|
20
20
|
- !ruby/object:Gem::Dependency
|