robots 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/CHANGELOG CHANGED
@@ -1,3 +1,5 @@
1
+ 0.10.0
2
+ - Make sure the fetch robots.txt operation happens with a user agent (via rb2k)
1
3
  0.9.0
2
4
  - Fix http://github.com/fizx/robots/issues#issue/1
3
5
  - Tests don't rely on network.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.9.0
1
+ 0.10.0
@@ -12,7 +12,7 @@ class Robots
12
12
  def initialize(uri, user_agent)
13
13
  @last_accessed = Time.at(1)
14
14
 
15
- io = Robots.get_robots_txt(uri)
15
+ io = Robots.get_robots_txt(uri, user_agent)
16
16
 
17
17
  if !io || io.content_type != "text/plain" || io.status != ["200", "OK"]
18
18
  io = StringIO.new("User-agent: *\nAllow: /\n")
@@ -98,7 +98,7 @@ class Robots
98
98
  end
99
99
  end
100
100
 
101
- def self.get_robots_txt(uri)
101
+ def self.get_robots_txt(uri, user_agent)
102
102
  begin
103
103
  Timeout::timeout(Robots.timeout) do
104
104
  io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = %q{robots}
8
- s.version = "0.9.0"
8
+ s.version = "0.10.0"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Kyle Maxwell"]
12
- s.date = %q{2010-05-29}
12
+ s.date = %q{2010-06-23}
13
13
  s.description = %q{It parses robots.txt files}
14
14
  s.email = %q{kyle@kylemaxwell.com}
15
15
  s.extra_rdoc_files = [
@@ -14,7 +14,7 @@ end
14
14
 
15
15
  class TestRobots < Test::Unit::TestCase
16
16
  def setup
17
- def Robots.get_robots_txt(uri)
17
+ def Robots.get_robots_txt(uri, user_agent)
18
18
  fixture_file = File.dirname(__FILE__) + "/fixtures/" + uri.host.split(".")[-2] + ".txt"
19
19
  File.open(fixture_file).extend(FakeHttp)
20
20
  end
@@ -23,7 +23,7 @@ class TestRobots < Test::Unit::TestCase
23
23
  end
24
24
 
25
25
  def test_allowed_if_no_robots
26
- def Robots.get_robots_txt(uri)
26
+ def Robots.get_robots_txt(uri, user_agent)
27
27
  return nil
28
28
  end
29
29
 
metadata CHANGED
@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
4
4
  prerelease: false
5
5
  segments:
6
6
  - 0
7
- - 9
7
+ - 10
8
8
  - 0
9
- version: 0.9.0
9
+ version: 0.10.0
10
10
  platform: ruby
11
11
  authors:
12
12
  - Kyle Maxwell
@@ -14,7 +14,7 @@ autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
16
 
17
- date: 2010-05-29 00:00:00 -07:00
17
+ date: 2010-06-23 00:00:00 -07:00
18
18
  default_executable:
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency