fizx-robots 0.7.1 → 0.7.2
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/robots.rb +3 -3
- metadata +3 -2
data/lib/robots.rb
CHANGED
|
@@ -38,12 +38,12 @@ class Robots
|
|
|
38
38
|
value = arr.join(":").strip
|
|
39
39
|
value.strip!
|
|
40
40
|
case key
|
|
41
|
-
when "User-agent"
|
|
41
|
+
when "User-agent"
|
|
42
42
|
agent = to_regex(value)
|
|
43
|
-
when "Allow"
|
|
43
|
+
when "Allow"
|
|
44
44
|
@allows[agent] ||= []
|
|
45
45
|
@allows[agent] << to_regex(value)
|
|
46
|
-
when "Disallow"
|
|
46
|
+
when "Disallow"
|
|
47
47
|
@disallows[agent] ||= []
|
|
48
48
|
@disallows[agent] << to_regex(value)
|
|
49
49
|
when "Crawl-delay"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fizx-robots
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.7.
|
|
4
|
+
version: 0.7.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Kyle Maxwell
|
|
@@ -36,6 +36,7 @@ files:
|
|
|
36
36
|
- lib/robots.rb
|
|
37
37
|
has_rdoc: true
|
|
38
38
|
homepage: http://github.com/fizx/robots
|
|
39
|
+
licenses:
|
|
39
40
|
post_install_message:
|
|
40
41
|
rdoc_options:
|
|
41
42
|
- --main
|
|
@@ -57,7 +58,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
57
58
|
requirements: []
|
|
58
59
|
|
|
59
60
|
rubyforge_project:
|
|
60
|
-
rubygems_version: 1.
|
|
61
|
+
rubygems_version: 1.3.5
|
|
61
62
|
signing_key:
|
|
62
63
|
specification_version: 2
|
|
63
64
|
summary: Simple robots.txt parser
|