colinramsay-robotstxt 1.0.1 → 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/robotstxtparser.rb +6 -6
- metadata +1 -1
data/lib/robotstxtparser.rb
CHANGED
@@ -5,13 +5,13 @@ class RobotsTxtParser
|
|
5
5
|
attr_reader :user_agents
|
6
6
|
|
7
7
|
def initialize(path)
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
rescue
|
8
|
+
begin
|
9
|
+
if path.include?("://")
|
10
|
+
raw_data = open(path)
|
11
|
+
else
|
12
|
+
raw_data = File.open(path)
|
14
13
|
end
|
14
|
+
rescue
|
15
15
|
end
|
16
16
|
|
17
17
|
@user_agents = Hash.new
|