driller 0.0.8 → 0.0.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/lib/driller/version.rb +1 -1
- data/lib/driller.rb +1 -38
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
---
|
|
2
2
|
!binary "U0hBMQ==":
|
|
3
3
|
metadata.gz: !binary |-
|
|
4
|
-
|
|
4
|
+
MmNjNmVkYjZhODQxMjI3ZTAyOWMxYzk0YzVkYmVkNzU0YTJmZjI0OQ==
|
|
5
5
|
data.tar.gz: !binary |-
|
|
6
|
-
|
|
6
|
+
OTE4YWE5ZThmMDgwNWFlNGNiZWZkZGQ0MmU1NDRiYjNmMGUwNDQ2Ng==
|
|
7
7
|
SHA512:
|
|
8
8
|
metadata.gz: !binary |-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
ZTgzNDVjODQ4ZTI0ZmE3YmEzNjM0Y2E1MjcyMmVjZTI5MWM3MDg1ZjZlMGY0
|
|
10
|
+
MjQzZWMwZmRlZmQzMmRjZTk5OTRkODEwNGI3YzU4NTIyN2JlYjlkOTM0ODFi
|
|
11
|
+
MzU4MzBiMTczNzE1NDY1ZTliZTIxNGQyZGIwOGM4Y2I1NjFhMDQ=
|
|
12
12
|
data.tar.gz: !binary |-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
ODIyM2M1MDI3Nzk0N2NhMTdhNjllMmU2ZGUyNDNiOWJhNWNmYjg0MjU4ZmI5
|
|
14
|
+
MmNjM2E3YjA0MDdlYWJlYTVkNmIyNWM2NjM3ZjhjMDNjNzJiNzFiOTdiNTE2
|
|
15
|
+
MDI0NTdhOGNhNWM0ZjcxMzIyYmFjY2Q3NDZiN2JjYmRkMDE5M2U=
|
data/lib/driller/version.rb
CHANGED
data/lib/driller.rb
CHANGED
|
@@ -4,42 +4,5 @@ require "driller/crawler"
|
|
|
4
4
|
require "anemone"
|
|
5
5
|
|
|
6
6
|
module Driller
|
|
7
|
-
|
|
8
|
-
Anemone.crawl(URL) do |anemone|
|
|
9
|
-
|
|
10
|
-
anemone.depth_limit = 2
|
|
11
|
-
|
|
12
|
-
anemone.focus_crawl do |page|
|
|
13
|
-
page.links.select { |url| url.starts_with? URL }
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
file = File.new('valid_pages.html', 'w')
|
|
17
|
-
file = File.new('broken.html', 'w')
|
|
18
|
-
file = File.new('slow_pages.html', 'w')
|
|
19
|
-
|
|
20
|
-
anemone.on_every_page do |page|
|
|
21
|
-
|
|
22
|
-
if page.code = 200 && page.code = 301
|
|
23
|
-
file = File.open('valid_pages.html', 'a')
|
|
24
|
-
file.puts page.url
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
if page.code != 200 && page.code != 301
|
|
28
|
-
puts "=======broken======\n"
|
|
29
|
-
puts page.url
|
|
30
|
-
file = File.open('broken.html', 'a')
|
|
31
|
-
file.puts page.url
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
if page.response_time > 5000
|
|
36
|
-
puts "=======Slow Page======\n"
|
|
37
|
-
puts page.url
|
|
38
|
-
file = File.open('slow_pages.html', 'a')
|
|
39
|
-
file.puts page.url
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
end
|
|
7
|
+
|
|
45
8
|
end
|