sqt 1.2.1 → 1.2.2
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/sqt.rb +28 -11
- metadata +1 -1
data/lib/sqt.rb
CHANGED
@@ -10,6 +10,7 @@ module SQT
|
|
10
10
|
require 'nokogiri'
|
11
11
|
|
12
12
|
require 'colorize'
|
13
|
+
require 'uri'
|
13
14
|
|
14
15
|
class Sqt
|
15
16
|
|
@@ -37,25 +38,41 @@ module SQT
|
|
37
38
|
|
38
39
|
# Curl with depth
|
39
40
|
def self.sarbotteCurlWithDepth(url, depth, result)
|
41
|
+
urlUrl = URI(url)
|
40
42
|
http = Curl.get(url)
|
41
43
|
file = http.body_str
|
42
44
|
xmlFile = Nokogiri::HTML(file)
|
45
|
+
allowedScheme = ['http', 'https']
|
46
|
+
if !(result.any? {|sqr| URI(sqr[:uri]).path.gsub(/\/$/, '') == urlUrl.path.gsub(/\/$/, '') })
|
47
|
+
result.push Sqt.buildResult(url, file)
|
48
|
+
end
|
43
49
|
if depth > 0
|
44
50
|
xmlFile.search('//a').each do |foundLink|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
51
|
+
begin
|
52
|
+
if foundLink['href'] =~ /^\//
|
53
|
+
toVisit = url.gsub(/\/$/, '') + foundLink['href']
|
54
|
+
else
|
55
|
+
toVisit = foundLink['href']
|
56
|
+
end
|
57
|
+
puts toVisit
|
58
|
+
if !(toVisit =~ /^\s*$/)
|
59
|
+
toVisitUrl = URI(toVisit)
|
60
|
+
result.any? do |sqr|
|
61
|
+
URI(sqr[:uri]).path.gsub(/\/$/, '') == toVisitUrl.path.gsub(/\/$/, '')
|
62
|
+
end
|
63
|
+
if urlUrl.host == toVisitUrl.host && allowedScheme.include?(toVisitUrl.scheme) && !result.any? { |sqr| URI(sqr[:uri]).path.gsub(/\/$/, '') == toVisitUrl.path.gsub(/\/$/, '') }
|
64
|
+
d = sarbotteCurlWithDepth(toVisit, depth - 1, result)
|
65
|
+
result = d if !d.nil?
|
66
|
+
end
|
67
|
+
end
|
68
|
+
rescue
|
69
|
+
puts "exception"
|
53
70
|
end
|
54
71
|
end
|
55
72
|
end
|
56
|
-
|
57
|
-
|
58
|
-
|
73
|
+
|
74
|
+
result
|
75
|
+
|
59
76
|
end
|
60
77
|
|
61
78
|
end
|