site_analyzer 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -1
- data/.rspec +1 -1
- data/Gemfile.lock +14 -10
- data/README.md +2 -2
- data/SiteAnalyzer.iml +5 -3
- data/lib/site_analyzer/open-uri-patching.rb +13 -0
- data/lib/site_analyzer/page.rb +12 -15
- data/lib/site_analyzer/report.rb +39 -35
- data/lib/site_analyzer/site.rb +63 -26
- data/lib/site_analyzer/version.rb +1 -1
- data/lib/site_analyzer.rb +5 -11
- data/site_analyzer-0.1.0.gem +0 -0
- data/site_analyzer.gemspec +8 -6
- metadata +43 -14
- data/.idea/uiDesigner.xml +0 -124
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1c3015952bf77ef4a5cd50162ebf6a19fe14f10d
|
4
|
+
data.tar.gz: 451008935f507a9dd8a3f9e755d53ae0c7df04b6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4d971e1d589205441cb576539d1b7b67e46fff2d24149c42cbcc3d3bc0fe99b96d9f7bdc662e397e9b6b873fb8469c9bbff8d6df1ae3b0535d22a7586f9a029c
|
7
|
+
data.tar.gz: 448e3e7a590170e578f88b046012ef035f65aded23ce157a99a32f3e91388534d27d291b193b8391708396c682a78f3e783a68365cf223ac2894ed2b54e741fa
|
data/.gitignore
CHANGED
data/.rspec
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,13 +1,14 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
site_analyzer (0.1.
|
4
|
+
site_analyzer (0.1.1)
|
5
5
|
|
6
6
|
GEM
|
7
7
|
remote: https://rubygems.org/
|
8
8
|
specs:
|
9
9
|
addressable (2.3.8)
|
10
10
|
diff-lcs (1.2.5)
|
11
|
+
log4r (1.1.10)
|
11
12
|
mini_portile (0.6.2)
|
12
13
|
nokogiri (1.6.6.2)
|
13
14
|
mini_portile (~> 0.6.0)
|
@@ -17,27 +18,30 @@ GEM
|
|
17
18
|
rspec-core (~> 3.3.0)
|
18
19
|
rspec-expectations (~> 3.3.0)
|
19
20
|
rspec-mocks (~> 3.3.0)
|
20
|
-
rspec-core (3.3.
|
21
|
+
rspec-core (3.3.2)
|
21
22
|
rspec-support (~> 3.3.0)
|
22
|
-
rspec-expectations (3.3.
|
23
|
+
rspec-expectations (3.3.1)
|
23
24
|
diff-lcs (>= 1.2.0, < 2.0)
|
24
25
|
rspec-support (~> 3.3.0)
|
25
|
-
rspec-mocks (3.3.
|
26
|
+
rspec-mocks (3.3.2)
|
26
27
|
diff-lcs (>= 1.2.0, < 2.0)
|
27
28
|
rspec-support (~> 3.3.0)
|
28
29
|
rspec-support (3.3.0)
|
30
|
+
terminal-table (1.5.2)
|
29
31
|
|
30
32
|
PLATFORMS
|
31
33
|
ruby
|
32
34
|
|
33
35
|
DEPENDENCIES
|
34
|
-
addressable (~> 2
|
35
|
-
bundler (~> 1
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
36
|
+
addressable (~> 2)
|
37
|
+
bundler (~> 1)
|
38
|
+
log4r (~> 1)
|
39
|
+
nokogiri (~> 1)
|
40
|
+
rake (~> 10)
|
41
|
+
robotstxt (~> 0)
|
42
|
+
rspec (~> 3)
|
40
43
|
site_analyzer!
|
44
|
+
terminal-table (~> 1)
|
41
45
|
|
42
46
|
BUNDLED WITH
|
43
47
|
1.10.5
|
data/README.md
CHANGED
@@ -22,7 +22,7 @@ You might need administrator privileges on your system to install it.
|
|
22
22
|
<b>How to use</b>
|
23
23
|
|
24
24
|
require 'site_analyzer'<br>
|
25
|
-
SiteAnalyzer.add_site 'http://savchuk.space' # add site to list<br>
|
25
|
+
SiteAnalyzer.add_site 'http://savchuk.space', 10, true # add site to list scan only 10 first pages with robot.txt <br>
|
26
26
|
SiteAnalyzer.start # start creating of reports<br>
|
27
27
|
SiteAnalyzer.show_all_reports # show all reports in console<br>
|
28
28
|
SiteAnalyzer.show_report 5 # show report number 5 starting from 1 in console<br>
|
@@ -38,4 +38,4 @@ Denis Savchuk <a href="mailto:denis@savchuk.space"><denis@savchuk.space></a>
|
|
38
38
|
|
39
39
|
<b>License</b>
|
40
40
|
|
41
|
-
Copyright © 2015 Denis Savchuk, SiteAnalyzer is released under the MIT license.
|
41
|
+
Copyright © 2015 Denis Savchuk, SiteAnalyzer is released under the MIT license.
|
data/SiteAnalyzer.iml
CHANGED
@@ -8,14 +8,16 @@
|
|
8
8
|
<orderEntry type="library" scope="PROVIDED" name="addressable (v2.3.8, rbenv: 2.2.2) [gem]" level="application" />
|
9
9
|
<orderEntry type="library" scope="PROVIDED" name="bundler (v1.10.5, rbenv: 2.2.2) [gem]" level="application" />
|
10
10
|
<orderEntry type="library" scope="PROVIDED" name="diff-lcs (v1.2.5, rbenv: 2.2.2) [gem]" level="application" />
|
11
|
+
<orderEntry type="library" scope="PROVIDED" name="log4r (v1.1.10, rbenv: 2.2.2) [gem]" level="application" />
|
11
12
|
<orderEntry type="library" scope="PROVIDED" name="mini_portile (v0.6.2, rbenv: 2.2.2) [gem]" level="application" />
|
12
13
|
<orderEntry type="library" scope="PROVIDED" name="nokogiri (v1.6.6.2, rbenv: 2.2.2) [gem]" level="application" />
|
13
14
|
<orderEntry type="library" scope="PROVIDED" name="rake (v10.4.2, rbenv: 2.2.2) [gem]" level="application" />
|
14
15
|
<orderEntry type="library" scope="PROVIDED" name="robotstxt (v0.5.4, rbenv: 2.2.2) [gem]" level="application" />
|
15
16
|
<orderEntry type="library" scope="PROVIDED" name="rspec (v3.3.0, rbenv: 2.2.2) [gem]" level="application" />
|
16
|
-
<orderEntry type="library" scope="PROVIDED" name="rspec-core (v3.3.
|
17
|
-
<orderEntry type="library" scope="PROVIDED" name="rspec-expectations (v3.3.
|
18
|
-
<orderEntry type="library" scope="PROVIDED" name="rspec-mocks (v3.3.
|
17
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec-core (v3.3.2, rbenv: 2.2.2) [gem]" level="application" />
|
18
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec-expectations (v3.3.1, rbenv: 2.2.2) [gem]" level="application" />
|
19
|
+
<orderEntry type="library" scope="PROVIDED" name="rspec-mocks (v3.3.2, rbenv: 2.2.2) [gem]" level="application" />
|
19
20
|
<orderEntry type="library" scope="PROVIDED" name="rspec-support (v3.3.0, rbenv: 2.2.2) [gem]" level="application" />
|
21
|
+
<orderEntry type="library" scope="PROVIDED" name="terminal-table (v1.5.2, rbenv: 2.2.2) [gem]" level="application" />
|
20
22
|
</component>
|
21
23
|
</module>
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module OpenURI
|
2
|
+
|
3
|
+
def OpenURI.redirectable?(uri1, uri2) # :nodoc:
|
4
|
+
# This test is intended to forbid a redirection from http://... to
|
5
|
+
# file:///etc/passwd, file:///dev/zero, etc. CVE-2011-1521
|
6
|
+
# https to http redirect is also forbidden intentionally.
|
7
|
+
# It avoids sending secure cookie or referer by non-secure HTTP protocol.
|
8
|
+
# (RFC 2109 4.3.1, RFC 2965 3.3, RFC 2616 15.1.3)
|
9
|
+
# However this is ad hoc. It should be extensible/configurable.
|
10
|
+
uri1.scheme.downcase == uri2.scheme.downcase ||
|
11
|
+
(/\A(?:http|ftp|https)\z/i =~ uri1.scheme && /\A(?:http|ftp|https)\z/i =~ uri2.scheme)
|
12
|
+
end
|
13
|
+
end
|
data/lib/site_analyzer/page.rb
CHANGED
@@ -2,10 +2,9 @@ module SiteAnalyzer
|
|
2
2
|
# Get site page and provide metods for analyse
|
3
3
|
require 'nokogiri'
|
4
4
|
require 'addressable/uri'
|
5
|
-
require 'open-uri'
|
6
5
|
require 'timeout'
|
7
6
|
class Page
|
8
|
-
attr_reader :page_url, :titles
|
7
|
+
attr_reader :page_url, :titles, :page
|
9
8
|
def initialize(url)
|
10
9
|
@page_url = url
|
11
10
|
@page = get_page url
|
@@ -13,30 +12,28 @@ module SiteAnalyzer
|
|
13
12
|
@titles = all_titles
|
14
13
|
end
|
15
14
|
|
15
|
+
def to_s
|
16
|
+
"Page url: #{@page_url} Site url: #{@site_url}"
|
17
|
+
end
|
18
|
+
|
16
19
|
def get_page(url)
|
17
|
-
return nil if url.nil?
|
18
20
|
timeout(10) { Nokogiri::HTML(open(url)) }
|
19
|
-
rescue
|
20
|
-
nil
|
21
21
|
end
|
22
22
|
|
23
23
|
def get_domain(url)
|
24
|
-
return nil if url.nil?
|
25
24
|
timeout(10) { Addressable::URI.parse(url).host }
|
26
25
|
rescue
|
27
|
-
|
26
|
+
'Error with parsing by Addressable'
|
28
27
|
end
|
29
28
|
|
30
29
|
def title_good?
|
31
|
-
|
32
|
-
title.text.size < 70
|
30
|
+
@page.css('title').size == 1 && @page.css('title').text.size < 70
|
33
31
|
end
|
34
32
|
# true if title and h1 have no dublicates
|
35
33
|
def title_and_h1_good?
|
36
|
-
out = @page.search('//title', '//h1').children
|
37
34
|
arr = []
|
38
|
-
|
39
|
-
|
35
|
+
@page.css('h1').each { |node| arr << node.text }
|
36
|
+
@page.css('title').size == 1 && arr.uniq.size == arr.size
|
40
37
|
end
|
41
38
|
# true if metadescription less then 200 symbols
|
42
39
|
def metadescription_good?
|
@@ -108,9 +105,9 @@ module SiteAnalyzer
|
|
108
105
|
def all_a_tags_href
|
109
106
|
tags = []
|
110
107
|
@page.css('a').each do |node|
|
111
|
-
tags << node['href']
|
108
|
+
tags << node['href']
|
112
109
|
end
|
113
|
-
tags
|
110
|
+
tags.compact
|
114
111
|
end
|
115
112
|
|
116
113
|
def wrong_a
|
@@ -134,7 +131,7 @@ module SiteAnalyzer
|
|
134
131
|
@page.css('a').each do |node|
|
135
132
|
tags << [node['href'], node['target'], node['rel']]
|
136
133
|
end
|
137
|
-
tags
|
134
|
+
tags.compact
|
138
135
|
end
|
139
136
|
|
140
137
|
def all_titles
|
data/lib/site_analyzer/report.rb
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
require 'terminal-table'
|
2
|
+
require 'uri'
|
3
|
+
|
1
4
|
module SiteAnalyzer
|
2
5
|
# Create report for site
|
3
6
|
class Report
|
@@ -28,76 +31,83 @@ module SiteAnalyzer
|
|
28
31
|
|
29
32
|
def to_s
|
30
33
|
return 'Report is empty' if @report.nil? || @report.empty?
|
31
|
-
|
34
|
+
header = Terminal::Table.new title: "Report for #{@site_url} with #{@max_pages} pages max_pages and robot check is #{@use_robot}"
|
35
|
+
puts header
|
32
36
|
@report.each_pair do |key, value|
|
33
|
-
|
34
|
-
|
35
|
-
|
37
|
+
rows = []
|
38
|
+
value.each do |r|
|
39
|
+
r = [r] if r.class == String
|
40
|
+
rows << r unless key == :a_tags_list
|
41
|
+
end
|
42
|
+
table = Terminal::Table.new title: key, rows: rows
|
43
|
+
puts table
|
36
44
|
end
|
37
45
|
end
|
38
46
|
|
39
47
|
def check_titles_text_less_than_70
|
40
48
|
result = []
|
41
|
-
@site.pages.
|
42
|
-
result <<
|
49
|
+
@site.pages.each do |page|
|
50
|
+
result << page.page_url unless page.title_good?
|
43
51
|
end
|
44
52
|
result
|
45
53
|
end
|
46
54
|
|
47
55
|
def check_title_and_h1_for_doubles
|
48
56
|
result = []
|
49
|
-
@site.pages.
|
50
|
-
result <<
|
57
|
+
@site.pages.each do |page|
|
58
|
+
result << page.page_url unless page.title_and_h1_good?
|
51
59
|
end
|
52
60
|
result
|
53
61
|
end
|
54
62
|
|
55
63
|
def check_meta_description_less_then_200
|
56
64
|
result = []
|
57
|
-
@site.pages.
|
58
|
-
result <<
|
65
|
+
@site.pages.each do |page|
|
66
|
+
result << page.page_url unless page.metadescription_good?
|
59
67
|
end
|
60
68
|
result
|
61
69
|
end
|
62
70
|
|
63
71
|
def check_meta_keywords_tags
|
64
72
|
result = []
|
65
|
-
@site.pages.
|
66
|
-
result <<
|
73
|
+
@site.pages.each do |page|
|
74
|
+
result << page.page_url unless page.keywords_good?
|
67
75
|
end
|
68
76
|
result
|
69
77
|
end
|
70
78
|
|
71
79
|
def check_h2
|
72
80
|
result = []
|
73
|
-
@site.pages.
|
74
|
-
result <<
|
81
|
+
@site.pages.each do |page|
|
82
|
+
result << page.page_url unless page.h2?
|
75
83
|
end
|
76
84
|
result
|
77
85
|
end
|
78
86
|
|
79
87
|
def pages_size
|
80
88
|
result = []
|
81
|
-
@site.pages.
|
82
|
-
result << [
|
89
|
+
@site.pages.each do |page|
|
90
|
+
result << [page.page_url , page.page_text_size]
|
83
91
|
end
|
84
92
|
result
|
85
93
|
end
|
86
94
|
|
87
95
|
def code_more
|
88
96
|
result = []
|
89
|
-
@site.pages.
|
90
|
-
result <<
|
97
|
+
@site.pages.each do |page|
|
98
|
+
result << page.page_url unless page.code_less?
|
91
99
|
end
|
92
100
|
result
|
93
101
|
end
|
94
102
|
|
95
103
|
def a_tag_array
|
104
|
+
@site.all_a
|
105
|
+
end
|
106
|
+
|
107
|
+
def bad_url
|
96
108
|
result = []
|
97
|
-
|
98
|
-
|
99
|
-
result << [home_page_of_url, link[0], link[1], link[2]]
|
100
|
-
end
|
109
|
+
a_tag_array.each do |url|
|
110
|
+
result << url unless URI(url[1][0]).path =~ /^[\/a-z0-9-]+$/
|
101
111
|
end
|
102
112
|
result
|
103
113
|
end
|
@@ -114,14 +124,6 @@ module SiteAnalyzer
|
|
114
124
|
find_doubles @site.all_descriptions
|
115
125
|
end
|
116
126
|
|
117
|
-
def bad_url
|
118
|
-
result = []
|
119
|
-
a_tag_array.each do |url|
|
120
|
-
result << url if url[1].include? '?meta='
|
121
|
-
end
|
122
|
-
result
|
123
|
-
end
|
124
|
-
|
125
127
|
def h2_doubles
|
126
128
|
find_doubles(@site.all_h2)
|
127
129
|
end
|
@@ -135,9 +137,11 @@ module SiteAnalyzer
|
|
135
137
|
all_words = []
|
136
138
|
counter = {}
|
137
139
|
result = []
|
138
|
-
in_array.each do |url_desc_cont|
|
139
|
-
url_desc_cont[
|
140
|
-
|
140
|
+
in_array.compact.each do |url_desc_cont|
|
141
|
+
if url_desc_cont[1][0]
|
142
|
+
url_desc_cont[1][0].scan(/\w+/).each do |word|
|
143
|
+
all_words << word
|
144
|
+
end
|
141
145
|
end
|
142
146
|
end
|
143
147
|
all_words.each do |word|
|
@@ -150,14 +154,14 @@ module SiteAnalyzer
|
|
150
154
|
counter.each_pair do |word, number|
|
151
155
|
result << word if number > 1
|
152
156
|
end
|
153
|
-
result
|
157
|
+
result.uniq
|
154
158
|
end
|
155
159
|
# in_array must be [[url_of_page, words_in_string_with_space],[next, same_element]]
|
156
160
|
def find_doubles(in_array)
|
157
161
|
result = []
|
158
162
|
find_not_uniq_words(in_array).each do |not_uniq_word|
|
159
163
|
in_array.each do |url_desc_cont|
|
160
|
-
result << url_desc_cont if url_desc_cont[0][1].include?
|
164
|
+
result << url_desc_cont if url_desc_cont[1][0] && url_desc_cont[1][0].include?(not_uniq_word)
|
161
165
|
end
|
162
166
|
end
|
163
167
|
result
|
data/lib/site_analyzer/site.rb
CHANGED
@@ -1,66 +1,103 @@
|
|
1
1
|
module SiteAnalyzer
|
2
|
+
# require 'log4r'
|
2
3
|
require 'robotstxt'
|
4
|
+
require 'open-uri'
|
5
|
+
require 'timeout'
|
3
6
|
# Create site object with all scans
|
4
7
|
class Site
|
5
|
-
|
8
|
+
# include Log4r
|
9
|
+
attr_reader :main_url, :pages, :domain, :pages_for_scan, :max_pages
|
6
10
|
def initialize(url, max_pages = 10, use_robot_txt = false)
|
7
11
|
@main_url = url
|
8
|
-
@pages =
|
12
|
+
@pages = []
|
9
13
|
@max_pages = max_pages
|
10
14
|
@domain = Addressable::URI.parse(url).host
|
11
15
|
@use_robot_txt = use_robot_txt
|
12
|
-
|
16
|
+
@scanned_pages = []
|
17
|
+
@pages_for_scan = []
|
18
|
+
add_page url
|
19
|
+
scan_site!
|
20
|
+
# logging!
|
13
21
|
end
|
14
22
|
|
23
|
+
# def logging!
|
24
|
+
# @log = Logger.new "Site:#{@main_url}"
|
25
|
+
# @log.outputters = FileOutputter.new('site', filename: "log/#{@domain}.log")
|
26
|
+
# @log.level = DEBUG
|
27
|
+
# end
|
28
|
+
|
15
29
|
def robot_txt_allowed?(url)
|
16
|
-
|
30
|
+
if @use_robot_txt
|
31
|
+
Robotstxt.allowed?(url, '*') rescue nil
|
32
|
+
else
|
33
|
+
true
|
34
|
+
end
|
17
35
|
end
|
18
36
|
|
19
|
-
def
|
20
|
-
|
21
|
-
@
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
urls_array.each do |url_in|
|
26
|
-
all_site_pages(url_in) unless @pages.key?(url_in) || url_in.nil?
|
27
|
-
end
|
37
|
+
def scan_site!
|
38
|
+
add_pages_for_scan!
|
39
|
+
while @pages_for_scan.size > 0
|
40
|
+
add_page @pages_for_scan.pop
|
41
|
+
return if @max_pages <= 0
|
42
|
+
add_pages_for_scan!
|
28
43
|
end
|
29
|
-
@pages[url] = page
|
30
44
|
end
|
31
45
|
|
32
|
-
def
|
33
|
-
|
46
|
+
def add_pages_for_scan!
|
47
|
+
@pages_for_scan = []
|
48
|
+
@pages.each do |page|
|
49
|
+
page.home_a.each do |link|
|
50
|
+
@pages_for_scan << link unless link.nil? || @scanned_pages.include?(link) || link.include?('mailto:') || link.end_with?('.jpg')
|
51
|
+
end
|
52
|
+
end
|
53
|
+
@pages_for_scan.clear if @pages_for_scan.size == 0
|
34
54
|
end
|
35
55
|
|
36
|
-
def
|
37
|
-
|
38
|
-
|
39
|
-
|
56
|
+
def add_page(url)
|
57
|
+
# @log.debug "#{Time.new} add page #{url} to site with option: allowed by robot? #{robot_txt_allowed?(url)}"
|
58
|
+
unless robot_txt_allowed?(url)
|
59
|
+
@scanned_pages << url
|
60
|
+
return nil
|
61
|
+
end
|
62
|
+
page = Page.new(url)
|
63
|
+
# @log.debug "#{Time.new} #{page.to_s}"
|
64
|
+
# @log.debug "#{Time.new} page created and not nil #{!page.nil?}"
|
65
|
+
@pages << page
|
66
|
+
@max_pages -= 1
|
67
|
+
@scanned_pages << url
|
40
68
|
end
|
41
69
|
|
42
70
|
def all_titles
|
43
71
|
result = []
|
44
|
-
@pages.
|
45
|
-
result << [
|
72
|
+
@pages.each do |page|
|
73
|
+
result << [page.page_url, page.titles]
|
46
74
|
end
|
47
75
|
result
|
48
76
|
end
|
49
77
|
|
50
78
|
def all_descriptions
|
51
79
|
result = []
|
52
|
-
@pages.
|
53
|
-
result << [
|
80
|
+
@pages.each do |page|
|
81
|
+
result << [page.page_url, page.all_meta_description_content]
|
54
82
|
end
|
55
83
|
result
|
56
84
|
end
|
57
85
|
|
58
86
|
def all_h2
|
59
87
|
result = []
|
60
|
-
@pages.
|
61
|
-
result << [
|
88
|
+
@pages.each do |page|
|
89
|
+
result << [page.page_url, page.h2]
|
62
90
|
end
|
63
91
|
result
|
64
92
|
end
|
93
|
+
|
94
|
+
def all_a
|
95
|
+
result = []
|
96
|
+
@pages.each do |page|
|
97
|
+
tags = page.all_a_tags.compact
|
98
|
+
result << [page.page_url, tags[0], tags[1], tags[2]]
|
99
|
+
end
|
100
|
+
result.compact
|
101
|
+
end
|
65
102
|
end
|
66
103
|
end
|
data/lib/site_analyzer.rb
CHANGED
@@ -1,18 +1,12 @@
|
|
1
1
|
# Main class for program
|
2
2
|
module SiteAnalyzer
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
require 'rubygems'
|
8
|
-
require 'nokogiri'
|
9
|
-
require 'addressable/uri'
|
10
|
-
require 'open-uri'
|
11
|
-
require 'timeout'
|
12
|
-
require 'robotstxt'
|
3
|
+
%w(page site report version open-uri-patching).each do |file|
|
4
|
+
require "site_analyzer/#{file}"
|
5
|
+
end
|
6
|
+
|
13
7
|
class << self
|
14
8
|
attr_reader :report
|
15
|
-
def add_site(site_url, max_pages =
|
9
|
+
def add_site(site_url, max_pages = 10, robottxt = false)
|
16
10
|
@report = []
|
17
11
|
@report << SiteAnalyzer::Report.new(site_url, max_pages, robottxt)
|
18
12
|
end
|
Binary file
|
data/site_analyzer.gemspec
CHANGED
@@ -18,10 +18,12 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
19
19
|
spec.require_paths = ['lib']
|
20
20
|
|
21
|
-
spec.add_development_dependency 'bundler', '~> 1
|
22
|
-
spec.add_development_dependency 'rake', '~> 10
|
23
|
-
spec.add_development_dependency 'rspec', '~>3
|
24
|
-
spec.add_development_dependency 'nokogiri', '~>1
|
25
|
-
spec.add_development_dependency 'addressable', '~>2
|
26
|
-
spec.add_development_dependency 'robotstxt', '~>0
|
21
|
+
spec.add_development_dependency 'bundler', '~> 1'
|
22
|
+
spec.add_development_dependency 'rake', '~> 10'
|
23
|
+
spec.add_development_dependency 'rspec', '~>3'
|
24
|
+
spec.add_development_dependency 'nokogiri', '~>1'
|
25
|
+
spec.add_development_dependency 'addressable', '~>2'
|
26
|
+
spec.add_development_dependency 'robotstxt', '~>0'
|
27
|
+
spec.add_development_dependency 'log4r', '~>1'
|
28
|
+
spec.add_development_dependency 'terminal-table', '~>1'
|
27
29
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: site_analyzer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Denis Savchuk
|
@@ -16,84 +16,112 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '1
|
19
|
+
version: '1'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '1
|
26
|
+
version: '1'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '10
|
33
|
+
version: '10'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '10
|
40
|
+
version: '10'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rspec
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '3
|
47
|
+
version: '3'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '3
|
54
|
+
version: '3'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: nokogiri
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '1
|
61
|
+
version: '1'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '1
|
68
|
+
version: '1'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: addressable
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '2
|
75
|
+
version: '2'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '2
|
82
|
+
version: '2'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: robotstxt
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '0
|
89
|
+
version: '0'
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: '0
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: log4r
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '1'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '1'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: terminal-table
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '1'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '1'
|
97
125
|
description: Create site report for SEO many options.
|
98
126
|
email:
|
99
127
|
- mordorreal@gmail.com
|
@@ -102,7 +130,6 @@ extensions: []
|
|
102
130
|
extra_rdoc_files: []
|
103
131
|
files:
|
104
132
|
- ".gitignore"
|
105
|
-
- ".idea/uiDesigner.xml"
|
106
133
|
- ".rspec"
|
107
134
|
- ".travis.yml"
|
108
135
|
- CODE_OF_CONDUCT.md
|
@@ -115,10 +142,12 @@ files:
|
|
115
142
|
- bin/console
|
116
143
|
- bin/setup
|
117
144
|
- lib/site_analyzer.rb
|
145
|
+
- lib/site_analyzer/open-uri-patching.rb
|
118
146
|
- lib/site_analyzer/page.rb
|
119
147
|
- lib/site_analyzer/report.rb
|
120
148
|
- lib/site_analyzer/site.rb
|
121
149
|
- lib/site_analyzer/version.rb
|
150
|
+
- site_analyzer-0.1.0.gem
|
122
151
|
- site_analyzer.gemspec
|
123
152
|
homepage: https://github.com/Mordorreal/SiteAnalyzer
|
124
153
|
licenses:
|
data/.idea/uiDesigner.xml
DELETED
@@ -1,124 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
-
<project version="4">
|
3
|
-
<component name="Palette2">
|
4
|
-
<group name="Swing">
|
5
|
-
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
6
|
-
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
|
7
|
-
</item>
|
8
|
-
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
9
|
-
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
|
10
|
-
</item>
|
11
|
-
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
12
|
-
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
|
13
|
-
</item>
|
14
|
-
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
|
15
|
-
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
|
16
|
-
</item>
|
17
|
-
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
18
|
-
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
|
19
|
-
<initial-values>
|
20
|
-
<property name="text" value="Button" />
|
21
|
-
</initial-values>
|
22
|
-
</item>
|
23
|
-
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
24
|
-
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
|
25
|
-
<initial-values>
|
26
|
-
<property name="text" value="RadioButton" />
|
27
|
-
</initial-values>
|
28
|
-
</item>
|
29
|
-
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
30
|
-
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
|
31
|
-
<initial-values>
|
32
|
-
<property name="text" value="CheckBox" />
|
33
|
-
</initial-values>
|
34
|
-
</item>
|
35
|
-
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
36
|
-
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
|
37
|
-
<initial-values>
|
38
|
-
<property name="text" value="Label" />
|
39
|
-
</initial-values>
|
40
|
-
</item>
|
41
|
-
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
42
|
-
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
43
|
-
<preferred-size width="150" height="-1" />
|
44
|
-
</default-constraints>
|
45
|
-
</item>
|
46
|
-
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
47
|
-
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
48
|
-
<preferred-size width="150" height="-1" />
|
49
|
-
</default-constraints>
|
50
|
-
</item>
|
51
|
-
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
52
|
-
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
53
|
-
<preferred-size width="150" height="-1" />
|
54
|
-
</default-constraints>
|
55
|
-
</item>
|
56
|
-
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
57
|
-
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
58
|
-
<preferred-size width="150" height="50" />
|
59
|
-
</default-constraints>
|
60
|
-
</item>
|
61
|
-
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
62
|
-
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
63
|
-
<preferred-size width="150" height="50" />
|
64
|
-
</default-constraints>
|
65
|
-
</item>
|
66
|
-
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
67
|
-
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
68
|
-
<preferred-size width="150" height="50" />
|
69
|
-
</default-constraints>
|
70
|
-
</item>
|
71
|
-
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
72
|
-
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
|
73
|
-
</item>
|
74
|
-
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
75
|
-
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
76
|
-
<preferred-size width="150" height="50" />
|
77
|
-
</default-constraints>
|
78
|
-
</item>
|
79
|
-
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
80
|
-
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
|
81
|
-
<preferred-size width="150" height="50" />
|
82
|
-
</default-constraints>
|
83
|
-
</item>
|
84
|
-
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
85
|
-
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
86
|
-
<preferred-size width="150" height="50" />
|
87
|
-
</default-constraints>
|
88
|
-
</item>
|
89
|
-
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
90
|
-
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
|
91
|
-
<preferred-size width="200" height="200" />
|
92
|
-
</default-constraints>
|
93
|
-
</item>
|
94
|
-
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
95
|
-
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
|
96
|
-
<preferred-size width="200" height="200" />
|
97
|
-
</default-constraints>
|
98
|
-
</item>
|
99
|
-
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
100
|
-
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
|
101
|
-
</item>
|
102
|
-
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
103
|
-
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
|
104
|
-
</item>
|
105
|
-
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
106
|
-
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
|
107
|
-
</item>
|
108
|
-
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
109
|
-
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
|
110
|
-
</item>
|
111
|
-
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
112
|
-
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
|
113
|
-
<preferred-size width="-1" height="20" />
|
114
|
-
</default-constraints>
|
115
|
-
</item>
|
116
|
-
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
117
|
-
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
|
118
|
-
</item>
|
119
|
-
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
120
|
-
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
|
121
|
-
</item>
|
122
|
-
</group>
|
123
|
-
</component>
|
124
|
-
</project>
|