site_analyzer 0.3.14 → 0.3.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 358a561c7c98802ee66fc102a2b63987e7c973df
-  data.tar.gz: f5863c85a3e4656490e9bb1a5a76321cc6dfc1f6
+  metadata.gz: 3fea66e1d0fa5a9b34e70289b4e475ff53a2afa2
+  data.tar.gz: 66f0019cbee985a9b5568c5f7dee4c9a1aabd984
 SHA512:
-  metadata.gz: 424a067b83a5f78cd7080e9126cb31be20495dd86fac0c4a1e6eb7543eb11bc8138303a00766253c261ba7d97ec085b579ab6ec0dcb07e8508f7a676c189b4d8
-  data.tar.gz: 0a8667179dc3f554b209f75c55764671afe646dd9008f2201f9c5ef644891fc3c571b824ea9f2fda2d1191d0f4e2197d42981080c3d067edd8f519852b69b257
+  metadata.gz: 04e6f0c4e25ffc460bd92fa40a2154129a503247c185eb87b7d24c02e5ff0400712983ddc296945f3f3ba0b907a526270417c9595658b1ea0f1a90ad4aad7dda
+  data.tar.gz: 403386d6cafffcbbb1034c133337ecfc27dea00c482a61b11f5f6aefb6bcacf18a6edcdcedbceb82411e1afaf8d1eeeeeed05f2adc9faf09e2c0b31783d205e3

data/README.md

@@ -26,7 +26,7 @@ require 'site_analyzer'<br>
                                           
 SiteAnalyzer::Report.create site: 'http://savchuk.space', pages: 10, robot: false, console: true<br>
 Return hash with report.<br>
-arguments: site - url mast start from http or https, pages - number of pages to scan, robot - use or not robot.txt file, console - output to console
+arguments: site - url must start from http or https, pages - number of pages to scan, robot - use or not robot.txt file, console - output to console
 
 <br>
 <b>Author</b>

data/SiteAnalyzer.iml

@@ -6,15 +6,22 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
     <orderEntry type="library" scope="PROVIDED" name="addressable (v2.3.8, rbenv: 2.2.2) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="bundler (v1.10.5, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="bundler (v1.10.6, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="coderay (v1.1.0, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="diff-lcs (v1.2.5, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="ffi (v1.9.10, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="formatador (v0.2.5, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="guard (v2.13.0, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="guard-compat (v1.2.1, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="listen (v3.0.3, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="lumberjack (v1.0.9, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="mini_portile (v0.6.2, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="nenv (v0.2.0, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="nokogiri (v1.6.6.2, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="notiffany (v0.0.7, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="pry (v0.10.1, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rake (v10.4.2, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rb-fsevent (v0.9.5, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rb-inotify (v0.9.5, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="robotstxt (v0.5.4, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rspec (v3.3.0, rbenv: 2.2.2) [gem]" level="application" />

data/lib/site_analyzer/page.rb

@@ -1,30 +1,76 @@
 module SiteAnalyzer
-  # Get site page and provide metods for analyse
+  # Get site page and provide data for future analyse
   require 'nokogiri'
   require 'addressable/uri'
   require 'timeout'
   require 'stringex_lite'
   require 'open-uri'
   class Page
-    attr_reader :page_url, :titles, :page, :page_path, :site_domain
+    attr_reader :page_url, :page_path, :site_domain,
+                :all_titles, :title_good, :title_and_h1_good,
+                :meta_description_good, :meta_keywords, :code_less,
+                :meta_data, :meta_title_duplicates, :title_h1_h2,
+                :have_h2, :page_text_size, :page_a_tags,
+                :meta_desc_content, :h2_text, :hlu
+    # create page object, fill date and clear don't needed elements
     def initialize(url)
       @page_url = url
-      @page = []
-      @site_domain = ''
-      @page_path = ''
-      @titles = []
       get_page(url)
-      fill_data_field!
+      fill_data_field! if @page
+      clear!
     end
-
-    def fill_data_field!
-      @titles = all_titles
-    end
-
+    # to_s for report
     def to_s
       "Page url: #{@page_url} Site url: #{@site_domain}"
     end
+    # get all home (that on this site) url on page
+    def home_a
+      if @page_a_tags
+        home_a = []
+        @page_a_tags.uniq.each do |link|
+          uri = URI(link[0].to_ascii) rescue nil #TODO: write additional logic for link to image
+          if uri && @site_domain
+            home_a << link[0] if uri.host == @site_domain
+          end
+        end
+        home_a
+      end
+    end
+    # get all remote link on page
+    def remote_a
+      if @page_a_tags
+        remote_a = []
+        @page_a_tags.uniq.each do |link|
+          uri = URI(link[0].to_ascii)
+          if uri && @site_domain
+            remote_a << link[0] unless uri.host == @site_domain
+          end
+        end
+        remote_a
+      end
+    end
+
+    private
 
+    # fill Page instant with data for report
+    def fill_data_field!
+      @all_titles = titles
+      @meta_data = collect_metadates
+      @title_h1_h2 = all_titles_h1_h2
+      @page_text_size = text_size
+      @page_a_tags = all_a_tags
+      @meta_desc_content = all_meta_description_content
+      @h2_text = h2
+      @hlu = bad_url
+      @title_good = title_good?
+      @title_and_h1_good = title_and_h1_good?
+      @meta_description_good = metadescription_good?
+      @meta_keywords = keywords_good?
+      @code_less = code_less?
+      @meta_title_duplicates = metadates_good?
+      @have_h2 = h2?
+    end
+    # get page with open-uri, then parse it with Nokogiri. Get site domain and path from URI
     def get_page(url)
       begin
         timeout(30) do
@@ -37,11 +83,11 @@ module SiteAnalyzer
         return nil
       end
     end
-
+    # check that title is one and less then 70 symbols
     def title_good?
       @page.css('title').size == 1 && @page.css('title').text.size < 70 if @page
     end
-    # true if title and h1 have no dublicates
+    # true if title and h1 have no duplicates
     def title_and_h1_good?
       if @page
         arr = []
@@ -75,7 +121,7 @@ module SiteAnalyzer
         true
       end
     end
-    # true if code less then text
+    # true if code of page less then text on it
     def code_less?
       if @page
         sum = 0
@@ -86,19 +132,20 @@ module SiteAnalyzer
         sum < page_text / 2
       end
     end
-
+    # collect meta tags for future report
     def collect_metadates
-      @page.css('meta') if @page
+      meta = []
+      meta = @page.css('meta') if @page
+      meta
     end
-
+    # check meta and title tags duplicates
     def metadates_good?
       if @page
-        meta_tags = collect_metadates
-        return false if @page.css('title').size > 1 || meta_tags.nil?
+        return false if @all_titles.size > 1 || @meta_data.empty?
         node_names = []
-        meta_tags.each { |node| node_names << node['name'] }
-        return false if node_names.compact!.size < 1
-        node_names.uniq.size == node_names.size
+        @meta_data.each { |node| node_names << node['name'] }
+        node_names.compact!
+        node_names.uniq.size == node_names.size unless node_names.nil? || node_names.size < 1
       end
     end
     # return hash with all titles, h1 and h2
@@ -107,53 +154,18 @@ module SiteAnalyzer
         out = []
         out << @page.css('title').text << { @page_url => @page.css('h1').text }
         out << { @page_url => @page.css('h2').text }
+        out
       end
     end
-
-    def home_a
-      if @page
-        home_a = []
-        all_a_tags_href.uniq.each do |link|
-          uri = URI(link.to_ascii) rescue nil #TODO: write additional logic for link to image
-          if uri && @site_domain
-            home_a << link if uri.host == @site_domain
-          end
-        end
-        home_a
-      end
-    end
-
-    def remote_a
-      if @page
-        remote_a = []
-        all_a_tags_href.uniq.each do |link|
-          uri = URI(link.to_ascii)
-          if uri && @site_domain
-            remote_a << link unless uri.host == @site_domain
-          end
-        end
-        remote_a
-      end
-    end
-
-    def all_a_tags_href
-      if @page
-        tags = []
-          @page.css('a').each do |node|
-            tags << node['href']
-          end
-        tags.compact
-      end
-    end
-
+    # check if page have h2 tags
     def h2?
       @page.css('h2').size > 0 if @page
     end
-
-    def page_text_size
+    # return page size in symbols
+    def text_size
       @page.text.size if @page
     end
-
+    # get all a tags
     def all_a_tags
       if @page
         tags = []
@@ -163,15 +175,15 @@ module SiteAnalyzer
         tags.compact
       end
     end
-
-    def all_titles
+    # return all page titles
+    def titles
       if @page
         titles = []
         @page.css('title').each { |tag| titles << tag.text }
         titles
       end
     end
-
+    # return all meta description content
     def all_meta_description_content
       if @page
         tags = []
@@ -181,7 +193,7 @@ module SiteAnalyzer
         tags
       end
     end
-
+    # return all h2 tags text
     def h2
       if @page
         h2s = []
@@ -189,9 +201,13 @@ module SiteAnalyzer
         h2s
       end
     end
-
+    # check url of page that is must be HLU
     def bad_url
       @page_url if @page_path.size > 1 unless @page_path =~ /^[\w.\-\/]+$/i
     end
+    # clear page from don't needed information
+    def clear!
+      @page = nil
+    end
   end
 end

data/lib/site_analyzer/report.rb

@@ -11,7 +11,7 @@ module SiteAnalyzer
       @use_robot = use_robot
       @site = Site.new(@site_domain, @max_pages, @use_robot)
     end
-
+    # Entry point for gem. Create and show report. return array, show in console if select
     def self.create(options)
       options[:robot] = false if options[:robot] == 'false'
       options[:console] = false if options[:console] == 'false'
@@ -40,7 +40,7 @@ module SiteAnalyzer
 
     def to_s
       return 'Report is empty' if @report.nil? || @report.empty?
-      header = Terminal::Table.new title: "Report for #{@site_domain} with #{@max_pages} pages max_pages and robot check is #{@use_robot}"
+      header = Terminal::Table.new title: "Report for #{@site_domain} with #{@max_pages} pages and robot.txt check is #{@use_robot}"
       puts header
       @report.each_pair do |key, value|
         rows = []
@@ -57,7 +57,7 @@ module SiteAnalyzer
     def check_titles_text_less_than_70
       result = []
       @site.pages.each do |page|
-        result << page.page_url unless page.title_good?
+        result << page.page_url unless page.title_good
       end
       result
     end
@@ -65,7 +65,7 @@ module SiteAnalyzer
     def check_title_and_h1_for_doubles
       result = []
       @site.pages.each do |page|
-        result << page.page_url unless page.title_and_h1_good?
+        result << page.page_url unless page.title_and_h1_good
       end
       result
     end
@@ -73,7 +73,7 @@ module SiteAnalyzer
     def check_meta_description
       result = []
       @site.pages.each do |page|
-        result << page.page_url unless page.metadescription_good?
+        result << page.page_url unless page.meta_description_good
       end
       result
     end
@@ -81,7 +81,7 @@ module SiteAnalyzer
     def check_meta_keywords_tags
       result = []
       @site.pages.each do |page|
-        result << page.page_url unless page.keywords_good?
+        result << page.page_url unless page.meta_keywords
       end
       result
     end
@@ -89,7 +89,7 @@ module SiteAnalyzer
     def check_h2
       result = []
       @site.pages.each do |page|
-        result << page.page_url unless page.h2?
+        result << page.page_url unless page.have_h2
       end
       result
     end
@@ -105,7 +105,7 @@ module SiteAnalyzer
     def code_more
       result = []
       @site.pages.each do |page|
-        result << page.page_url unless page.code_less?
+        result << page.page_url unless page.code_less
       end
       result
     end

data/lib/site_analyzer/site.rb

@@ -16,7 +16,7 @@ module SiteAnalyzer
       @pages << Page.new(convert_to_valid(@main_url))
       scan_site!
     end
-
+    # check if page blocked by robot txt
     def robot_txt_allowed?(url)
       if @use_robot_txt
         Robotstxt.allowed?(url, '*') rescue nil
@@ -24,7 +24,7 @@ module SiteAnalyzer
         true
       end
     end
-
+    # scan pages: add page to scan, if still can scan do it, add new pages for scan from it and optimize massive of links
     def scan_site!
       add_pages_for_scan!
       while @pages_for_scan.size > 0
@@ -38,20 +38,20 @@ module SiteAnalyzer
         end
       end
     end
-
+    # add pages for scan array, also add bad pages to bad_pages array
     def add_pages_for_scan!
       @pages_for_scan = []
       @bad_pages = []
       @pages.each do |page|
-        @bad_pages << page.page_url unless page.page
-        if page.page
+        @bad_pages << page.page_url unless page.page_a_tags
+        if page.page_a_tags
           page.home_a.each do |link|
             @pages_for_scan << link
           end
         end
       end
     end
-
+    # create Page and add to to site
     def add_page(url)
       unless robot_txt_allowed?(url)
         @scanned_pages << url
@@ -61,42 +61,42 @@ module SiteAnalyzer
       @pages << page
       @scanned_pages << url
     end
-
+    # get all titles on site and return array of them
     def all_titles
       result = []
       @pages.each do |page|
-        if page.page
-          result << [page.page_url, page.titles]
+        if page.page_a_tags
+          result << [page.page_url, page.all_titles]
         end
       end
       result
     end
-
+    # get all meta description tags content and return it as array
     def all_descriptions
       result = []
       @pages.each do |page|
-        if page.page
-          result << [page.page_url, page.all_meta_description_content]
+        if page.page_a_tags
+          result << [page.page_url, page.meta_desc_content]
         end
       end
       result
     end
-
+    # get all h2 tags and return array of it
     def all_h2
       result = []
       @pages.each do |page|
-        unless page.page
-          result << [page.page_url, page.h2]
+        unless page.page_a_tags
+          result << [page.page_url, page.h2_text]
         end
       end
       result
     end
-
+    # get all a tags and return array of it
     def all_a
       result = []
       @pages.each do |page|
-        if page.page
-          page.all_a_tags.compact.each do |tag|
+        if page.page_a_tags
+          page.page_a_tags.compact.each do |tag|
             tag[0] = '-' unless tag[0]
             tag[1] = '-' unless tag[1]
             tag[2] = '-' unless tag[2]
@@ -106,29 +106,21 @@ module SiteAnalyzer
       end
       result.compact
     end
-
-    def pages_url
-      result = []
-      @pages.each do |page|
-         result << page.page_url if page.page
-      end
-      result
-    end
-
+    # get all non HLU url and return array
     def bad_urls
       result = []
       @pages.each do |page|
-        result << page.bad_url
+        result << page.hlu
       end
       result.compact!
     end
-
+    # get new array pages for scan and compact it
     def optimize_scan!
       @pages_for_scan = @pages_for_scan.compact.uniq
       @scanned_pages = @scanned_pages.compact.uniq
       @pages_for_scan = @pages_for_scan - @scanned_pages
     end
-
+    # check url and try to convert it to valid, remove .jpg links, add scheme to url
     def convert_to_valid(url)
       return nil if url =~ /.jpg$/i
       url.insert(0, @main_url.first(5)) if url.start_with? '//'

data/lib/site_analyzer/version.rb

@@ -1,3 +1,3 @@
 module SiteAnalyzer
-  VERSION = '0.3.14'
+  VERSION = '0.3.15'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: site_analyzer
 version: !ruby/object:Gem::Version
-  version: 0.3.14
+  version: 0.3.15
 platform: ruby
 authors:
 - Denis Savchuk