site_analyzer 0.3.9 → 0.3.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0c6718b5591b49460b14466007203bb65c3471b7
-  data.tar.gz: f5ca34a57a400647859935f706c576df74d85941
+  metadata.gz: d800ec488fdf5882e08d325ceb904af2b8559e0d
+  data.tar.gz: 7e963e3c8f71905e0e1dafadabd8d81c43bcb2a2
 SHA512:
-  metadata.gz: d29b19234afe6f5f93e43f8283bce187298435fc5e46a6254ca32928f55efab8a29e49e2349cf817bd9d320c04ca4cad0681a3c0d9eef2e426ea36466e2a95ab
-  data.tar.gz: bfce8893fc7918c6837be128a3f5efd21c269bb38be65c1be1316548aeab99cc5e5900d8c2c76ede67dc8e483668bca7446b413bc9995981bd2eed5d5a1af6d7
+  metadata.gz: 6bdb11a2a9c98b982c4f901e62e9a8000cdd4750f986a976414d8474065d695db8482f7ba8742237a28aa0d3d41e3e9fa233bb402695ad4b709c348675666fdb
+  data.tar.gz: d5c457238252264ae2722a1b1f361e8ab3f6be2210295542a347a468cbe16c807dd0a0d9a97264856b4bd347b3f4862494d2dc982ba4b50e4793853590ba981f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    site_analyzer (0.3.8)
+    site_analyzer (0.3.9)
       addressable (~> 2.3)
       nokogiri (~> 1.6)
       robotstxt (~> 0.5)

data/lib/site_analyzer/page.rb

@@ -7,7 +7,7 @@ module SiteAnalyzer
     attr_reader :page_url, :titles, :page
     def initialize(url)
       @page_url = url
-      @page = get_page url
+      @page = get_page(url)
       @site_url = get_domain url
       @titles = all_titles
     end
@@ -17,142 +17,171 @@ module SiteAnalyzer
     end
 
     def get_page(url)
-      timeout(30) { Nokogiri::HTML(open(url)) }
-    rescue Timeout::Error
-      ['Timeout exception']
+      begin
+        timeout(30) { Nokogiri::HTML(open(url)) }
+      rescue Timeout::Error, EOFError, OpenURI::HTTPError
+        return nil
+      end
     end
 
     def get_domain(url)
-      timeout(30) { Addressable::URI.parse(url).host }
-    rescue
-      'Error with parsing by Addressable'
+      timeout(30) { Addressable::URI.parse(url).host } rescue nil
     end
 
     def title_good?
-      @page.css('title').size == 1 &&  @page.css('title').text.size < 70
+      @page.css('title').size == 1 && @page.css('title').text.size < 70 if @page
     end
     # true if title and h1 have no dublicates
     def title_and_h1_good?
-      arr = []
-      @page.css('h1').each { |node| arr << node.text }
-      @page.css('title').size == 1 && arr.uniq.size == arr.size
+      if @page
+        arr = []
+        @page.css('h1').each { |node| arr << node.text }
+        @page.css('title').size == 1 && arr.uniq.size == arr.size
+      end
     end
     # true if metadescription less then 200 symbols
     def metadescription_good?
-      tags = @page.css("meta[name='description']")
-      return false if tags.size == 0
-      tags.each do |t|
-        unless t['value'].nil?
-          return false if t['content'].size == 0 || t['content'].size > 200
+      if @page
+        tags = @page.css("meta[name='description']")
+        return false if tags.size == 0
+        tags.each do |t|
+          unless t['value'].nil?
+            return false if t['content'].size == 0 || t['content'].size > 200
+          end
         end
+        true
       end
-      true
     end
     # true if keywords less then 600 symbols
     def keywords_good?
-      tags = @page.css("meta[name='keywords']")
-      return false if tags.size == 0
-      tags.each do |t|
-        unless t['value'].nil?
-          return false if t['content'].size == 0 || t['content'].size > 600
+      if @page
+        tags = @page.css("meta[name='keywords']")
+        return false if tags.size == 0
+        tags.each do |t|
+          unless t['value'].nil?
+            return false if t['content'].size == 0 || t['content'].size > 600
+          end
         end
+        true
       end
-      true
     end
     # true if code less then text
     def code_less?
-      sum = 0
-      page_text = @page.text.size
-      @page.css('script').each do |tag|
-        sum += tag.text.size
+      if @page
+        sum = 0
+        page_text = @page.text.size
+        @page.css('script').each do |tag|
+          sum += tag.text.size
+        end
+        sum < page_text / 2
       end
-      sum < page_text / 2
     end
 
     def collect_metadates
-      @page.css('meta')
+      @page.css('meta') if @page
     end
 
     def metadates_good?
-      meta_tags = collect_metadates
-      return false if @page.css('title').size > 1 || meta_tags.nil?
-      node_names = []
-      meta_tags.each { |node| node_names << node['name'] }
-      return false if node_names.compact!.size < 1
-      node_names.uniq.size == node_names.size
+      if @page
+        meta_tags = collect_metadates
+        return false if @page.css('title').size > 1 || meta_tags.nil?
+        node_names = []
+        meta_tags.each { |node| node_names << node['name'] }
+        return false if node_names.compact!.size < 1
+        node_names.uniq.size == node_names.size
+      end
     end
     # return hash with all titles, h1 and h2
     def all_titles_h1_h2
-      out = []
-      out << @page.css('title').text << { @page_url => @page.css('h1').text }
-      out << { @page_url => @page.css('h2').text }
+      if @page
+        out = []
+        out << @page.css('title').text << { @page_url => @page.css('h1').text }
+        out << { @page_url => @page.css('h2').text }
+      end
     end
 
     def home_a
-      home_a = []
-      all_a_tags_href.each do |link|
-        home_a << link if link.include? @site_url
+      if @page
+        home_a = []
+        all_a_tags_href.each do |link|
+          home_a << link if link.include? @site_url
+        end
+        home_a
       end
-      home_a
     end
 
     def remote_a
-      remote_a = []
-      all_a_tags_href.uniq.each do |link|
-        remote_a << link unless link.include? @site_url
+      if @page
+        remote_a = []
+        all_a_tags_href.uniq.each do |link|
+          remote_a << link unless link.include? @site_url
+        end
+        remote_a
       end
-      remote_a
     end
 
     def all_a_tags_href
-      tags = []
-      @page.css('a').each do |node|
-        tags << node['href']
+      if @page
+        tags = []
+          @page.css('a').each do |node|
+            tags << node['href']
+          end
+        tags.compact
       end
-      tags.compact
     end
 
     def wrong_a
-      wrong_a = []
-      all_a_tags_href.each do |link|
-        wrong_a << link if link.include? '?meta='
+      if @page
+        wrong_a = []
+        all_a_tags_href.each do |link|
+          wrong_a << link if link.include? '?meta='
+        end
+        wrong_a
       end
-      wrong_a
     end
 
     def h2?
-      @page.css('h2').size > 0
+      @page.css('h2').size > 0 if @page
     end
 
     def page_text_size
-      @page.text.size
+      @page.text.size if @page
     end
 
     def all_a_tags
-      tags = []
-      @page.css('a').each do |node|
-        tags << [node['href'], node['target'], node['rel']]
+      if @page
+        tags = []
+        @page.css('a').each do |node|
+          tags << [node['href'], node['target'], node['rel']]
+        end
+        tags.compact
       end
-      tags.compact
     end
 
     def all_titles
-      titles = []
-      @page.css('title').each { |tag| titles << tag.text }
-      titles
+      if @page
+        titles = []
+        @page.css('title').each { |tag| titles << tag.text }
+        titles
+      end
     end
 
     def all_meta_description_content
-      tags = []
-      @page.css("meta[name='description']").each do |t|
-        tags << t['content']
+      if @page
+        tags = []
+        @page.css("meta[name='description']").each do |t|
+          tags << t['content']
+        end
+        tags
       end
-      tags
     end
+
     def h2
-      h2s = []
-      @page.css('h2').each { |tag| h2s << tag.text }
-      h2s
+      if @page
+        h2s = []
+        @page.css('h2').each { |tag| h2s << tag.text }
+        h2s
+      end
     end
   end
 end

data/lib/site_analyzer/site.rb

@@ -36,9 +36,13 @@ module SiteAnalyzer
 
     def add_pages_for_scan!
       @pages_for_scan = []
+      @bad_pages = []
       @pages.each do |page|
-        page.home_a.each do |link|
-          @pages_for_scan << link unless link.nil? || @scanned_pages.include?(link) || link.include?('mailto:') || link.end_with?('.jpg')
+        @bad_pages << page.page_url unless page.page
+        if page.page
+          page.home_a.each do |link|
+            @pages_for_scan << link unless link.nil? || @scanned_pages.include?(link) || link.start_with?('mailto:') || link.start_with?('skype:') || link.end_with?('.jpg')
+          end
         end
       end
       @pages_for_scan.clear if @pages_for_scan.size == 0
@@ -58,7 +62,9 @@ module SiteAnalyzer
     def all_titles
       result = []
       @pages.each do |page|
-        result << [page.page_url, page.titles]
+        if page.page
+          result << [page.page_url, page.titles]
+        end
       end
       result
     end
@@ -66,7 +72,9 @@ module SiteAnalyzer
     def all_descriptions
       result = []
       @pages.each do |page|
-        result << [page.page_url, page.all_meta_description_content]
+        if page.page
+          result << [page.page_url, page.all_meta_description_content]
+        end
       end
       result
     end
@@ -74,7 +82,9 @@ module SiteAnalyzer
     def all_h2
       result = []
       @pages.each do |page|
-        result << [page.page_url, page.h2]
+        if page.page
+          result << [page.page_url, page.h2]
+        end
       end
       result
     end
@@ -82,11 +92,13 @@ module SiteAnalyzer
     def all_a
       result = []
       @pages.each do |page|
-        page.all_a_tags.compact.each do |tag|
-          tag[0] = '-' unless tag[0]
-          tag[1] = '-' unless tag[1]
-          tag[2] = '-' unless tag[2]
-          result << [page.page_url, tag[0], tag[1], tag[2]]
+        if page.page
+          page.all_a_tags.compact.each do |tag|
+            tag[0] = '-' unless tag[0]
+            tag[1] = '-' unless tag[1]
+            tag[2] = '-' unless tag[2]
+            result << [page.page_url, tag[0], tag[1], tag[2]]
+          end
         end
       end
       result.compact

data/lib/site_analyzer/version.rb

@@ -1,3 +1,3 @@
 module SiteAnalyzer
-  VERSION = "0.3.9"
+  VERSION = "0.3.10"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: site_analyzer
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.3.10
 platform: ruby
 authors:
 - Denis Savchuk