site_analyzer 0.3.12 → 0.3.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fe1d0ebb0469f94188c31f5111d4493337eb0c01
-  data.tar.gz: 0f4e260c964d757cfe17ba1f0a24c6ced3ccc004
+  metadata.gz: a1808537ffb31d17a123c4494015333f4569246a
+  data.tar.gz: 7e742cfaf72ff0173b58467dec770a4cc1d7db68
 SHA512:
-  metadata.gz: 0cf6c7e6ffa7819b39bb2bd9acab7e461056206f4d7de38d45a719287c67e361f0d92471c2209aea308b10d8b795e8fe68aa91a45e375535122c753f29876497
-  data.tar.gz: 705126a665745b90106c274cea37d2c550ca2b8de2b4d941d1d62a4a89800f3618059dfaa35596ee7fd2f040b6b9dfe89ca62cd672b847c2489522b808987f3a
+  metadata.gz: 83cd277f8e22a695b5a5ae6cd6ef3711ad39a2cf1e60efbc9497232b68ab2610ec8874d5ce816a9e383b1bf73c40612f2ad8065876db3abb870333e7a64b177c
+  data.tar.gz: 795456151c0b3c9256d020837e5c0294d77a06504d8b60a9126a406cfc7e0baa0bd103e0693daa8696f6cdf5652b5aa4975ea266e6698b37724eca7db4f4867c

data/Gemfile.lock

@@ -1,21 +1,55 @@
 PATH
   remote: .
   specs:
-    site_analyzer (0.3.11)
+    site_analyzer (0.3.12)
       addressable (~> 2.3)
       nokogiri (~> 1.6)
       robotstxt (~> 0.5)
+      stringex (~> 2.5)
       terminal-table (~> 1.5)
 
 GEM
   remote: https://rubygems.org/
   specs:
     addressable (2.3.8)
+    coderay (1.1.0)
     diff-lcs (1.2.5)
+    ffi (1.9.10)
+    formatador (0.2.5)
+    guard (2.13.0)
+      formatador (>= 0.2.4)
+      listen (>= 2.7, <= 4.0)
+      lumberjack (~> 1.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.9.12)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.6.3)
+      guard (~> 2.1)
+      guard-compat (~> 1.1)
+      rspec (>= 2.99.0, < 4.0)
+    listen (3.0.3)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+    lumberjack (1.0.9)
+    method_source (0.8.2)
     mini_portile (0.6.2)
+    nenv (0.2.0)
     nokogiri (1.6.6.2)
       mini_portile (~> 0.6.0)
+    notiffany (0.0.7)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
     rake (10.4.2)
+    rb-fsevent (0.9.5)
+    rb-inotify (0.9.5)
+      ffi (>= 0.5.0)
     robotstxt (0.5.4)
     rspec (3.3.0)
       rspec-core (~> 3.3.0)
@@ -30,13 +64,18 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.3.0)
     rspec-support (3.3.0)
+    shellany (0.0.1)
+    slop (3.6.0)
+    stringex (2.5.2)
     terminal-table (1.5.2)
+    thor (0.19.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 1.10)
+  guard-rspec (~> 4.6)
   rake (~> 10.4)
   rspec (~> 3.3)
   site_analyzer!

data/Guardfile

@@ -0,0 +1,70 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec features) \
+#  .select{|d| Dir.exists?(d) ? d : UI.warning("Directory #{d} does not exist")}
+
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), then you will want to move
+## the Guardfile to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+# Note: The cmd option is now required due to the increasing number of ways
+#       rspec may be run, below are examples of the most common uses.
+#  * bundler: 'bundle exec rspec'
+#  * bundler binstubs: 'bin/rspec'
+#  * spring: 'bin/rspec' (This will use spring if running and you have
+#                          installed the spring binstubs per the docs)
+#  * zeus: 'zeus rspec' (requires the server to be started separately)
+#  * 'just' rspec: 'rspec'
+
+guard :rspec, cmd: 'bundle exec rspec' do
+  require 'guard/rspec/dsl'
+  dsl = Guard::RSpec::Dsl.new
+
+  # Feel free to open issues for suggestions and improvements
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+
+  # Rails files
+  rails = dsl.rails(view_extensions: %w(erb haml slim))
+  dsl.watch_spec_files_for(rails.app_files)
+  dsl.watch_spec_files_for(rails.views)
+
+  watch(rails.controllers) do |m|
+    [
+      rspec.spec.("routing/#{m[1]}_routing"),
+      rspec.spec.("controllers/#{m[1]}_controller"),
+      rspec.spec.("acceptance/#{m[1]}")
+    ]
+  end
+
+  # Rails config changes
+  watch(rails.spec_helper)     { rspec.spec_dir }
+  watch(rails.routes)          { "#{rspec.spec_dir}/routing" }
+  watch(rails.app_controller)  { "#{rspec.spec_dir}/controllers" }
+
+  # Capybara features specs
+  watch(rails.view_dirs)     { |m| rspec.spec.("features/#{m[1]}") }
+  watch(rails.layouts)       { |m| rspec.spec.("features/#{m[1]}") }
+
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$}) do |m|
+    Dir[File.join("**/#{m[1]}.feature")][0] || 'spec/acceptance'
+  end
+end

data/Rakefile

@@ -1,5 +1,5 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 

data/SiteAnalyzer.iml

@@ -8,15 +8,24 @@
     <orderEntry type="library" scope="PROVIDED" name="addressable (v2.3.8, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="bundler (v1.10.5, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="diff-lcs (v1.2.5, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="ffi (v1.9.10, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="formatador (v0.2.5, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="guard-compat (v1.2.1, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="mini_portile (v0.6.2, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="nenv (v0.2.0, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="nokogiri (v1.6.6.2, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rake (v10.4.2, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rb-inotify (v0.9.5, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="robotstxt (v0.5.4, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rspec (v3.3.0, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rspec-core (v3.3.2, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rspec-expectations (v3.3.1, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rspec-mocks (v3.3.2, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rspec-support (v3.3.0, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="shellany (v0.0.1, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="slop (v3.6.0, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="stringex (v2.5.2, rbenv: 2.2.2) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="terminal-table (v1.5.2, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="thor (v0.19.1, rbenv: 2.2.2) [gem]" level="application" />
   </component>
 </module>

data/bin/console

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "site_analizer"
+require 'bundler/setup'
+require 'site_analizer'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
@@ -10,5 +10,5 @@ require "site_analizer"
 # require "pry"
 # Pry.start
 
-require "irb"
+require 'irb'
 IRB.start

data/lib/site_analyzer.rb

@@ -1,6 +1,8 @@
 # Main class for program
 module SiteAnalyzer
-  %w(page site report version open-uri-patching).each do |file|
+  require 'stringex_lite'
+  %w(open-uri-patching page report site version).each do |file|
     require "site_analyzer/#{file}"
   end
+  Stringex::Localization.default_locale = :en
 end

data/lib/site_analyzer/page.rb

@@ -3,31 +3,41 @@ module SiteAnalyzer
   require 'nokogiri'
   require 'addressable/uri'
   require 'timeout'
+  require 'stringex_lite'
+  require 'open-uri'
   class Page
-    attr_reader :page_url, :titles, :page
+    attr_reader :page_url, :titles, :page, :page_path, :site_domain
     def initialize(url)
       @page_url = url
-      @page = get_page(url)
-      @site_url = get_domain url
+      @page = []
+      @site_domain = ''
+      @page_path = ''
+      @titles = []
+      get_page(url)
+      fill_data_field!
+    end
+
+    def fill_data_field!
       @titles = all_titles
     end
 
     def to_s
-      "Page url: #{@page_url} Site url: #{@site_url}"
+      "Page url: #{@page_url} Site url: #{@site_domain}"
     end
 
     def get_page(url)
       begin
-        timeout(30) { Nokogiri::HTML(open(url)) }
+        timeout(30) do
+          page = open(url)
+          @site_domain = page.base_uri.host
+          @page_path = page.base_uri.request_uri
+          @page = Nokogiri::HTML(page)
+        end
       rescue Timeout::Error, EOFError, OpenURI::HTTPError, Errno::ENOENT
         return nil
       end
     end
 
-    def get_domain(url)
-      timeout(30) { Addressable::URI.parse(url).host } rescue nil
-    end
-
     def title_good?
       @page.css('title').size == 1 && @page.css('title').text.size < 70 if @page
     end
@@ -104,9 +114,9 @@ module SiteAnalyzer
       if @page
         home_a = []
         all_a_tags_href.uniq.each do |link|
-          if get_domain(link) && @site_url
-            domain = get_domain(link)
-            home_a << link if domain == @site_url
+          uri = URI(link.to_ascii) rescue nil #TODO: write additional logic for link to image
+          if uri && @site_domain
+            home_a << link if uri.host == @site_domain
           end
         end
         home_a
@@ -117,9 +127,9 @@ module SiteAnalyzer
       if @page
         remote_a = []
         all_a_tags_href.uniq.each do |link|
-          if get_domain(link) && @site_url
-            domain = get_domain(link)
-            remote_a << link unless domain == @site_url
+          uri = URI(link.to_ascii)
+          if uri && @site_domain
+            remote_a << link unless uri.host == @site_domain
           end
         end
         remote_a
@@ -179,5 +189,9 @@ module SiteAnalyzer
         h2s
       end
     end
+
+    def bad_url
+      @page_url unless @page_path.size <= 1 && @page_path =~ /^[\w.\-\/]+$/
+    end
   end
 end

data/lib/site_analyzer/report.rb

@@ -6,10 +6,10 @@ module SiteAnalyzer
   class Report
     attr_reader :site, :report
     def initialize(site_url, max_pages = 10, use_robot = false)
-      @site_url = site_url
+      @site_domain = site_url
       @max_pages = max_pages
       @use_robot = use_robot
-      @site = Site.new(@site_url, @max_pages, @use_robot)
+      @site = Site.new(@site_domain, @max_pages, @use_robot)
     end
 
     def self.create(options)
@@ -25,7 +25,7 @@ module SiteAnalyzer
       @report = {}
       @report[:title_more_then_70_symbols] = check_titles_text_less_than_70
       @report[:title_and_h1_have_doubles] = check_title_and_h1_for_doubles
-      @report[:meta_description_more_than_200] = check_meta_description_less_then_200
+      @report[:meta_description_more_than_200] = check_meta_description
       @report[:meta_keywords_tags_more_than_600] = check_meta_keywords_tags
       @report[:dont_have_h2_tags] = check_h2
       @report[:pages_size_with_url] = pages_size
@@ -40,7 +40,7 @@ module SiteAnalyzer
 
     def to_s
       return 'Report is empty' if @report.nil? || @report.empty?
-      header = Terminal::Table.new title: "Report for #{@site_url} with #{@max_pages} pages max_pages and robot check is #{@use_robot}"
+      header = Terminal::Table.new title: "Report for #{@site_domain} with #{@max_pages} pages max_pages and robot check is #{@use_robot}"
       puts header
       @report.each_pair do |key, value|
         rows = []
@@ -70,7 +70,7 @@ module SiteAnalyzer
       result
     end
 
-    def check_meta_description_less_then_200
+    def check_meta_description
       result = []
       @site.pages.each do |page|
         result << page.page_url unless page.metadescription_good?
@@ -115,16 +115,7 @@ module SiteAnalyzer
     end
 
     def bad_url
-      result = []
-      a_tag_array.each do |url|
-        begin
-          uri = URI(url[1])
-          result << url if (uri.scheme == 'http' || uri.scheme == 'https' ) unless uri.path && uri.path =~ /^[\w\-\/\+\.]+$/
-        rescue URI::InvalidURIError
-          result << url
-        end
-      end
-      result
+      @site.bad_urls
     end
 
     def title_doubles

data/lib/site_analyzer/site.rb

@@ -4,16 +4,16 @@ module SiteAnalyzer
   require 'timeout'
   # Create site object with all scans
   class Site
-    attr_reader :main_url, :pages, :domain, :pages_for_scan, :max_pages
+    attr_reader :main_url, :pages, :pages_for_scan, :max_pages, :scanned_pages
     def initialize(url, max_pages = 10, use_robot_txt = false)
+      Stringex::Localization.default_locale = :en
       @main_url = url
       @pages = []
-      @max_pages = max_pages
-      @domain = Addressable::URI.parse(url).host
       @use_robot_txt = use_robot_txt
       @scanned_pages = []
       @pages_for_scan = []
-      add_page url
+      @max_pages = max_pages - 1
+      @pages << Page.new(convert_to_valid(@main_url))
       scan_site!
     end
 
@@ -28,11 +28,14 @@ module SiteAnalyzer
     def scan_site!
       add_pages_for_scan!
       while @pages_for_scan.size > 0
-        add_page @pages_for_scan.pop
-        return if @max_pages <= 0
-        add_pages_for_scan!
-        optimize_scan!
-        return if @pages_for_scan.size == 0
+        page = convert_to_valid @pages_for_scan.pop
+        if page
+          @max_pages -= 1
+          add_page convert_to_valid(page)
+          return if @max_pages <= 0
+          add_pages_for_scan!
+          optimize_scan!
+        end
       end
     end
 
@@ -43,7 +46,7 @@ module SiteAnalyzer
         @bad_pages << page.page_url unless page.page
         if page.page
           page.home_a.each do |link|
-            @pages_for_scan << link unless link.nil? || link.start_with?('mailto:') || link.start_with?('skype:') || link.end_with?('.jpg')
+            @pages_for_scan << link
           end
         end
       end
@@ -56,7 +59,6 @@ module SiteAnalyzer
       end
       page = Page.new(url)
       @pages << page
-      @max_pages -= 1
       @scanned_pages << url
     end
 
@@ -83,7 +85,7 @@ module SiteAnalyzer
     def all_h2
       result = []
       @pages.each do |page|
-        if page.page
+        unless page.page
           result << [page.page_url, page.h2]
         end
       end
@@ -105,10 +107,49 @@ module SiteAnalyzer
       result.compact
     end
 
+    def pages_url
+      result = []
+      @pages.each do |page|
+         result << page.page_url if page.page
+      end
+      result
+    end
+
+    def bad_urls
+      result = []
+      @pages.each do |page|
+        result << page.bad_url
+      end
+      result.compact!
+    end
+
     def optimize_scan!
-      @pages_for_scan.uniq.compact!
-      @scanned_pages.uniq.compact!
+      @pages_for_scan = @pages_for_scan.compact.uniq
+      @scanned_pages = @scanned_pages.compact.uniq
       @pages_for_scan = @pages_for_scan - @scanned_pages
     end
+
+    def convert_to_valid(url)
+      return nil if url =~ /.jpg$/i
+      url.insert(0, @main_url.first(5)) if url.start_with? '//'
+      link = URI(url)
+      main_page = URI(@main_url)
+      if link && link.scheme && link.scheme.empty?
+        link.scheme = main_page.scheme
+      elsif link.nil?
+        return nil
+      end
+      if link.scheme =~ /^http/
+        request = link.scheme + '://' + link.host
+        if link.request_uri
+          request += link.request_uri
+        end
+      else
+        request = nil
+      end
+      request
+    rescue
+      link
+    end
   end
 end

data/lib/site_analyzer/version.rb

@@ -1,3 +1,3 @@
 module SiteAnalyzer
-  VERSION = "0.3.12"
+  VERSION = '0.3.13'
 end

data/site_analyzer.gemspec

@@ -21,9 +21,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 1.10'
   spec.add_development_dependency 'rake', '~> 10.4'
   spec.add_development_dependency 'rspec', '~> 3.3'
+  spec.add_development_dependency 'guard-rspec', '~> 3.3'
   spec.add_runtime_dependency 'nokogiri', '~> 1.6'
   spec.add_runtime_dependency 'addressable', '~> 2.3'
   spec.add_runtime_dependency 'robotstxt', '~> 0.5'
   spec.add_runtime_dependency 'terminal-table', '~> 1.5'
+  spec.add_runtime_dependency 'stringex', '~> 2.5'
   spec.required_ruby_version = '~> 2.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: site_analyzer
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.13
 platform: ruby
 authors:
 - Denis Savchuk
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: stringex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
 description: Create site report for SEO many options.
 email:
 - mordorreal@gmail.com
@@ -121,6 +149,7 @@ files:
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
+- Guardfile
 - LICENSE
 - README.md
 - Rakefile