site_analyzer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ce267934c8aefcd9a1144ba3cda530c4fcfde312
+  data.tar.gz: eb2f6963c302c2fcba81c064226881b18bea107f
+SHA512:
+  metadata.gz: c305c773cd2e43c32b81d3f19c9b7c12a0dfd4c6c763c9645080b0b167fb96b6ba538ee2ad72a6e620776382ae240904c23c840d5b9a0fd12d37f6edab2164c0
+  data.tar.gz: a182a03fabb43e1a9a0376552dec05aba9bd354954b9de7159488a0482babe9a2110a01db4c5f64c26f50633ff6c1f3842446c4f07439e703efefad3f4ee4716

data/.gitignore

@@ -0,0 +1,3 @@
+/docs/
+/.idea/
+SiteAnalyzer.iml

data/.idea/uiDesigner.xml

@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Palette2">
+    <group name="Swing">
+      <item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
+      </item>
+      <item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
+      </item>
+      <item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
+      </item>
+      <item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
+        <default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
+      </item>
+      <item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
+        <initial-values>
+          <property name="text" value="Button" />
+        </initial-values>
+      </item>
+      <item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
+        <initial-values>
+          <property name="text" value="RadioButton" />
+        </initial-values>
+      </item>
+      <item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
+        <initial-values>
+          <property name="text" value="CheckBox" />
+        </initial-values>
+      </item>
+      <item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
+        <initial-values>
+          <property name="text" value="Label" />
+        </initial-values>
+      </item>
+      <item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
+          <preferred-size width="150" height="-1" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
+          <preferred-size width="150" height="-1" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
+          <preferred-size width="150" height="-1" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
+          <preferred-size width="150" height="50" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
+          <preferred-size width="150" height="50" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
+          <preferred-size width="150" height="50" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
+      </item>
+      <item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
+          <preferred-size width="150" height="50" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
+          <preferred-size width="150" height="50" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
+          <preferred-size width="150" height="50" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
+          <preferred-size width="200" height="200" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
+          <preferred-size width="200" height="200" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
+      </item>
+      <item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
+      </item>
+      <item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
+      </item>
+      <item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
+      </item>
+      <item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
+          <preferred-size width="-1" height="20" />
+        </default-constraints>
+      </item>
+      <item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
+        <default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
+      </item>
+      <item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
+        <default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
+      </item>
+    </group>
+  </component>
+</project>

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--require spec_helper
+--format doc

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.2
+before_install: gem install bundler -v 1.10.5

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile

@@ -0,0 +1,3 @@
+# A sample Gemfile
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,43 @@
+PATH
+  remote: .
+  specs:
+    site_analyzer (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.8)
+    diff-lcs (1.2.5)
+    mini_portile (0.6.2)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    rake (10.4.2)
+    robotstxt (0.5.4)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.1)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  addressable (~> 2.3)
+  bundler (~> 1.10)
+  nokogiri (~> 1.6)
+  rake (~> 10.0)
+  robotstxt (~> 0.5)
+  rspec (~> 3.3)
+  site_analyzer!
+
+BUNDLED WITH
+   1.10.5

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Denis Savchuk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,41 @@
+<a href="https://github.com/Mordorreal/SiteAnalyzer"><b>Site Analyzer</b><a/>
+
+Site Analyzer is a Ruby SEO site analizer gem.
+
+Site Analyzer check site and make report for you.
+
+<b>Requirements</b>
+
+    Ruby >= 1.8.7
+    Nokogiri >= 1.6.6.2
+    Addressable >= 2.3.8
+    Robotstxt >= 0.5.4
+
+<b>Installation</b>
+
+This library is intended to be installed via the RubyGems system.
+
+$ gem install site_analyzer
+
+You might need administrator privileges on your system to install it.
+
+<b>How to use</b>
+
+require 'site_analyzer'<br>
+SiteAnalyzer.add_site 'http://savchuk.space' # add site to list<br>
+SiteAnalyzer.start # start creating of reports<br>
+SiteAnalyzer.show_all_reports # show all reports in console<br>
+SiteAnalyzer.show_report 5 # show report number 5 starting from 1 in console<br>
+SiteAnalyzer.report[0].report # return Hash report<br>
+
+<b>Author</b>
+
+Denis Savchuk <a href="mailto:denis@savchuk.space"><denis@savchuk.space></a>
+
+<b>Resources</b>
+
+   Homepage  <a href="savchuk.space" target="_blank">savchuk.space</a>
+
+<b>License</b>
+
+Copyright © 2015 Denis Savchuk, SiteAnalyzer is released under the MIT license.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/SiteAnalyzer.iml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="RUBY_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" scope="PROVIDED" name="addressable (v2.3.8, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="bundler (v1.10.5, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="diff-lcs (v1.2.5, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="mini_portile (v0.6.2, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="nokogiri (v1.6.6.2, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rake (v10.4.2, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="robotstxt (v0.5.4, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec (v3.3.0, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-core (v3.3.1, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-expectations (v3.3.0, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-mocks (v3.3.1, rbenv: 2.2.2) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-support (v3.3.0, rbenv: 2.2.2) [gem]" level="application" />
+  </component>
+</module>

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "site_analizer"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/site_analyzer/page.rb

@@ -0,0 +1,159 @@
+module SiteAnalyzer
+  # Get site page and provide metods for analyse
+  require 'nokogiri'
+  require 'addressable/uri'
+  require 'open-uri'
+  require 'timeout'
+  class Page
+    attr_reader :page_url, :titles
+    def initialize(url)
+      @page_url = url
+      @page = get_page url
+      @site_url = get_domain url
+      @titles = all_titles
+    end
+
+    def get_page(url)
+      return nil if url.nil?
+      timeout(10) { Nokogiri::HTML(open(url)) }
+    rescue
+      nil
+    end
+
+    def get_domain(url)
+      return nil if url.nil?
+      timeout(10) { Addressable::URI.parse(url).host }
+    rescue
+      nil
+    end
+
+    def title_good?
+      return false if (title = @page.search('//title').children) == 0
+      title.text.size < 70
+    end
+    # true if title and h1 have no dublicates
+    def title_and_h1_good?
+      out = @page.search('//title', '//h1').children
+      arr = []
+      out.each { |node| arr << node.text }
+      arr.size > 1 || arr.uniq.size == arr.size
+    end
+    # true if metadescription less then 200 symbols
+    def metadescription_good?
+      tags = @page.css("meta[name='description']")
+      return false if tags.size == 0
+      tags.each do |t|
+        unless t['value'].nil?
+          return false if t['content'].size == 0 || t['content'].size > 200
+        end
+      end
+      true
+    end
+    # true if keywords less then 600 symbols
+    def keywords_good?
+      tags = @page.css("meta[name='keywords']")
+      return false if tags.size == 0
+      tags.each do |t|
+        unless t['value'].nil?
+          return false if t['content'].size == 0 || t['content'].size > 600
+        end
+      end
+      true
+    end
+    # true if code less then text
+    def code_less?
+      sum = 0
+      page_text = @page.text.size
+      @page.css('script').each do |tag|
+        sum += tag.text.size
+      end
+      sum < page_text / 2
+    end
+
+    def collect_metadates
+      @page.css('meta')
+    end
+
+    def metadates_good?
+      meta_tags = collect_metadates
+      return false if @page.css('title').size > 1 || meta_tags.nil?
+      node_names = []
+      meta_tags.each { |node| node_names << node['name'] }
+      return false if node_names.compact!.size < 1
+      node_names.uniq.size == node_names.size
+    end
+    # return hash with all titles, h1 and h2
+    def all_titles_h1_h2
+      out = []
+      out << @page.css('title').text << { @page_url => @page.css('h1').text }
+      out << { @page_url => @page.css('h2').text }
+    end
+
+    def home_a
+      home_a = []
+      all_a_tags_href.each do |link|
+        home_a << link if link.include? @site_url
+      end
+      home_a
+    end
+
+    def remote_a
+      remote_a = []
+      all_a_tags_href.uniq.each do |link|
+        remote_a << link unless link.include? @site_url
+      end
+      remote_a
+    end
+
+    def all_a_tags_href
+      tags = []
+      @page.css('a').each do |node|
+        tags << node['href'] unless node['href'].nil? || node['href'].size == 0
+      end
+      tags
+    end
+
+    def wrong_a
+      wrong_a = []
+      all_a_tags_href.each do |link|
+        wrong_a << link if link.include? '?meta='
+      end
+      wrong_a
+    end
+
+    def h2?
+      @page.css('h2').size > 0
+    end
+
+    def page_text_size
+      @page.text.size
+    end
+
+    def all_a_tags
+      tags = []
+      @page.css('a').each do |node|
+        tags << [node['href'], node['target'], node['rel']]
+      end
+      tags
+    end
+
+    def all_titles
+      titles = []
+      @page.css('title').each { |tag| titles << tag.text }
+      titles
+    end
+
+    def all_meta_description_content
+      tags = []
+      @page.css("meta[name='description']").each do |t|
+        tags << t['content']
+      end
+      tags
+    end
+    def h2
+      h2s = []
+      @page.css('h2').each { |tag| h2s << tag.text }
+      h2s
+    end
+  end
+end

data/lib/site_analyzer/report.rb

@@ -0,0 +1,166 @@
+module SiteAnalyzer
+  # Create report for site
+  class Report
+    attr_reader :site, :report
+    def initialize(site_url, max_pages, use_robot = false)
+      @site_url = site_url
+      @max_pages = max_pages
+      @use_robot = use_robot
+      @site = Site.new(@site_url, @max_pages, @use_robot)
+    end
+
+    def make_report
+      @report = {}
+      @report[:title_more_then_70_symbols] = check_titles_text_less_than_70
+      @report[:title_and_h1_have_doubles] = check_title_and_h1_for_doubles
+      @report[:meta_description_more_than_200] = check_meta_description_less_then_200
+      @report[:meta_keywords_tags_more_than_600] = check_meta_keywords_tags
+      @report[:dont_have_h2_tags] = check_h2
+      @report[:pages_size_with_url] = pages_size
+      @report[:code_more_then_text_on] = code_more
+      @report[:a_tags_list] = a_tag_array
+      @report[:title_doubles] = title_doubles
+      @report[:meta_description_doubles] = meta_description_doubles
+      @report[:bad_url] = bad_url
+      @report[:h2_doubles] = h2_doubles
+      @report
+    end
+
+    def to_s
+      return 'Report is empty' if @report.nil? || @report.empty?
+      puts "================Report for #{@site_url} with #{@max_pages} pages max_pages and robot check is #{@use_robot}================"
+      @report.each_pair do |key, value|
+        puts "=====================================#{key}====================================="
+        puts value
+        puts '===================================================================================================='
+      end
+    end
+
+    def check_titles_text_less_than_70
+      result = []
+      @site.pages.each_pair do |url, page|
+        result << url unless page.title_good?
+      end
+      result
+    end
+
+    def check_title_and_h1_for_doubles
+      result = []
+      @site.pages.each_pair do |url, page|
+        result << url unless page.title_and_h1_good?
+      end
+      result
+    end
+
+    def check_meta_description_less_then_200
+      result = []
+      @site.pages.each_pair do |url, page|
+        result << url unless page.metadescription_good?
+      end
+      result
+    end
+
+    def check_meta_keywords_tags
+      result = []
+      @site.pages.each_pair do |url, page|
+        result << url unless page.keywords_good?
+      end
+      result
+    end
+
+    def check_h2
+      result = []
+      @site.pages.each_pair do |url, page|
+        result << url unless page.h2?
+      end
+      result
+    end
+
+    def pages_size
+      result = []
+      @site.pages.each_pair do |url, page|
+        result << [url, page.page_text_size]
+      end
+      result
+    end
+
+    def code_more
+      result = []
+      @site.pages.each_pair do |url, page|
+        result << url unless page.code_less?
+      end
+      result
+    end
+
+    def a_tag_array
+      result = []
+      @site.pages.each_pair do |home_page_of_url, page|
+        page.all_a_tags.each do |link|
+          result << [home_page_of_url, link[0], link[1], link[2]]
+        end
+      end
+      result
+    end
+
+    def title_doubles
+      find_doubles @site.all_titles
+    end
+
+    def not_uniq_words_in_meta
+      find_not_uniq_words @site.all_descriptions
+    end
+
+    def meta_description_doubles
+      find_doubles @site.all_descriptions
+    end
+
+    def bad_url
+      result = []
+      a_tag_array.each do |url|
+        result << url if url[1].include? '?meta='
+      end
+      result
+    end
+
+    def h2_doubles
+      find_doubles(@site.all_h2)
+    end
+
+    def not_uniq_words_in_h2
+      find_not_uniq_words @site.all_h2
+    end
+
+    # in_array must be [[url_of_page, words_in_string_with_space],[next, same_element]]
+    def find_not_uniq_words(in_array)
+      all_words = []
+      counter = {}
+      result = []
+      in_array.each do |url_desc_cont|
+        url_desc_cont[0][1].scan(/\w+/).each do |word|
+          all_words << word
+        end
+      end
+      all_words.each do |word|
+        if counter[word]
+          counter[word] += 1
+        else
+          counter[word] = 1
+        end
+      end
+      counter.each_pair do |word, number|
+        result << word if number > 1
+      end
+      result
+    end
+    # in_array must be [[url_of_page, words_in_string_with_space],[next, same_element]]
+    def find_doubles(in_array)
+      result = []
+      find_not_uniq_words(in_array).each do |not_uniq_word|
+        in_array.each do |url_desc_cont|
+          result << url_desc_cont if url_desc_cont[0][1].include? not_uniq_word
+        end
+      end
+      result
+    end
+  end
+end

data/lib/site_analyzer/site.rb

@@ -0,0 +1,66 @@
+module SiteAnalyzer
+  require 'robotstxt'
+  # Create site object with all scans
+  class Site
+    attr_reader :main_url, :pages, :domain
+    def initialize(url, max_pages = 10, use_robot_txt = false)
+      @main_url = url
+      @pages = {}
+      @max_pages = max_pages
+      @domain = Addressable::URI.parse(url).host
+      @use_robot_txt = use_robot_txt
+      all_site_pages(url)
+    end
+
+    def robot_txt_allowed?(url)
+      Robotstxt.allowed?(url, '*') rescue nil
+    end
+
+    def all_site_pages(url = main_url)
+      return nil if @max_pages < 0 || redirection_off(url).nil? || (robot_txt_allowed?(url) if @use_robot_txt)
+      @max_pages -= 1
+      page = Page.new url
+      urls_array = urls_of_pages(page)
+      if urls_array.size > 0
+        urls_array.each do |url_in|
+          all_site_pages(url_in) unless @pages.key?(url_in) || url_in.nil?
+        end
+      end
+      @pages[url] = page
+    end
+
+    def urls_of_pages(page)
+      page.home_a.uniq.delete_if { |url| url.start_with?('mailto') }
+    end
+
+    def redirection_off(url)
+      timeout(10) { open(url).base_uri.to_s }
+    rescue
+      nil
+    end
+
+    def all_titles
+      result = []
+      @pages.each_pair do |url, page|
+        result << [url, page.titles]
+      end
+      result
+    end
+
+    def all_descriptions
+      result = []
+      @pages.each_pair do |url, page|
+        result << [url, page.all_meta_description_content]
+      end
+      result
+    end
+
+    def all_h2
+      result = []
+      @pages.each_pair do |url, page|
+        result << [url, page.h2]
+      end
+      result
+    end
+  end
+end

data/lib/site_analyzer/version.rb

@@ -0,0 +1,3 @@
+module SiteAnalyzer
+  VERSION = "0.1.0"
+end

data/lib/site_analyzer.rb

@@ -0,0 +1,39 @@
+# Main class for program
+module SiteAnalyzer
+  require 'site_analyzer/page'
+  require 'site_analyzer/site'
+  require 'site_analyzer/report'
+  require 'site_analyzer/version'
+  require 'rubygems'
+  require 'nokogiri'
+  require 'addressable/uri'
+  require 'open-uri'
+  require 'timeout'
+  require 'robotstxt'
+  class << self
+    attr_reader :report
+    def add_site(site_url, max_pages = 100, robottxt = false)
+      @report = []
+      @report << SiteAnalyzer::Report.new(site_url, max_pages, robottxt)
+    end
+
+    def start
+      @report.each { |report| report.make_report }
+    end
+
+    def show_all_reports
+      return "Don't have any reports" if @report.nil?
+      @report.each {|report| report.to_s}
+    end
+
+    def show_report(number)
+      return "Don't have report with number #{number}" if @report[number - 1].nil?
+      number -= 1
+      @report[number].to_s
+    end
+
+    def clear
+      @report.clear
+    end
+  end
+end

data/site_analyzer.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'site_analyzer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'site_analyzer'
+  spec.version       = SiteAnalyzer::VERSION
+  spec.authors       = ['Denis Savchuk']
+  spec.email         = ['mordorreal@gmail.com']
+  spec.date          = '2015-07-01'
+  spec.summary       = %q{Make report for SEO. Analyse site like SEOs like. }
+  spec.description   = %q{Create site report for SEO many options.}
+  spec.homepage      = 'https://github.com/Mordorreal/SiteAnalyzer'
+  spec.license       = 'MIT'
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~>3.3'
+  spec.add_development_dependency 'nokogiri', '~>1.6'
+  spec.add_development_dependency 'addressable', '~>2.3'
+  spec.add_development_dependency 'robotstxt', '~>0.5'
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: site_analyzer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Denis Savchuk
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-07-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: robotstxt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+description: Create site report for SEO many options.
+email:
+- mordorreal@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".idea/uiDesigner.xml"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- SiteAnalyzer.iml
+- bin/console
+- bin/setup
+- lib/site_analyzer.rb
+- lib/site_analyzer/page.rb
+- lib/site_analyzer/report.rb
+- lib/site_analyzer/site.rb
+- lib/site_analyzer/version.rb
+- site_analyzer.gemspec
+homepage: https://github.com/Mordorreal/SiteAnalyzer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Make report for SEO. Analyse site like SEOs like.
+test_files: []