seo_app 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e86fc597edfa061ebcd58df1e2ef5fcd18a74a51
+  data.tar.gz: b1377333eb14d8f461d59b3eb5085da5494661cd
+SHA512:
+  metadata.gz: 2e944ee375622d6758bef2259789e1193a701bf697e225074c7d0fbc168f5e7af25e0b8c769223aa8ca533d3450fc65d72ce20e998f2116db722c7cab815a208
+  data.tar.gz: 668d9290fc8f7737f704917d61076345034b071f36bd2c0adc731f67d4eab77b6f7b2b162122a40b98b5600cedaf0a55bc0814f72447073837f46c2ecc2c2ab0

data/.gitignore

@@ -0,0 +1,28 @@
+.bundle
+db/*.sqlite3
+log/*.log
+tmp/
+
+.bundle/
+config/*.yml
+config/*.conf
+config/logrotate-config
+config/nginx-config
+db/schema.rb
+db/sphinx/
+log/
+public/cache/
+public/assets/
+public/ckeditor_assets/
+public/uploads
+vendor/bundle/
+.rails_footnotes
+.redcar/
+doc/
+vendor/
+dump.rdb
+
+coverage/
+*.gem
+
+seo_tools.sublime-workspace

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,5 @@
+Lint/UnderscorePrefixedVariableName:
+  Enabled: false
+
+Metrics/LineLength:
+  Max: 85

data/Gemfile

@@ -0,0 +1,30 @@
+source 'http://rubygems.org'
+ruby '2.2.3'
+
+gem 'rack', '~> 1.6.4'
+# gem 'rack-contrib', '~> 1.4.0'
+gem 'rake', '~> 10.4.2'
+gem 'thin', '~> 1.6.3'
+gem 'pg', '~> 0.18.3'
+
+gem 'sinatra', '~> 1.4.6'
+gem 'sinatra-param', '~> 1.4.0', require: 'sinatra/param'
+
+# Needed for cli gem
+gem 'slim', '~> 3.0.6'
+
+gem 'httparty', '~> 0.13.5'
+gem 'nokogiri', '~> 1.6.7.rc3'
+gem 'geoip', '~> 1.6.1'
+gem 'thor', '~> 0.19.1'
+
+group :development do
+end
+
+group :test do
+  gem 'rspec', '~> 3.3.0'
+  gem 'rack-test', '~> 0.6.3'
+  gem 'fakeweb', '~> 1.3.0'
+  # gem 'fakefs', '~> 0.6.7'
+  gem 'simplecov', require: false
+end

data/Gemfile.lock

@@ -0,0 +1,106 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    byebug (5.0.0)
+      columnize (= 0.9.0)
+    coderay (1.1.0)
+    columnize (0.9.0)
+    daemons (1.2.3)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    eventmachine (1.0.8)
+    fakeweb (1.3.0)
+    geoip (1.6.1)
+    httparty (0.13.5)
+      json (~> 1.8)
+      multi_xml (>= 0.5.2)
+    json (1.8.3)
+    method_source (0.8.2)
+    mini_portile (0.7.0.rc4)
+    multi_xml (0.5.5)
+    nokogiri (1.6.7.rc3)
+      mini_portile (~> 0.7.0.rc4)
+    nokogiri (1.6.7.rc3-x64-mingw32)
+      mini_portile (~> 0.7.0.rc4)
+    pg (0.18.3)
+    pg (0.18.3-x64-mingw32)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry (0.10.1-x64-mingw32)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+      win32console (~> 1.3)
+    pry-byebug (3.2.0)
+      byebug (~> 5.0)
+      pry (~> 0.10)
+    rack (1.6.4)
+    rack-protection (1.5.3)
+      rack
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rake (10.4.2)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.2)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    simplecov (0.10.0)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    sinatra (1.4.6)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    sinatra-param (1.4.0)
+      sinatra (~> 1.3)
+    slim (3.0.6)
+      temple (~> 0.7.3)
+      tilt (>= 1.3.3, < 2.1)
+    slop (3.6.0)
+    temple (0.7.6)
+    thin (1.6.3)
+      daemons (~> 1.0, >= 1.0.9)
+      eventmachine (~> 1.0)
+      rack (~> 1.0)
+    thor (0.19.1)
+    tilt (2.0.1)
+    win32console (1.3.2)
+
+PLATFORMS
+  ruby
+  x64-mingw32
+
+DEPENDENCIES
+  fakeweb (~> 1.3.0)
+  geoip (~> 1.6.1)
+  httparty (~> 0.13.5)
+  nokogiri (~> 1.6.7.rc3)
+  pg (~> 0.18.3)
+  pry
+  pry-byebug
+  rack (~> 1.6.4)
+  rack-test (~> 0.6.3)
+  rake (~> 10.4.2)
+  rspec (~> 3.3.0)
+  simplecov
+  sinatra (~> 1.4.6)
+  sinatra-param (~> 1.4.0)
+  slim (~> 3.0.6)
+  thin (~> 1.6.3)
+  thor (~> 0.19.1)
+
+BUNDLED WITH
+   1.10.6

data/Procfile

@@ -0,0 +1 @@
+web: bundle exec rackup config.ru -p $PORT

data/README.md

@@ -0,0 +1,29 @@
+# Seo tools
+
+Provides ability to scan selected pages for headers and all the links on page. Generating static html reports and serves them.
+
+### Heroku link
+
+http://rubyseoapp.herokuapp.com/
+
+### Example
+
+![Index image](https://raw.githubusercontent.com/Faulik/ruby_seo_tools/master/public/img/index_page.png)
+
+![Report image](https://raw.githubusercontent.com/Faulik/ruby_seo_tools/master/public/img/report_page.png)
+
+### Install
+  
+`bundle install --path vendor/bundle`
+
+### Run
+  
+`rackup config.ru`
+
+`bundle exec rackup config.ru`
+
+### Test
+
+`bundle exec rspec`
+
+Released under the MIT license

data/Rakefile

@@ -0,0 +1,11 @@
+require 'rubygems'
+require 'bundler/setup'
+#require 'rspec/core/rake_task'
+
+task :default => :test
+
+task :environment do
+  require File.expand_path('../config/environment', __FILE__)
+end
+
+Dir.glob('lib/tasks/*.rake').each { |r| import r }

data/bin/seo_app

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+require 'thor'
+require 'seo_app'
+require 'pry'
+
+require 'pathname'
+
+SeoApp.configure do |config|
+  config.database_type = 'files'
+end
+
+class Seo_app < Thor
+  desc 'parse url', 'Creates report html in current directory'
+  method_option :path, :aliases => '-p', :desc => 'Supply with path to look for'
+  def parse(_url)
+    puts "Parsing from url: #{_url}"
+    path_option options[:path]
+    puts "Saving in #{@path}"
+
+    SeoApp::LinkParser.new(_url).parse!
+    puts "Done!"
+  end
+
+  desc 'list [options]', ' get all reports in current folder or specific path'
+  method_option :path, :aliases => '-p', :desc => 'Supply with path to look for'
+  def list() 
+    path_option options[:path]
+    puts "Getting all reports from #{@path}"
+    puts 'Site url -- date of parsing -- key'
+    SeoApp::Storage.all_reports.each do |report|
+      puts "#{report[:site_url]} -- #{report[:date]} -- #{report[:key]}"
+    end
+    puts "That's all"
+  end
+
+  no_commands do
+    def path_option(_path)
+      @path = _path ? File.join(Dir.pwd, _path) : Dir.pwd
+      SeoApp::Storage.database.reports_path = Pathname.new(@path)
+    end
+  end
+end
+
+Seo_app.start

data/config.ru

@@ -0,0 +1,3 @@
+require_relative 'config/environment'
+
+run SeoApp::Application

data/config/boot.rb

@@ -0,0 +1,6 @@
+require 'rubygems'
+
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])

data/config/environment.rb

@@ -0,0 +1,15 @@
+$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
+
+require_relative 'boot'
+
+ENV['RACK_ENV'] ||= 'development'
+
+# If you have a Gemfile, require the gems listed there, including any gems
+# you've limited to :test, :development, or :production.
+Bundler.require(:default, ENV['RACK_ENV']) if defined?(Bundler)
+
+require 'seo_app'
+
+Dir["#{File.dirname(__FILE__)}/initializers/*.rb"].sort.each do |path|
+  require File.expand_path("../initializers/#{File.basename(path, '.rb')}", __FILE__)
+end

data/config/initializers/app.rb

@@ -0,0 +1,8 @@
+SeoApp.configure do |config|
+  # Postgress config
+  config.db_host = ENV['DATABASE_URL']
+  config.db_port = '5432'
+  config.db_name = 'd9972ipr26em8d'
+
+  config.database_type = 'files'
+end

data/lib/seo_app.rb

@@ -0,0 +1,13 @@
+require 'pathname'
+
+require 'seo_app/configuration'
+require 'seo_app/application'
+
+# Config module
+module SeoApp
+  # SeoApp.root_path.join('..')
+  #
+  def self.root_path
+    @root_path ||= Pathname.new(File.dirname(File.expand_path('../', __FILE__)))
+  end
+end

data/lib/seo_app/application.rb

@@ -0,0 +1,47 @@
+require 'sinatra'
+require 'sinatra/param'
+
+require_relative 'link_parser'
+require_relative 'storages/storage'
+
+module SeoApp
+  # Main routes
+  class Application < ::Sinatra::Application
+    # Configuration
+    set :public_folder, -> { SeoApp.root_path.join('public').to_s }
+    set :views, -> { SeoApp.root_path.join('views').to_s }
+    set :static, true
+    set :bind, '0.0.0.0'
+
+    # Helpers
+    helpers Sinatra::Param
+
+    # Middleware
+    use Rack::CommonLogger
+    use Rack::Reloader
+
+    get '/' do
+      param :error, String
+
+      @reports = SeoApp::Storage.all_reports
+      @error = params[:error] ? params[:error] : nil
+
+      slim :index
+    end
+
+    get '/reports/:key' do
+      param :key, String
+
+      SeoApp::Storage.report(params['key']) || redirect('/?error=NotFound')
+    end
+
+    post '/links' do
+      param :url, String, required: true
+
+      @link = LinkParser.new(params[:url])
+
+      redirect '/?error=NonValidURL' unless @link.parse!
+      redirect '/'
+    end
+  end
+end

data/lib/seo_app/configuration.rb

@@ -0,0 +1,42 @@
+# Config class
+module SeoApp
+  class << self
+    attr_accessor :configuration
+  end
+
+  def self.configure
+    self.configuration ||= Configuration.new
+    yield(configuration)
+  end
+
+  # Main confoguration class for app
+  class Configuration
+    attr_reader :database
+
+    # Files Storage
+    attr_accessor :reports_folder
+
+    # Db configuration
+    attr_accessor :db_host
+    attr_accessor :db_port
+    attr_accessor :db_name
+    attr_accessor :db_user
+    attr_accessor :db_password
+
+    def initialize
+      @reports_folder = '/public/reports/'
+    end
+
+    def database_type=(_type)
+      case _type
+      when 'postgres'
+        SeoApp::Storage.database = SeoApp::PgStorage.new
+        @database_type = 'postgres'
+      when 'files'
+        SeoApp::Storage.database = SeoApp::FileStorage.new
+        @database_type = 'files'
+      end
+
+    end
+  end
+end

data/lib/seo_app/link_parser.rb

@@ -0,0 +1,73 @@
+require 'httparty'
+require 'nokogiri'
+require 'slim'
+require 'uri'
+require 'geoip'
+
+# Links parsing parts of app
+module SeoApp
+  #
+  class LinkParser
+    attr_reader :links, :headers, :time, :geo, :url
+
+    def initialize(url)
+      @url = URI.parse(url)
+      @links = []
+      @headers = []
+      @geo = {}
+      @time = Time.now.strftime('%Y.%d.%m_%H-%M-%S')
+    end
+
+    def parse!
+      return false unless check_url
+
+      begin
+        _response = HTTParty.get @url
+      rescue
+        return false
+      end
+
+      procces_response _response
+    end
+
+    def procces_response(response)
+      @headers = response.headers
+      @geo = retrieve_geo_params
+      gather_links! response.body
+
+      SeoApp::Storage.save_report(generate_report, generate_file_name)
+    end
+
+    def gather_links!(body)
+      ::Nokogiri::HTML(body).css('a').each do |link|
+        @links << {
+          name: link.text || 'None',
+          url: link['href'] || 'None',
+          rel: link['rel'] || 'None',
+          target: link['target'] || 'None'
+        }
+      end
+    end
+
+    def generate_file_name
+      "#{@url.host}__#{@time}.html"
+    end
+
+    def generate_report
+      Slim::Template.new('views/reports.slim').render(self)
+    end
+
+    def retrieve_geo_params
+      GeoIP.new(SeoApp.root_path.join('db/GeoLiteCity.dat')).city(@url.host.to_s)
+    end
+
+    def check_url
+      @url = !@url.scheme ? URI.parse('http://' + @url.to_s) : @url
+      @url.is_a?(URI::HTTP) ? true : false
+    end
+
+    def self.say_hello
+      puts 'Hello'
+    end
+  end
+end