seo_app 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +28 -0
- data/.rspec +2 -0
- data/.rubocop.yml +5 -0
- data/Gemfile +30 -0
- data/Gemfile.lock +106 -0
- data/Procfile +1 -0
- data/README.md +29 -0
- data/Rakefile +11 -0
- data/bin/seo_app +44 -0
- data/config.ru +3 -0
- data/config/boot.rb +6 -0
- data/config/environment.rb +15 -0
- data/config/initializers/app.rb +8 -0
- data/db/GeoLiteCity.dat +0 -0
- data/lib/seo_app.rb +13 -0
- data/lib/seo_app/application.rb +47 -0
- data/lib/seo_app/configuration.rb +42 -0
- data/lib/seo_app/link_parser.rb +73 -0
- data/lib/seo_app/storages/file_storage.rb +40 -0
- data/lib/seo_app/storages/pg_storage.rb +16 -0
- data/lib/seo_app/storages/storage.rb +46 -0
- data/public/favicon.ico +0 -0
- data/public/img/index_page.png +0 -0
- data/public/img/report_page.png +0 -0
- data/public/js/code.js +6 -0
- data/public/js/tablesort.min.js +5 -0
- data/public/reports/.gitignore +2 -0
- data/public/robots.txt +2 -0
- data/public/styles/bootstrap-theme.css +587 -0
- data/public/styles/bootstrap-theme.css.map +1 -0
- data/public/styles/bootstrap-theme.min.css +5 -0
- data/public/styles/bootstrap.css +6800 -0
- data/public/styles/bootstrap.css.map +1 -0
- data/public/styles/bootstrap.min.css +5 -0
- data/public/styles/styles.css +26 -0
- data/seo_app.gemspec +23 -0
- data/spec/parser/link_parser_spec.rb +51 -0
- data/spec/parser/reports_spec.rb +20 -0
- data/spec/routes/index_spec.rb +44 -0
- data/spec/spec_helper.rb +22 -0
- data/views/index.slim +42 -0
- data/views/reports.slim +67 -0
- metadata +191 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: e86fc597edfa061ebcd58df1e2ef5fcd18a74a51
|
4
|
+
data.tar.gz: b1377333eb14d8f461d59b3eb5085da5494661cd
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 2e944ee375622d6758bef2259789e1193a701bf697e225074c7d0fbc168f5e7af25e0b8c769223aa8ca533d3450fc65d72ce20e998f2116db722c7cab815a208
|
7
|
+
data.tar.gz: 668d9290fc8f7737f704917d61076345034b071f36bd2c0adc731f67d4eab77b6f7b2b162122a40b98b5600cedaf0a55bc0814f72447073837f46c2ecc2c2ab0
|
data/.gitignore
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
.bundle
|
2
|
+
db/*.sqlite3
|
3
|
+
log/*.log
|
4
|
+
tmp/
|
5
|
+
|
6
|
+
.bundle/
|
7
|
+
config/*.yml
|
8
|
+
config/*.conf
|
9
|
+
config/logrotate-config
|
10
|
+
config/nginx-config
|
11
|
+
db/schema.rb
|
12
|
+
db/sphinx/
|
13
|
+
log/
|
14
|
+
public/cache/
|
15
|
+
public/assets/
|
16
|
+
public/ckeditor_assets/
|
17
|
+
public/uploads
|
18
|
+
vendor/bundle/
|
19
|
+
.rails_footnotes
|
20
|
+
.redcar/
|
21
|
+
doc/
|
22
|
+
vendor/
|
23
|
+
dump.rdb
|
24
|
+
|
25
|
+
coverage/
|
26
|
+
*.gem
|
27
|
+
|
28
|
+
seo_tools.sublime-workspace
|
data/.rspec
ADDED
data/.rubocop.yml
ADDED
data/Gemfile
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
source 'http://rubygems.org'
|
2
|
+
ruby '2.2.3'
|
3
|
+
|
4
|
+
gem 'rack', '~> 1.6.4'
|
5
|
+
# gem 'rack-contrib', '~> 1.4.0'
|
6
|
+
gem 'rake', '~> 10.4.2'
|
7
|
+
gem 'thin', '~> 1.6.3'
|
8
|
+
gem 'pg', '~> 0.18.3'
|
9
|
+
|
10
|
+
gem 'sinatra', '~> 1.4.6'
|
11
|
+
gem 'sinatra-param', '~> 1.4.0', require: 'sinatra/param'
|
12
|
+
|
13
|
+
# Needed for cli gem
|
14
|
+
gem 'slim', '~> 3.0.6'
|
15
|
+
|
16
|
+
gem 'httparty', '~> 0.13.5'
|
17
|
+
gem 'nokogiri', '~> 1.6.7.rc3'
|
18
|
+
gem 'geoip', '~> 1.6.1'
|
19
|
+
gem 'thor', '~> 0.19.1'
|
20
|
+
|
21
|
+
group :development do
|
22
|
+
end
|
23
|
+
|
24
|
+
group :test do
|
25
|
+
gem 'rspec', '~> 3.3.0'
|
26
|
+
gem 'rack-test', '~> 0.6.3'
|
27
|
+
gem 'fakeweb', '~> 1.3.0'
|
28
|
+
# gem 'fakefs', '~> 0.6.7'
|
29
|
+
gem 'simplecov', require: false
|
30
|
+
end
|
data/Gemfile.lock
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://rubygems.org/
|
3
|
+
specs:
|
4
|
+
byebug (5.0.0)
|
5
|
+
columnize (= 0.9.0)
|
6
|
+
coderay (1.1.0)
|
7
|
+
columnize (0.9.0)
|
8
|
+
daemons (1.2.3)
|
9
|
+
diff-lcs (1.2.5)
|
10
|
+
docile (1.1.5)
|
11
|
+
eventmachine (1.0.8)
|
12
|
+
fakeweb (1.3.0)
|
13
|
+
geoip (1.6.1)
|
14
|
+
httparty (0.13.5)
|
15
|
+
json (~> 1.8)
|
16
|
+
multi_xml (>= 0.5.2)
|
17
|
+
json (1.8.3)
|
18
|
+
method_source (0.8.2)
|
19
|
+
mini_portile (0.7.0.rc4)
|
20
|
+
multi_xml (0.5.5)
|
21
|
+
nokogiri (1.6.7.rc3)
|
22
|
+
mini_portile (~> 0.7.0.rc4)
|
23
|
+
nokogiri (1.6.7.rc3-x64-mingw32)
|
24
|
+
mini_portile (~> 0.7.0.rc4)
|
25
|
+
pg (0.18.3)
|
26
|
+
pg (0.18.3-x64-mingw32)
|
27
|
+
pry (0.10.1)
|
28
|
+
coderay (~> 1.1.0)
|
29
|
+
method_source (~> 0.8.1)
|
30
|
+
slop (~> 3.4)
|
31
|
+
pry (0.10.1-x64-mingw32)
|
32
|
+
coderay (~> 1.1.0)
|
33
|
+
method_source (~> 0.8.1)
|
34
|
+
slop (~> 3.4)
|
35
|
+
win32console (~> 1.3)
|
36
|
+
pry-byebug (3.2.0)
|
37
|
+
byebug (~> 5.0)
|
38
|
+
pry (~> 0.10)
|
39
|
+
rack (1.6.4)
|
40
|
+
rack-protection (1.5.3)
|
41
|
+
rack
|
42
|
+
rack-test (0.6.3)
|
43
|
+
rack (>= 1.0)
|
44
|
+
rake (10.4.2)
|
45
|
+
rspec (3.3.0)
|
46
|
+
rspec-core (~> 3.3.0)
|
47
|
+
rspec-expectations (~> 3.3.0)
|
48
|
+
rspec-mocks (~> 3.3.0)
|
49
|
+
rspec-core (3.3.2)
|
50
|
+
rspec-support (~> 3.3.0)
|
51
|
+
rspec-expectations (3.3.1)
|
52
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
53
|
+
rspec-support (~> 3.3.0)
|
54
|
+
rspec-mocks (3.3.2)
|
55
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
56
|
+
rspec-support (~> 3.3.0)
|
57
|
+
rspec-support (3.3.0)
|
58
|
+
simplecov (0.10.0)
|
59
|
+
docile (~> 1.1.0)
|
60
|
+
json (~> 1.8)
|
61
|
+
simplecov-html (~> 0.10.0)
|
62
|
+
simplecov-html (0.10.0)
|
63
|
+
sinatra (1.4.6)
|
64
|
+
rack (~> 1.4)
|
65
|
+
rack-protection (~> 1.4)
|
66
|
+
tilt (>= 1.3, < 3)
|
67
|
+
sinatra-param (1.4.0)
|
68
|
+
sinatra (~> 1.3)
|
69
|
+
slim (3.0.6)
|
70
|
+
temple (~> 0.7.3)
|
71
|
+
tilt (>= 1.3.3, < 2.1)
|
72
|
+
slop (3.6.0)
|
73
|
+
temple (0.7.6)
|
74
|
+
thin (1.6.3)
|
75
|
+
daemons (~> 1.0, >= 1.0.9)
|
76
|
+
eventmachine (~> 1.0)
|
77
|
+
rack (~> 1.0)
|
78
|
+
thor (0.19.1)
|
79
|
+
tilt (2.0.1)
|
80
|
+
win32console (1.3.2)
|
81
|
+
|
82
|
+
PLATFORMS
|
83
|
+
ruby
|
84
|
+
x64-mingw32
|
85
|
+
|
86
|
+
DEPENDENCIES
|
87
|
+
fakeweb (~> 1.3.0)
|
88
|
+
geoip (~> 1.6.1)
|
89
|
+
httparty (~> 0.13.5)
|
90
|
+
nokogiri (~> 1.6.7.rc3)
|
91
|
+
pg (~> 0.18.3)
|
92
|
+
pry
|
93
|
+
pry-byebug
|
94
|
+
rack (~> 1.6.4)
|
95
|
+
rack-test (~> 0.6.3)
|
96
|
+
rake (~> 10.4.2)
|
97
|
+
rspec (~> 3.3.0)
|
98
|
+
simplecov
|
99
|
+
sinatra (~> 1.4.6)
|
100
|
+
sinatra-param (~> 1.4.0)
|
101
|
+
slim (~> 3.0.6)
|
102
|
+
thin (~> 1.6.3)
|
103
|
+
thor (~> 0.19.1)
|
104
|
+
|
105
|
+
BUNDLED WITH
|
106
|
+
1.10.6
|
data/Procfile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
web: bundle exec rackup config.ru -p $PORT
|
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# Seo tools
|
2
|
+
|
3
|
+
Provides ability to scan selected pages for headers and all the links on page. Generating static html reports and serves them.
|
4
|
+
|
5
|
+
### Heroku link
|
6
|
+
|
7
|
+
http://rubyseoapp.herokuapp.com/
|
8
|
+
|
9
|
+
### Example
|
10
|
+
|
11
|
+
![Index image](https://raw.githubusercontent.com/Faulik/ruby_seo_tools/master/public/img/index_page.png)
|
12
|
+
|
13
|
+
![Report image](https://raw.githubusercontent.com/Faulik/ruby_seo_tools/master/public/img/report_page.png)
|
14
|
+
|
15
|
+
### Install
|
16
|
+
|
17
|
+
`bundle install --path vendor/bundle`
|
18
|
+
|
19
|
+
### Run
|
20
|
+
|
21
|
+
`rackup config.ru`
|
22
|
+
|
23
|
+
`bundle exec rackup config.ru`
|
24
|
+
|
25
|
+
### Test
|
26
|
+
|
27
|
+
`bundle exec rspec`
|
28
|
+
|
29
|
+
Released under the MIT license
|
data/Rakefile
ADDED
data/bin/seo_app
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require 'thor'
|
3
|
+
require 'seo_app'
|
4
|
+
require 'pry'
|
5
|
+
|
6
|
+
require 'pathname'
|
7
|
+
|
8
|
+
SeoApp.configure do |config|
|
9
|
+
config.database_type = 'files'
|
10
|
+
end
|
11
|
+
|
12
|
+
class Seo_app < Thor
|
13
|
+
desc 'parse url', 'Creates report html in current directory'
|
14
|
+
method_option :path, :aliases => '-p', :desc => 'Supply with path to look for'
|
15
|
+
def parse(_url)
|
16
|
+
puts "Parsing from url: #{_url}"
|
17
|
+
path_option options[:path]
|
18
|
+
puts "Saving in #{@path}"
|
19
|
+
|
20
|
+
SeoApp::LinkParser.new(_url).parse!
|
21
|
+
puts "Done!"
|
22
|
+
end
|
23
|
+
|
24
|
+
desc 'list [options]', ' get all reports in current folder or specific path'
|
25
|
+
method_option :path, :aliases => '-p', :desc => 'Supply with path to look for'
|
26
|
+
def list()
|
27
|
+
path_option options[:path]
|
28
|
+
puts "Getting all reports from #{@path}"
|
29
|
+
puts 'Site url -- date of parsing -- key'
|
30
|
+
SeoApp::Storage.all_reports.each do |report|
|
31
|
+
puts "#{report[:site_url]} -- #{report[:date]} -- #{report[:key]}"
|
32
|
+
end
|
33
|
+
puts "That's all"
|
34
|
+
end
|
35
|
+
|
36
|
+
no_commands do
|
37
|
+
def path_option(_path)
|
38
|
+
@path = _path ? File.join(Dir.pwd, _path) : Dir.pwd
|
39
|
+
SeoApp::Storage.database.reports_path = Pathname.new(@path)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
Seo_app.start
|
data/config.ru
ADDED
data/config/boot.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
|
2
|
+
|
3
|
+
require_relative 'boot'
|
4
|
+
|
5
|
+
ENV['RACK_ENV'] ||= 'development'
|
6
|
+
|
7
|
+
# If you have a Gemfile, require the gems listed there, including any gems
|
8
|
+
# you've limited to :test, :development, or :production.
|
9
|
+
Bundler.require(:default, ENV['RACK_ENV']) if defined?(Bundler)
|
10
|
+
|
11
|
+
require 'seo_app'
|
12
|
+
|
13
|
+
Dir["#{File.dirname(__FILE__)}/initializers/*.rb"].sort.each do |path|
|
14
|
+
require File.expand_path("../initializers/#{File.basename(path, '.rb')}", __FILE__)
|
15
|
+
end
|
data/db/GeoLiteCity.dat
ADDED
Binary file
|
data/lib/seo_app.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
|
3
|
+
require 'seo_app/configuration'
|
4
|
+
require 'seo_app/application'
|
5
|
+
|
6
|
+
# Config module
|
7
|
+
module SeoApp
|
8
|
+
# SeoApp.root_path.join('..')
|
9
|
+
#
|
10
|
+
def self.root_path
|
11
|
+
@root_path ||= Pathname.new(File.dirname(File.expand_path('../', __FILE__)))
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'sinatra'
|
2
|
+
require 'sinatra/param'
|
3
|
+
|
4
|
+
require_relative 'link_parser'
|
5
|
+
require_relative 'storages/storage'
|
6
|
+
|
7
|
+
module SeoApp
|
8
|
+
# Main routes
|
9
|
+
class Application < ::Sinatra::Application
|
10
|
+
# Configuration
|
11
|
+
set :public_folder, -> { SeoApp.root_path.join('public').to_s }
|
12
|
+
set :views, -> { SeoApp.root_path.join('views').to_s }
|
13
|
+
set :static, true
|
14
|
+
set :bind, '0.0.0.0'
|
15
|
+
|
16
|
+
# Helpers
|
17
|
+
helpers Sinatra::Param
|
18
|
+
|
19
|
+
# Middleware
|
20
|
+
use Rack::CommonLogger
|
21
|
+
use Rack::Reloader
|
22
|
+
|
23
|
+
get '/' do
|
24
|
+
param :error, String
|
25
|
+
|
26
|
+
@reports = SeoApp::Storage.all_reports
|
27
|
+
@error = params[:error] ? params[:error] : nil
|
28
|
+
|
29
|
+
slim :index
|
30
|
+
end
|
31
|
+
|
32
|
+
get '/reports/:key' do
|
33
|
+
param :key, String
|
34
|
+
|
35
|
+
SeoApp::Storage.report(params['key']) || redirect('/?error=NotFound')
|
36
|
+
end
|
37
|
+
|
38
|
+
post '/links' do
|
39
|
+
param :url, String, required: true
|
40
|
+
|
41
|
+
@link = LinkParser.new(params[:url])
|
42
|
+
|
43
|
+
redirect '/?error=NonValidURL' unless @link.parse!
|
44
|
+
redirect '/'
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# Config class
|
2
|
+
module SeoApp
|
3
|
+
class << self
|
4
|
+
attr_accessor :configuration
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.configure
|
8
|
+
self.configuration ||= Configuration.new
|
9
|
+
yield(configuration)
|
10
|
+
end
|
11
|
+
|
12
|
+
# Main confoguration class for app
|
13
|
+
class Configuration
|
14
|
+
attr_reader :database
|
15
|
+
|
16
|
+
# Files Storage
|
17
|
+
attr_accessor :reports_folder
|
18
|
+
|
19
|
+
# Db configuration
|
20
|
+
attr_accessor :db_host
|
21
|
+
attr_accessor :db_port
|
22
|
+
attr_accessor :db_name
|
23
|
+
attr_accessor :db_user
|
24
|
+
attr_accessor :db_password
|
25
|
+
|
26
|
+
def initialize
|
27
|
+
@reports_folder = '/public/reports/'
|
28
|
+
end
|
29
|
+
|
30
|
+
def database_type=(_type)
|
31
|
+
case _type
|
32
|
+
when 'postgres'
|
33
|
+
SeoApp::Storage.database = SeoApp::PgStorage.new
|
34
|
+
@database_type = 'postgres'
|
35
|
+
when 'files'
|
36
|
+
SeoApp::Storage.database = SeoApp::FileStorage.new
|
37
|
+
@database_type = 'files'
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
require 'httparty'
|
2
|
+
require 'nokogiri'
|
3
|
+
require 'slim'
|
4
|
+
require 'uri'
|
5
|
+
require 'geoip'
|
6
|
+
|
7
|
+
# Links parsing parts of app
|
8
|
+
module SeoApp
|
9
|
+
#
|
10
|
+
class LinkParser
|
11
|
+
attr_reader :links, :headers, :time, :geo, :url
|
12
|
+
|
13
|
+
def initialize(url)
|
14
|
+
@url = URI.parse(url)
|
15
|
+
@links = []
|
16
|
+
@headers = []
|
17
|
+
@geo = {}
|
18
|
+
@time = Time.now.strftime('%Y.%d.%m_%H-%M-%S')
|
19
|
+
end
|
20
|
+
|
21
|
+
def parse!
|
22
|
+
return false unless check_url
|
23
|
+
|
24
|
+
begin
|
25
|
+
_response = HTTParty.get @url
|
26
|
+
rescue
|
27
|
+
return false
|
28
|
+
end
|
29
|
+
|
30
|
+
procces_response _response
|
31
|
+
end
|
32
|
+
|
33
|
+
def procces_response(response)
|
34
|
+
@headers = response.headers
|
35
|
+
@geo = retrieve_geo_params
|
36
|
+
gather_links! response.body
|
37
|
+
|
38
|
+
SeoApp::Storage.save_report(generate_report, generate_file_name)
|
39
|
+
end
|
40
|
+
|
41
|
+
def gather_links!(body)
|
42
|
+
::Nokogiri::HTML(body).css('a').each do |link|
|
43
|
+
@links << {
|
44
|
+
name: link.text || 'None',
|
45
|
+
url: link['href'] || 'None',
|
46
|
+
rel: link['rel'] || 'None',
|
47
|
+
target: link['target'] || 'None'
|
48
|
+
}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def generate_file_name
|
53
|
+
"#{@url.host}__#{@time}.html"
|
54
|
+
end
|
55
|
+
|
56
|
+
def generate_report
|
57
|
+
Slim::Template.new('views/reports.slim').render(self)
|
58
|
+
end
|
59
|
+
|
60
|
+
def retrieve_geo_params
|
61
|
+
GeoIP.new(SeoApp.root_path.join('db/GeoLiteCity.dat')).city(@url.host.to_s)
|
62
|
+
end
|
63
|
+
|
64
|
+
def check_url
|
65
|
+
@url = !@url.scheme ? URI.parse('http://' + @url.to_s) : @url
|
66
|
+
@url.is_a?(URI::HTTP) ? true : false
|
67
|
+
end
|
68
|
+
|
69
|
+
def self.say_hello
|
70
|
+
puts 'Hello'
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|