kleos_test 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7cdb0602baff97def0b5d1325c7d59221d966871
+  data.tar.gz: e203bfa4a9c52005eb6278e75362026d3affeb5d
+SHA512:
+  metadata.gz: b9a81643e49f042a013da0936c9dce670d888dee3909d903f5c37624e297e4c72e073b1a7cc7eccf4eff0a807051258ba656175090ea8746c5c43668d3dbc23f
+  data.tar.gz: f97a3373ee5241a0399a9a42a068e7783a93895c207ead7769b91d89ebeb79fb649406a770e226d02e362b34bc877a2449ef8f52476c2a5ad1e2d006e4f58847

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'pry'
+gem 'rest-client'
+gem 'nokogiri'

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Akkuzin Maxim
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# KleosTest
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/kleos_test`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'kleos_test'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install kleos_test
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/kleos_test.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "kleos_test"
+require "pry"
+
+Pry.start

data/bin/kleos_test

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+require 'pry'
+$LOAD_PATH << File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'kleos_test'
+
+puts "*** Check for inside links ***\n\n"
+begin
+  collector = KleosTest::LinksCollector.new
+  collector.get_new_links
+end while KleosTest::LinksCollector.unverified_inside_links.size > 0
+
+puts "\n\n*** Check for outside links ***\n\n"
+collector = KleosTest::LinksCollector.new
+collector.verify_outside_links
+
+broken_inside = KleosTest::LinksCollector.invalid_inside_links
+broken_outside = KleosTest::LinksCollector.invalid_outside_links
+
+result = <<EOF
+*** Result: ***
+Total count of inside links:  #{KleosTest::LinksCollector.inside_links.count}
+- valid:   #{KleosTest::LinksCollector.valid_inside_links.count}
+- invalid: #{broken_inside.count}
+Total count of outside links: #{KleosTest::LinksCollector.outside_links.count}
+- valid:   #{KleosTest::LinksCollector.valid_outside_links.count}
+- invalid: #{broken_outside.count}
+---------------
+Broken links:
+EOF
+
+if broken_inside.empty? and broken_outside.empty?
+  result << "none.\n"
+elsif broken_inside.empty?
+  result << "- inside: none\n"
+  result << broken_outside.join("\n")
+elsif broken_outside.empty?
+  result << broken_inside.join("\n")
+  result << "- outside: none\n"
+end
+
+puts result

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/kleos_test.gemspec

@@ -0,0 +1,23 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kleos_test/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "kleos_test"
+  spec.version       = KleosTest::VERSION
+  spec.authors       = ["Akkuzin Maxim"]
+  spec.email         = ["akkmaxon2307@gmail.com"]
+
+  spec.summary       = "Test task from kleos.ru"
+  spec.description   = "Looking for broken links on kleos.ru"
+  spec.homepage      = "https://github.com/makkuzin/kleos_test"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.bindir        = "bin"
+  spec.executables   = ["kleos_test"]
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/kleos_test/links_collector.rb

@@ -0,0 +1,124 @@
+class KleosTest::LinksCollector
+  BASE_ADDRESS = 'https://www.kleos.ru'
+
+  @@unverified_inside_links = ['/']
+  @@unverified_outside_links = []
+  @@verified_links = []
+  @@inside_links = { valid: [], invalid: [] }
+  @@outside_links = { valid: [], invalid: [] }
+  @@inside_downloads = 0
+
+  def initialize
+    @target = get_target_link
+  end
+
+  def get_target_link
+    @@unverified_inside_links.empty? ? '/' : @@unverified_inside_links.shift
+  end
+
+  def get_new_links
+    page, response_code = download_inside_webpage
+    if response_code == 200
+      @@inside_links[:valid] << @target
+      @@verified_links << @target
+      links = extract_links(page)
+      refill_unverified_links(links)
+    else
+      @@inside_links[:invalid] << @target
+      @@verified_links << @target
+    end
+  end
+
+  def verify_outside_links
+    @@unverified_outside_links.each_with_index do |link, i|
+      code = download_outside_webpage(link, i + 1)
+      if code == 200
+        @@outside_links[:valid] << link
+      else
+        @@outside_links[:invalid] << link
+      end
+    end
+  end
+
+  def download_inside_webpage
+    address = @target.include?('http') ? @target : BASE_ADDRESS + @target
+    @@inside_downloads += 1
+    print "Downloading (#{@@inside_downloads}|#{@@unverified_inside_links.size})\
+ #{address}..."
+    response = RestClient.get(address)
+    puts "OK"
+    [response.body, response.code]
+  rescue RestClient::ExceptionWithResponse => e
+    puts "ERROR"
+    [e.response.body, e.response.code]
+  rescue URI::InvalidURIError
+    puts "ERROR"
+    puts "BAD URI"
+    ['fake body', 1]
+  end
+
+  def download_outside_webpage(address, counter)
+    print "Downloading (#{counter}|#{@@unverified_outside_links.size - counter})\
+ #{address}..."
+    response = RestClient.get(address)
+    puts "OK"
+    response.code
+  rescue RestClient::ExceptionWithResponse => e
+    puts "ERROR"
+    e.response.code
+  rescue URI::InvalidURIError
+    puts "ERROR"
+    puts "BAD URI"
+    1
+  end
+
+  def extract_links(page)
+    query = '//a[@href!=""]'
+    query += '[not(starts-with(@href, "javascript:void(0)"))]'
+    query += '[not(starts-with(@href, "#"))]'
+    query += '[not(starts-with(@href, "mailto"))]'
+    Nokogiri::HTML(page).xpath(query).map { |link| link.attribute('href').value }
+  end
+
+  def refill_unverified_links(links)
+    regexp = /^(\/|\?|https?:\/\/(www\.)?kleos\.ru)(?!\/forum)/
+    @@unverified_inside_links.concat(links.select do |link|
+      link.match(regexp) && !(
+        @@unverified_inside_links.include?(link) ||
+        @@verified_links.include?(link))
+    end.uniq)
+
+    @@unverified_outside_links.concat(
+      links.reject { |link| link.match(regexp) }).uniq!
+  end
+
+  class << self
+    def unverified_inside_links
+      @@unverified_inside_links
+    end
+
+    def inside_links
+      @@inside_links[:valid] | @@inside_links[:invalid]
+    end
+
+    def valid_inside_links
+      @@inside_links[:valid]
+    end
+
+    def invalid_inside_links
+      @@inside_links[:invalid]
+    end
+
+    def outside_links
+      @@outside_links[:valid] | @@outside_links[:invalid]
+    end
+
+    def valid_outside_links
+      @@outside_links[:valid]
+    end
+
+    def invalid_outside_links
+      @@outside_links[:invalid]
+    end
+  end
+end

data/lib/kleos_test/version.rb

@@ -0,0 +1,3 @@
+module KleosTest
+  VERSION = "0.1.0"
+end

data/lib/kleos_test.rb

@@ -0,0 +1,7 @@
+require 'rest-client'
+require 'nokogiri'
+require 'kleos_test/version'
+require 'kleos_test/links_collector'
+
+module KleosTest
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: kleos_test
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Akkuzin Maxim
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-01-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Looking for broken links on kleos.ru
+email:
+- akkmaxon2307@gmail.com
+executables:
+- kleos_test
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/kleos_test
+- bin/setup
+- kleos_test.gemspec
+- lib/kleos_test.rb
+- lib/kleos_test/links_collector.rb
+- lib/kleos_test/version.rb
+homepage: https://github.com/makkuzin/kleos_test
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Test task from kleos.ru
+test_files: []