just_crawl 1.1.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 97e1b466ee0a52002c3e35a3f5ed622e5fa7208761dd831fe695c9b3104ed7db
+  data.tar.gz: 4ded6f28ab8e36e144969fba44f7d017fd0405fa2bece4efeb45f67e9b5985d5
+SHA512:
+  metadata.gz: b0f0ea51add4876a3b397abb74c0b6a0d4c21585eae7ecdf58069178f38396c6ec9022f96af42dbc69ecb50755d7237d9a533c42777cc6cf9debd977e9362c1e
+  data.tar.gz: 16a5673b5aa8d7d727de7f4cfbdedb05abbd9dab576738f1bf53ed04494a842626f04a86e0ff59d8515879dbe7225938479fd0b3a7927991964f853d4f5bebba

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+spec/examples.txt
+tmp
+.DS_Store
+.idea

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+2.5.1

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 AlphaSights
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,53 @@
+# JustCrawl
+
+JustCrawl crawls pages within a domain, reporting any page that returns a bad response code
+
+Usage:
+
+    > just_crawl [options] domain
+
+    Usage: just_crawl [options] domain
+    -s, --start /home,/about         Starting path(s), defaults to /
+    -u, --username username          Basic auth username
+    -p, --password password          Basic auth password
+    -c, --connections count          Max mumber of parallel connections to use. The default is 5.
+    -v, --verbose                    Give details when crawling
+    -h, --help                       Show this message
+        --version                    Print version
+
+
+
+Example:
+
+    > just_crawl https://vaskohandmade.com --connections=5 --start=/ --verbose
+
+      Adding /
+    Fetching / ...
+      Adding index.html
+      Adding assets/custom/images/vasko/portfolio/reverb-con-modulo-belton.jpg
+      Adding assets/custom/images/vasko/portfolio/amplificador-fender-champ.jpg
+      Adding assets/custom/images/vasko/portfolio/overdrive-simil-zendrive.jpg
+      Adding assets/custom/images/vasko/portfolio/booster-simil-super-hardon.jpg
+      Adding assets/custom/images/vasko/portfolio/amplificador-valvular.jpg
+      Adding assets/custom/images/vasko/portfolio/rehousing-fender-superchamp.jpg
+      Adding assets/custom/images/vasko/portfolio/fuzz-simil-big-muff-violet-ram.jpg
+      Adding assets/custom/images/vasko/portfolio/amplificador-valvular-simil-marshall-1974x.jpg
+      Adding assets/custom/images/vasko/portfolio/distorsion-high-gain-simil-triple-wreck.jpg
+      Adding assets/custom/images/vasko/portfolio/booster-simil-rc-booster.jpg
+      Adding assets/custom/images/vasko/portfolio/amplificador-blues-mojo-7w-valvular.jpg
+      Adding assets/custom/images/vasko/portfolio/rehousing-fender-super-champ.jpg
+    Fetching index.html ...
+    Fetching assets/custom/images/vasko/portfolio/reverb-con-modulo-belton.jpg ...
+    Fetching assets/custom/images/vasko/portfolio/amplificador-fender-champ.jpg ...
+    Fetching assets/custom/images/vasko/portfolio/overdrive-simil-zendrive.jpg ...
+    Fetching assets/custom/images/vasko/portfolio/booster-simil-super-hardon.jpg ...
+    Fetching assets/custom/images/vasko/portfolio/amplificador-valvular.jpg ...
+    Fetching assets/custom/images/vasko/portfolio/rehousing-fender-superchamp.jpg ...
+    Fetching assets/custom/images/vasko/portfolio/fuzz-simil-big-muff-violet-ram.jpg ...
+    Fetching assets/custom/images/vasko/portfolio/amplificador-valvular-simil-marshall-1974x.jpg ...
+    Fetching assets/custom/images/vasko/portfolio/distorsion-high-gain-simil-triple-wreck.jpg ...
+    Fetching assets/custom/images/vasko/portfolio/booster-simil-rc-booster.jpg ...
+    Fetching assets/custom/images/vasko/portfolio/amplificador-blues-mojo-7w-valvular.jpg ...
+    Fetching assets/custom/images/vasko/portfolio/rehousing-fender-super-champ.jpg ...
+     
+    14 pages crawled without errors.

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/bin/just_crawl

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require_relative '../lib/just_crawl.rb'
+
+EM.threadpool_size = 5
+
+options = {}
+optparse = OptionParser.new do |opts|
+  opts.banner = "JustCrawl pages within a domain, reporting any page that returns a bad response code\nUsage: just_crawl [options] domain"
+  opts.on('-s', '--start /home,/about', Array, 'Starting path(s), defaults to /') { |o| options[:start] = o }
+  opts.on('-u', '--username username', String, 'Basic auth username') { |o| options[:username] = o }
+  opts.on('-p', '--password password', String, 'Basic auth password') { |o| options[:password] = o }
+  opts.on('-c', '--connections count', Integer, "Max mumber of parallel connections to use. The default is #{EM.threadpool_size}.") { |o| EM.threadpool_size = o }
+  opts.on('-v', '--verbose', 'Give details when crawling') { |o| $verbose = o }
+  opts.on_tail('-h', '--help', 'Show this message') { |o| puts opts; exit }
+  opts.on_tail('-v', '--version', 'Print version') { |o| puts JustCrawl::VERSION; exit }
+end.parse!
+
+options.merge!(domain: optparse.first)
+
+unless options[:domain]
+  puts 'Must provide a domain'
+  exit -1
+end
+
+options[:domain] = "http://#{options[:domain]}" unless options[:domain].include?('://')
+
+crawler = JustCrawl::Engine.new(options)
+
+trap('SIGINT') do
+  puts "\n\nAborting just_crawl.."
+  crawler.summarize
+  exit -1
+end
+
+crawler.run
+crawler.summarize
+
+exit -1 if crawler.errors? || crawler.no_links_found?

data/circle.yml

@@ -0,0 +1,3 @@
+test:
+  override:
+    - RAILS_ENV=test bundle exec rspec -r rspec_junit_formatter --format RspecJunitFormatter -o $CIRCLE_TEST_REPORTS/rspec/junit.xml

data/just_crawl.gemspec

@@ -0,0 +1,26 @@
+require File.expand_path('../lib/just_crawl/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ['Nicolas Sebastian Vidal']
+  gem.email         = ['nicolas.s.vidal@gmail.com']
+  gem.description   = 'JustCrawl crawls all pages on a domain, checking for errors'
+  gem.summary       = 'JustCrawl crawls pages within a domain, reporting any page that returns a bad response code'
+  gem.homepage      = 'http://github.com/nisevi/just_crawl'
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = 'just_crawl'
+  gem.require_paths = ['lib']
+  gem.required_ruby_version = '>= 2.5.1'
+  gem.version       = JustCrawl::VERSION
+  gem.licenses      = ['MIT']
+  gem.add_dependency 'em-http-request', '~> 1.1', '>= 1.1.5'
+  gem.add_dependency 'eventmachine', '~> 1.2', '>= 1.2.6'
+  gem.add_dependency 'nokogiri', '~> 1.8', '>= 1.8.2'
+  gem.add_dependency 'rest-client', '~> 2.0', '>= 2.0.2'
+  gem.add_development_dependency 'rspec-core', '~> 3.7', '>= 3.7.1'
+  gem.add_development_dependency 'rspec-expectations', '~> 3.7'
+  gem.add_development_dependency 'rspec_junit_formatter', '~> 0.3.0'
+  gem.add_development_dependency 'simplecov', '~> 0.16.1'
+end

data/lib/just_crawl/engine.rb

@@ -0,0 +1,116 @@
+class JustCrawl::Engine
+  DEFAULT_OPTIONS = {
+    domain:  '',
+    start: ['/'],
+    username: '',
+    password: '',
+    verbose: false,
+    session_id: false
+  }.freeze
+
+  IGNORE = [/#/, /mailto:/, /skype:/, /logout/, /javascript:/, %r{/xhr/}, /https:/, /\.pdf$/, /^$/, /tel:/].freeze
+  VALID_RESPONSE_CODES = [200, 302].freeze
+  MAX_REDIRECTS = 3
+  LINE_WIDTH = 78
+
+  attr_reader :options
+
+  def initialize(caller_options = {})
+    @options = DEFAULT_OPTIONS.merge(caller_options)
+    @authorization = Base64.encode64("#{options[:username]}:#{options[:password]}")
+    @register = JustCrawl::Register.new
+
+    start_pages = options[:start].to_a.map { |page| Page.new(@register, page, '/') }
+
+    @register.add(start_pages)
+  end
+
+  def run
+    EventMachine.run do
+      process_next
+    end
+  end
+
+  def process_next
+    return if @register.processing_size >= EM.threadpool_size
+    if @register.finished?
+      EventMachine.stop
+    elsif (page = @register.next_page)
+      retrieve(page)
+      process_next
+    end
+  end
+
+  def summarize
+    @register.summarize
+  end
+
+  def errors?
+    @register.errors?
+  end
+
+  def no_links_found?
+    @register.no_links_found?
+  end
+
+  private
+
+  def retrieve(page)
+    puts "Fetching #{page.url} ..." if $verbose
+
+    absolute_url = options[:domain] + page.relative_url
+
+    http = EventMachine::HttpRequest.new(absolute_url)
+    req = http.get redirects: MAX_REDIRECTS,
+                   connect_timeout: 20,
+                   inactivity_timeout: 20,
+                   head: {
+                     'authorization' => [
+                       options[:username], options[:password]
+                     ]
+                   }
+    req.errback do
+      if req.nil?
+        page.intermittent('Req is nil. WAT?')
+      elsif msg = req.error
+        page.intermittent(msg)
+      elsif req.response.nil? || req.response.empty?
+        page.intermittent('Timeout?')
+      else
+        page.intermittent('Partial response: Server Broke Connection?')
+      end
+      process_next
+    end
+
+    req.callback do
+      status_code = req.response_header.status
+      if VALID_RESPONSE_CODES.include?(status_code)
+        page.success
+        if req.response_header['CONTENT_TYPE'] =~ %r{text/html}
+          @register.add find_linked_pages(page, req.response.to_str)
+        end
+      elsif status_code == 503
+        page.intermittent('Status code: 503')
+      else
+        page.fatal("Status code: #{status_code}")
+      end
+      process_next
+    end
+  end
+
+  def find_linked_pages(page, body)
+    doc = Nokogiri::HTML(body)
+    anchors = doc.css('a').to_a
+    anchors.reject! { |anchor| anchor['onclick'].to_s =~ /f.method = 'POST'/ }
+    anchors.reject! { |anchor| anchor['data-method'] =~ /put|post|delete/ }
+    anchors.reject! { |anchor| anchor['data-remote'] =~ /true/ }
+    anchors.reject! { |anchor| anchor['class'].to_s =~ /unobtrusive_/ }
+    anchors.reject! { |anchor| anchor['rel'].to_s =~ /nofollow/ }
+    raw_links = anchors.map { |anchor| anchor['href'] }
+    raw_links.compact!
+    raw_links.map! { |link| link.sub(options[:domain], '') }
+    raw_links.delete_if { |link| link =~ %r{^http(s)?://} && !link.include?(options[:domain]) }
+    raw_links.delete_if { |link| IGNORE.any? { |pattern| link =~ pattern } }
+    raw_links.map { |url| Page.new(@register, url, page.url) }
+  end
+end

data/lib/just_crawl/failure.rb

@@ -0,0 +1,29 @@
+class JustCrawl::Failure
+  attr_reader :link, :code, :from
+
+  def initialize(link, code, from)
+    @link = link
+    @code = code
+    @from = from
+  end
+
+  def failure?
+    true
+  end
+
+  def error?
+    !failure?
+  end
+
+  def name
+    link
+  end
+
+  def message
+    "Status code was #{code}"
+  end
+
+  def location
+    "Linked from #{from}"
+  end
+end

data/lib/just_crawl/page.rb

@@ -0,0 +1,59 @@
+require 'uri'
+
+class Page
+  include Comparable
+
+  attr_reader :register, :url, :source, :error
+
+  ATTEMPTS = 3
+
+  def initialize(register, url, source)
+    @register = register
+    @url = url
+    @source = source
+    @attempts = 0
+    @errors = nil
+  end
+
+  def relative_url
+    @relative_url ||= URI.join('http://example.com', source, url).path
+  end
+
+  def <=>(other)
+    relative_url <=> other.relative_url
+  end
+
+  def eql?(other)
+    relative_url.eql?(other.relative_url)
+  end
+
+  def hash
+    relative_url.hash
+  end
+
+  def success
+    @error = nil
+    @register.completed(self)
+  end
+
+  def fatal(error)
+    puts "  Fatal - #{error}" if $VERBOSE
+    @error = error
+    @register.completed(self)
+  end
+
+  def intermittent(error)
+    puts "  Intermittent - #{error}" if $VERBOSE
+    if @attempts >= ATTEMPTS
+      @error = error
+      @register.completed(self)
+    else
+      @attempts += 1
+      @register.retry(self)
+    end
+  end
+
+  def to_s
+    "#{url} found on #{source} - #{error || 'OK'}"
+  end
+end

data/lib/just_crawl/register.rb

@@ -0,0 +1,69 @@
+class JustCrawl::Register
+
+  Result = Struct.new(:url, :object)
+
+  def initialize
+    @unprocessed = Set.new
+    @processing = Set.new
+    @processed = Set.new
+  end
+
+  def add(pages)
+    new_pages = pages.to_set - @processed - @processing - @unprocessed
+    new_pages.each do |new_page|
+      puts "  Adding #{new_page.url}" if $verbose
+    end
+    @unprocessed.merge(new_pages)
+  end
+
+  def next_page
+    page = @unprocessed.first
+    @unprocessed.delete(page)
+    @processing << page if page
+    if @processing.size > EM.threadpool_size
+      puts "WARNING: #{@processing.size} pages are being process when EM threadpool only has #{EM.threadpool_size} threads."
+    end
+    page
+  end
+
+  def retry(page)
+    @unprocessed << page
+    @processing.delete(page)
+  end
+
+  def completed(page)
+    @processed << page
+    @processing.delete(page)
+  end
+
+  def finished?
+    (@unprocessed.size + @processing.size).zero?
+  end
+
+  def processing_size
+    @processing.size
+  end
+
+  def error_pages
+    @processed.select(&:error)
+  end
+
+  def errors?
+    !error_pages.empty?
+  end
+
+  def summarize
+    if errors?
+      puts "\nPages with errors:"
+      error_pages.each do |page|
+        puts page.to_s
+      end
+    else
+      puts "\n#{@processed.size} pages crawled without errors."
+    end
+  end
+
+  def no_links_found?
+    @processed.size <= 1
+  end
+end

data/lib/just_crawl/string.rb

@@ -0,0 +1,7 @@
+class String
+  def word_wrap(line_width = 80)
+    split("\n").collect do |line|
+      line.length > line_width ? line.gsub(/(.{1,#{line_width}})(\s+|$)/, "\\1\n").strip : line
+    end * "\n"
+  end
+end

data/lib/just_crawl/version.rb

@@ -0,0 +1,3 @@
+module JustCrawl
+  VERSION = '1.1.8'.freeze
+end

data/lib/just_crawl.rb

@@ -0,0 +1,18 @@
+require 'nokogiri'
+require 'rest_client'
+require 'eventmachine'
+require 'em-http-request'
+require 'base64'
+require 'set'
+require 'fileutils'
+require 'digest/sha1'
+require 'json'
+require 'tempfile'
+require 'tmpdir'
+
+require_relative 'just_crawl/version'
+require_relative 'just_crawl/engine'
+require_relative 'just_crawl/string'
+require_relative 'just_crawl/failure'
+require_relative 'just_crawl/register'
+require_relative 'just_crawl/page'

data/spec/lib/just_crawl/engine_spec.rb

@@ -0,0 +1,9 @@
+require_relative '../../spec_helper'
+
+RSpec.describe 'JustCrawl::Engine' do
+  xit '#initialize'
+  xit '#run'
+  xit '#summarize'
+  xit '#errors?'
+  xit '#no_links_found?'
+end

data/spec/lib/just_crawl/failure_spec.rb

@@ -0,0 +1,10 @@
+require_relative '../../spec_helper'
+
+RSpec.describe 'JustCrawl::Failure' do
+  xit '#initialize'
+  xit '#failure?'
+  xit '#error?'
+  xit '#name'
+  xit '#message'
+  xit '#location'
+end

data/spec/lib/just_crawl/page_spec.rb

@@ -0,0 +1,24 @@
+require_relative '../../spec_helper'
+require './lib/just_crawl/page'
+
+RSpec.describe Page do
+  xit 'initialize'
+
+  it '#relative_url' do
+    expect(Page.new(:register, '/', '/').relative_url).to eq '/'
+    expect(Page.new(:register, './', '/').relative_url).to eq '/'
+    expect(Page.new(:register, 'page.html', '').relative_url).to eq '/page.html'
+    expect(Page.new(:register, '/interview', '/').relative_url).to eq '/interview'
+    expect(Page.new(:register, 'overview.html', '/').relative_url).to eq '/overview.html'
+    expect(Page.new(:register, 'post-5.html', '/posts/index.html').relative_url).to eq '/posts/post-5.html'
+    expect(Page.new(:register, 'https://staging.alphasights.com/careers/meet-us', '/posts/foo').relative_url).to eq '/careers/meet-us'
+  end
+
+  xit '#<=>'
+  xit '#eql?'
+  xit '#hash'
+  xit '#success'
+  xit '#fatal'
+  xit '#intermittent'
+  xit '#to_s'
+end

data/spec/lib/just_crawl/register_spec.rb

@@ -0,0 +1,15 @@
+require_relative '../../spec_helper'
+
+RSpec.describe 'JustCrawl::Register' do
+  xit '#initialize'
+  xit '#add'
+  xit '#next_page'
+  xit '#retry'
+  xit '#completed'
+  xit '#finished?'
+  xit '#processing_size'
+  xit '#error_pages'
+  xit '#errors?'
+  xit '#summarize'
+  xit '#no_links_found?'
+end

data/spec/lib/just_crawl/string_spec.rb

@@ -0,0 +1,5 @@
+require_relative '../../spec_helper'
+
+RSpec.describe String do
+  xit '#word_wrap'
+end

data/spec/lib/just_crawl/version_spec.rb

@@ -0,0 +1,8 @@
+require_relative '../../spec_helper'
+require_relative '../../../lib/just_crawl/version'
+
+RSpec.describe 'Gem version' do
+  it 'should match the current gem version' do
+    expect(JustCrawl::VERSION).to eq '1.1.8'
+  end
+end

data/spec/lib/just_crawl_spec.rb

@@ -0,0 +1,5 @@
+require_relative '../spec_helper'
+
+RSpec.describe 'JustCrawl require files.' do
+  xit 'should load all required files'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,18 @@
+require 'simplecov'
+SimpleCov.start
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.default_formatter = 'doc' if config.files_to_run.one?
+
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+  config.filter_run_when_matching :focus
+  config.example_status_persistence_file_path = 'spec/examples.txt'
+  config.disable_monkey_patching!
+  config.warnings = true
+  config.order = :random
+  Kernel.srand config.seed
+end

metadata

@@ -0,0 +1,213 @@
+--- !ruby/object:Gem::Specification
+name: just_crawl
+version: !ruby/object:Gem::Version
+  version: 1.1.8
+platform: ruby
+authors:
+- Nicolas Sebastian Vidal
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-05-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.5
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.1
+- !ruby/object:Gem::Dependency
+  name: rspec-expectations
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.1
+description: JustCrawl crawls all pages on a domain, checking for errors
+email:
+- nicolas.s.vidal@gmail.com
+executables:
+- just_crawl
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/just_crawl
+- circle.yml
+- just_crawl.gemspec
+- lib/just_crawl.rb
+- lib/just_crawl/engine.rb
+- lib/just_crawl/failure.rb
+- lib/just_crawl/page.rb
+- lib/just_crawl/register.rb
+- lib/just_crawl/string.rb
+- lib/just_crawl/version.rb
+- spec/lib/just_crawl/engine_spec.rb
+- spec/lib/just_crawl/failure_spec.rb
+- spec/lib/just_crawl/page_spec.rb
+- spec/lib/just_crawl/register_spec.rb
+- spec/lib/just_crawl/string_spec.rb
+- spec/lib/just_crawl/version_spec.rb
+- spec/lib/just_crawl_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/nisevi/just_crawl
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: JustCrawl crawls pages within a domain, reporting any page that returns a
+  bad response code
+test_files: []