iron-crawler 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8a2dc8e70c7883e03bbc90ec200e0f80ef59d059
+  data.tar.gz: a348eb768509de52f3406d1fa23d52ddea636c62
+SHA512:
+  metadata.gz: b5f54a41ccbdadccecf7d522dcd81c0a13e424e3f3064f9aa9e45775e1f597b0b073d0ae6212d7cc163db3db4bd238031f13e4a86801d35eb84c6cc113f8440c
+  data.tar.gz: 522d5666aa95be4477ffb6d4922606fe6c02fe4f5afec4adacd5338ae36ea641f5bd852c68fe04188a6e590537c10e2b9b16a1ac3ee05390b24a1fde1a1516f3

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,20 @@
+source 'http://rubygems.org'
+
+gem 'mechanize'
+
+group :development do
+  gem 'shoulda', '>= 0'
+  gem 'rdoc', '~> 3.12'
+  gem 'bundler', '~> 1.0'
+  gem 'jeweler', '~> 2.0.1'
+  gem 'simplecov', '>= 0'
+  gem 'rspec', '~> 3.4.0'
+  gem "reek", "~> 3.10.0"
+  gem "roodi", "~> 5.0.0"
+  gem "yard", "~> 0.8.0"
+  gem "churn", "~> 1.0.2"
+  gem "flay", "~> 2.7.0"
+  gem "flog", "~> 4.3.2"
+  gem "inch", "~> 0.7.0"
+  gem "rubocop", "~> 0.37.0"
+end

data/Gemfile.lock

@@ -0,0 +1,215 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (4.2.5.1)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.4.0)
+    arrayfields (4.9.2)
+    ast (2.2.0)
+    axiom-types (0.1.1)
+      descendants_tracker (~> 0.0.4)
+      ice_nine (~> 0.11.0)
+      thread_safe (~> 0.3, >= 0.3.1)
+    builder (3.2.2)
+    chronic (0.10.2)
+    churn (1.0.2)
+      chronic (>= 0.2.3)
+      hirb
+      json_pure
+      main
+      rest-client (>= 1.6.0)
+      ruby_parser (~> 3.0)
+      sexp_processor (~> 4.1)
+    codeclimate-engine-rb (0.3.1)
+      virtus (~> 1.0)
+    coderay (1.1.0)
+    coercible (1.0.0)
+      descendants_tracker (~> 0.0.1)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    domain_name (0.5.20160128)
+      unf (>= 0.0.5, < 1.0.0)
+    equalizer (0.0.11)
+    erubis (2.7.0)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    fattr (2.2.2)
+    flay (2.7.0)
+      erubis (~> 2.7.0)
+      ruby_parser (~> 3.0)
+      sexp_processor (~> 4.0)
+    flog (4.3.2)
+      ruby_parser (~> 3.1, > 3.1.0)
+      sexp_processor (~> 4.4)
+    git (1.2.9.1)
+    github_api (0.13.1)
+      addressable (~> 2.4.0)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.4)
+      multi_json (>= 1.7.5, < 2.0)
+      oauth2
+    hashie (3.4.3)
+    highline (1.7.8)
+    hirb (0.7.3)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    i18n (0.7.0)
+    ice_nine (0.11.2)
+    inch (0.7.0)
+      pry
+      sparkr (>= 0.2.0)
+      term-ansicolor
+      yard (~> 0.8.7.5)
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+    json (1.8.3)
+    json_pure (1.8.3)
+    jwt (1.5.1)
+    main (6.2.0)
+      arrayfields (>= 4.7.4)
+      chronic (>= 0.6.2)
+      fattr (>= 2.2.0)
+      map (>= 6.1.0)
+    map (6.6.0)
+    mechanize (2.7.4)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0)
+      mime-types (>= 1.17.2, < 3)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.6)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
+    method_source (0.8.2)
+    mime-types (2.99)
+    mini_portile2 (2.0.0)
+    minitest (5.8.4)
+    multi_json (1.11.2)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    net-http-digest_auth (1.4)
+    net-http-persistent (2.9.4)
+    netrc (0.11.0)
+    nokogiri (1.6.7.2)
+      mini_portile2 (~> 2.0.0.rc2)
+    ntlm-http (0.1.1)
+    oauth2 (1.1.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0, < 1.5.2)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (>= 1.2, < 3)
+    parser (2.3.0.3)
+      ast (~> 2.2)
+    powerpack (0.1.1)
+    private_attr (1.1.0)
+    pry (0.10.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rack (1.6.4)
+    rainbow (2.1.0)
+    rake (10.5.0)
+    rdoc (3.12.2)
+      json (~> 1.4)
+    reek (3.10.0)
+      codeclimate-engine-rb (~> 0.3.1)
+      parser (~> 2.3)
+      private_attr (~> 1.1)
+      rainbow (~> 2.0)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    roodi (5.0.0)
+      ruby_parser (~> 3.2, >= 3.2.2)
+    rspec (3.4.0)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+    rspec-core (3.4.2)
+      rspec-support (~> 3.4.0)
+    rspec-expectations (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-mocks (3.4.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-support (3.4.1)
+    rubocop (0.37.0)
+      parser (>= 2.3.0.2, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 0.3)
+    ruby-progressbar (1.7.5)
+    ruby_parser (3.7.3)
+      sexp_processor (~> 4.1)
+    sexp_processor (4.6.1)
+    shoulda (3.5.0)
+      shoulda-context (~> 1.0, >= 1.0.1)
+      shoulda-matchers (>= 1.4.1, < 3.0)
+    shoulda-context (1.2.1)
+    shoulda-matchers (2.8.0)
+      activesupport (>= 3.0.0)
+    simplecov (0.11.2)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    slop (3.6.0)
+    sparkr (0.4.1)
+    term-ansicolor (1.3.2)
+      tins (~> 1.0)
+    thread_safe (0.3.5)
+    tins (1.8.2)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.2)
+    unicode-display_width (0.3.1)
+    virtus (1.0.5)
+      axiom-types (~> 0.1)
+      coercible (~> 1.0)
+      descendants_tracker (~> 0.0, >= 0.0.3)
+      equalizer (~> 0.0, >= 0.0.9)
+    webrobots (0.1.2)
+    yard (0.8.7.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  churn (~> 1.0.2)
+  flay (~> 2.7.0)
+  flog (~> 4.3.2)
+  inch (~> 0.7.0)
+  jeweler (~> 2.0.1)
+  mechanize
+  rdoc (~> 3.12)
+  reek (~> 3.10.0)
+  roodi (~> 5.0.0)
+  rspec (~> 3.4.0)
+  rubocop (~> 0.37.0)
+  shoulda
+  simplecov
+  yard (~> 0.8.0)
+
+BUNDLED WITH
+   1.10.6

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2016 Ben Visser
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,23 @@
+# Iron Crawler
+
+A generic web crawler.
+
+## Requirements
+
+From a starting URL, crawl all links on that URL and print a list of URLs visited.
+
+- Follow href attributes contained in tags from the same domain
+- Ignores href attributes contained in tags from other domains (even subdomains)
+- Captures script src and link href tags for script and link tags respectively
+
+# Getting Started
+
+It's easy to get started!
+
+```
+iron-crawler <url>
+```
+
+will crawl any site for you.
+
+

data/Rakefile

@@ -0,0 +1,15 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+# import rake tasks
+Dir.glob('tasks/**/*.rake').each(&method(:import))

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/bin/iron-crawler

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require 'iron-crawler'
+require 'terminal-announce'
+
+if ARGV.empty? || %w(-h --help help).include?(ARGV.first)
+  Announce.info 'Syntax: iron-crawler <url>'
+  abort
+end
+
+url = ARGV.first
+
+Announce.info "Crawling #{url}..."
+agent = Crawler.new
+agent.spiderize(url)
+
+puts agent.history

data/lib/iron-crawler.rb

@@ -0,0 +1 @@
+require_relative 'iron-crawler/crawler'

data/lib/iron-crawler/crawler.rb

@@ -0,0 +1,80 @@
+require 'mechanize'
+
+# Enables the spidering of websites by utilizing Mechanize
+class Crawler < Mechanize
+
+  def initialize
+    @mech = Mechanize.new
+  end
+
+
+  # Kicks off the spidering of a site.
+  # @param [String] A simple URL string to crawl.
+  # @return [Hash] A hash of URls crawled.
+  #
+  def spiderize(url)
+    @mech.max_history = nil
+    stack = @mech.get(url).links
+    while link = stack.pop
+      next if reject(link)
+      puts "crawling #{link.uri}"
+      begin
+        page = link.click
+        next unless Mechanize::Page === page
+        stack.push(*page.links)
+      rescue Mechanize::ResponseCodeError
+      end
+    end
+  end
+
+
+  # Whether we should reject to spider a URL.
+  # @param [Mechanize::Page::Link] A mechanize page link.
+  # @return [Boolean] true if we should reject URL.
+  #
+  def reject(link)
+    # TODO: are we accounting for subdomains?
+    if not_valid_uri?(link) || not_same_domain?(link) || already_spidered?(link)
+      return true
+    else
+      return false
+    end
+  end
+
+
+  # Checks whether a link has already been crawled.
+  # @param [Mechanize::Page::Link] A mechanize page link.
+  # @return [Booolean] true when already spidered.
+  #
+  def already_spidered?(link)
+    begin
+      abs_url = @mech.history.first.uri.to_s.chomp('/') + link.href + '/'
+      return true if @mech.visited? link.href
+      return true if @mech.visited? abs_url
+    rescue Mechanize::UnsupportedSchemeError
+      puts "skipping #{link.uri}"
+      return true
+    end
+  end
+
+
+  # Checks whether a URL is able to be crawled.
+  # @param [Mechanize::Page::Link] A mechanize page link.
+  # @return [Booolean] true when valid URL.
+  #
+  def not_valid_uri?(link)
+    return true unless link.uri && (/^http.+/ =~ link.uri.to_s || /\/.+/ =~ link.uri.to_s)
+  end
+
+
+  # Checks whether a URL is from the same domain.
+  # @param [Mechanize::Page::Link] A mechanize page link.
+  # @return [Booolean] true when not the same domain as original URL.
+  #
+  def not_same_domain?(link)
+    host = link.uri.host
+    return true unless host.nil? || host == @mech.history.first.uri.host
+  end
+
+  private :not_valid_uri?, :not_same_domain?, :already_spidered?
+end

data/tasks/jeweler.rake

@@ -0,0 +1,14 @@
+require 'jeweler'
+
+Jeweler::Tasks.new do |gem|
+  gem.name = "iron-crawler"
+  gem.homepage = "http://github.com/noqcks/iron-crawler"
+  gem.license = "MIT"
+  gem.summary = %Q{A generic web crawler.}
+  gem.description = %Q{A generic web crawler that doesn't crawl outside URLs.}
+  gem.email = "theodore.r.visser@gmail.com"
+  gem.authors = ["Ben Visser"]
+  # dependencies defined in Gemfile
+end
+
+Jeweler::RubygemsDotOrgTasks.new

data/tasks/qa.rake

@@ -0,0 +1,47 @@
+require 'bundler/setup'
+require 'rubocop/rake_task'
+require 'reek/rake/task'
+require 'roodi_task'
+
+task default: [:audit]
+
+task audit: [:style, :complexity, :duplication, :design, :documentation]
+
+task style: [:rubocop]
+
+desc 'Enforce Style Conformance with RuboCop'
+RuboCop::RakeTask.new(:rubocop) do |task|
+  task.patterns = ['lib/**/*.rb']
+  task.fail_on_error = false
+end
+
+task complexity: [:flog]
+
+desc 'Assess Complexity with flog'
+task :flog do
+  sh 'flog lib/**/*.rb'
+end
+
+task duplication: [:flay]
+
+desc 'Detect duplication with flay'
+task :flay do
+  sh 'flay lib/**/*.rb'
+end
+
+task design: [:reek]
+
+desc 'Sniff out code smells with reek'
+Reek::Rake::Task.new(:reek) do |task|
+  task.fail_on_error = false
+end
+
+desc 'Identity volatile areas with churn'
+task :rework do
+  sh 'churn'
+end
+
+desc 'Critique documentation with inch'
+task :documentation do
+  sh 'inch'
+end

metadata

@@ -0,0 +1,268 @@
+--- !ruby/object:Gem::Specification
+name: iron-crawler
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Ben Visser
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-02-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+- !ruby/object:Gem::Dependency
+  name: reek
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.10.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.10.0
+- !ruby/object:Gem::Dependency
+  name: roodi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: churn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: flay
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+- !ruby/object:Gem::Dependency
+  name: flog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.3.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.3.2
+- !ruby/object:Gem::Dependency
+  name: inch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.37.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.37.0
+description: A generic web crawler that doesn't crawl outside URLs.
+email: theodore.r.visser@gmail.com
+executables:
+- iron-crawler
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+files:
+- ".document"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- VERSION
+- bin/iron-crawler
+- lib/iron-crawler.rb
+- lib/iron-crawler/crawler.rb
+- tasks/jeweler.rake
+- tasks/qa.rake
+homepage: http://github.com/noqcks/iron-crawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: A generic web crawler.
+test_files: []