spidermech 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 637a38fca20a36c523b57ae807ffa30b1b9d63f3
+  data.tar.gz: 9b417e4ce15fea26b72127c4b4b6624f0ac85847
+SHA512:
+  metadata.gz: 11c1e177153ab63db942a826ab877242997d9d22e2e8971cf87d391fc7969e00d82475f504724054c4722bbc981c8257150f4c75ba19d9bd59013b38f6bff6b3
+  data.tar.gz: fed4c3d45c6949e190239e435eab66aa74ae5c13e613be34a72edf3eb16d669bf495095d0804c7c07d4ed94470d5db2228cc7806c3e3dfcb1e89d12e43f8a58f

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+notes.txt

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.ruby-version

@@ -0,0 +1 @@
+2.1.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in crawler.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Caleb Albritton
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# Crawler
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'crawler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install spidermech
+
+## Gem Usage
+
+TODO: Write usage instructions here
+
+## Command Line Usage
+
+The gem provides a command line tool. You can invoke it via
+
+	bundle exec crawl http://google.com
+
+It will crawl the page and give you the appropriate output.
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/crawler/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('spec')
+
+task :default => :spec
+task :test => :spec

data/bin/spidermech

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.join(Dir.getwd, 'lib')
+
+require 'uri'
+require 'json'
+require 'spidermech'
+
+raise ArgumentError, 'You must provide a url to crawl.' unless !ARGV.empty?
+
+url = ARGV[0]
+
+spider = SpiderMech.new url
+spider.run
+spider.save_json

data/lib/spidermech.rb

@@ -0,0 +1,134 @@
+require 'mechanize'
+require 'logger'
+require 'json'
+require 'pry'
+
+class SpiderMech
+  attr_reader :queue
+  attr_reader :crawled
+  attr_reader :data
+
+  def initialize(start_page)
+    @logger = Logger.new 'spidermech.log'
+    @start_page = start_page
+
+    @queue   = []
+    @crawled = []
+    @data    = []
+
+    @queue << @start_page
+
+    @bot = Mechanize.new
+  end
+
+  def left_in_queue
+    i = 0
+
+    @queue.each do |link|
+      if @crawled.include? link
+        # we don't need to crawl this one
+      else
+        i += 1
+      end
+    end
+
+    i
+  end
+
+  def save_json
+    filename = "#{URI.parse(@start_page).host}.json"
+    @logger.info "Writing sitemap data to #{filename}"
+    json = @data.to_json
+    File.open(filename, 'w') { |f| f.write json }
+  end
+
+  def run
+    while !@queue.empty?
+      crawl
+    end
+
+    @data
+  end
+
+  def crawl
+    url = @queue.shift
+
+    if @crawled.include? url
+      # @logger.warn "Already crawled #{url}"
+      return
+    else
+      @logger.info "Crawling #{url}"
+      @logger.info "Left in Queue: #{left_in_queue}"
+    end
+
+    page = @bot.get url
+
+    if page.class != Mechanize::Page
+      @logger.info "File crawling is not supported."
+      return
+    end
+
+    @crawled << url
+
+    # get all the assets
+    data = {
+      :url => url,
+      :assets => {
+        :scripts => find_scripts(page),
+        :images => find_images(page),
+        :css => find_css(page)
+      },
+
+      :links => []
+    }
+
+    page.links.each do |link|
+
+      begin
+        if link.href[0] == '/' # this is a relative link
+          @queue << link.href
+          data[:links] << link.href
+        elsif link.href[0..@start_page.length] == @start_page # still part of this domain
+          @queue << link.href
+          data[:links] << link.href
+        else
+          # @logger.info "This link did not fall under our jurisdiction: #{link.href}"
+        end
+      rescue Exception => e
+        # @logger.error e
+      end
+    end
+
+    @data << data
+  end
+
+  def find_scripts(page)
+    page.search('script').map do |script| 
+      begin
+        script.attributes['src'].value
+      rescue Exception => e
+        # @logger.error e
+      end
+    end
+  end
+
+  def find_images(page)
+    page.search('img').map do |img|
+      begin
+        img.attributes['src'].value
+      rescue Exception => e
+        # @logger.error e
+      end
+    end
+  end
+
+  def find_css(page)
+    page.search('link').map do |css|
+      begin
+        css.attributes['href'].value
+       rescue Exception => e
+         # @logger.error e
+       end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,28 @@
+require 'spidermech'
+require 'webrick'
+
+web_server = WEBrick::HTTPServer.new :Port => 8321, :DocumentRoot => Dir.pwd + '/test_site', :AccessLog => []
+
+server_thread = Thread.new do
+  web_server.start
+end
+
+sleep 1 # wait a sec for the server to start
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

data/spec/spidermech_spec.rb

@@ -0,0 +1,66 @@
+require 'spec_helper'
+
+describe SpiderMech do
+
+  before(:all) do
+    @example_dataset = [{:url=>"http://localhost:8321", :assets=>{:scripts=>["https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js", "http://getbootstrap.com/dist/js/bootstrap.min.js"], :images=>[], :css=>["http://getbootstrap.com/dist/css/bootstrap.min.css", "http://getbootstrap.com/examples/starter-template/starter-template.css"]}, :links=>["/", "/about.html", "/contact.html"]},
+                        {:url=>"/", :assets=>{:scripts=>["https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js", "http://getbootstrap.com/dist/js/bootstrap.min.js"], :images=>[], :css=>["http://getbootstrap.com/dist/css/bootstrap.min.css", "http://getbootstrap.com/examples/starter-template/starter-template.css"]}, :links=>["/", "/about.html", "/contact.html"]},
+                        {:url=>"/about.html", :assets=>{:scripts=>["https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js", "http://getbootstrap.com/dist/js/bootstrap.min.js"], :images=>[], :css=>["http://getbootstrap.com/dist/css/bootstrap.min.css", "http://getbootstrap.com/examples/starter-template/starter-template.css"]}, :links=>["/", "/about.html", "/contact.html"]},
+                        {:url=>"/contact.html", :assets=>{:scripts=>["https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js", "http://getbootstrap.com/dist/js/bootstrap.min.js"], :images=>[], :css=>["http://getbootstrap.com/dist/css/bootstrap.min.css", "http://getbootstrap.com/examples/starter-template/starter-template.css"]}, :links=>["/", "/about.html", "/contact.html"]}
+                      ]
+
+    @empty_dataset = [{:url=>"http://localhost:8321/empty.html", :assets=>{:scripts=>[], :images=>[], :css=>[]}, :links=>[]}]
+  end
+
+  after(:all) do #cleanup
+    File.delete 'spidermech.log'
+    File.delete 'localhost.json'
+  end
+
+  it 'should crawl a site' do
+    spider = SpiderMech.new 'http://localhost:8321'
+    data = spider.run
+    data.should eq(@example_dataset)
+  end
+
+  it 'should save a json file' do
+    spider = SpiderMech.new 'http://localhost:8321'
+    spider.run
+    spider.save_json
+
+    File.exist?('localhost.json').should eq(true)
+
+    file = File.open 'localhost.json', 'r'
+    data = JSON.parse file.read
+    data.should eq(JSON.parse(@example_dataset.to_json)) # quick way to change symbols to quotes to test equality
+  end
+
+  it 'should return a hash with empty assets for empty page' do
+    
+    spider = SpiderMech.new 'http://localhost:8321/empty.html'
+    data = spider.run
+    data.length.should eq(1) # should only have on result as it contains links to no other pages
+    data.should eq(@empty_dataset)
+  end
+
+  it 'should only have one image' do
+    spider = SpiderMech.new 'http://localhost:8321/image_only.html'
+    data = spider.run
+    data.length.should eq(1)
+    data.first[:assets][:images].first.should eq('test.png')
+  end
+
+  it 'should only have one script' do
+    spider = SpiderMech.new 'http://localhost:8321/script_only.html'
+    data = spider.run
+    data.length.should eq(1)
+    data.first[:assets][:scripts].first.should eq('stuffs.js')
+  end
+
+  it 'should only have one css file' do
+    spider = SpiderMech.new 'http://localhost:8321/css_only.html'
+    data = spider.run
+    data.length.should eq(1)
+    data.first[:assets][:css].first.should eq('style.css')
+  end
+end

data/spidermech.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('./lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "spidermech"
+  spec.version       = '0.0.1'
+  spec.authors       = ["Caleb Albritton"]
+  spec.email         = ["ithinkincode@gmail.com"]
+  spec.summary       = "Single URL crawler."
+  spec.description   = "Does things"
+  spec.homepage      = "http://github.com/C0deMaver1ck/spidermech"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'mechanize', "~> 2.7"
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "pry", "~> 0.9"
+  spec.add_development_dependency "rake", "~> 10.2"
+  spec.add_development_dependency "rspec", "~> 2.14"
+  spec.add_development_dependency "webrick", "~> 1.3"
+end

data/test_site/about.html

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="description" content="">
+    <meta name="author" content="">
+
+    <title>About</title>
+
+    <!-- Bootstrap core CSS -->
+    <link href="http://getbootstrap.com/dist/css/bootstrap.min.css" rel="stylesheet">
+
+    <!-- Custom styles for this template -->
+    <link href="http://getbootstrap.com/examples/starter-template/starter-template.css" rel="stylesheet">
+
+    <!-- Just for debugging purposes. Don't actually copy this line! -->
+    <!--[if lt IE 9]><script src="../../assets/js/ie8-responsive-file-warning.js"></script><![endif]-->
+
+    <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+    <!--[if lt IE 9]>
+      <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+      <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+    <![endif]-->
+  </head>
+
+  <body>
+
+    <div class="navbar navbar-inverse navbar-fixed-top" role="navigation">
+      <div class="container">
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <a class="navbar-brand" href="#">Project name</a>
+        </div>
+        <div class="collapse navbar-collapse">
+          <ul class="nav navbar-nav">
+            <li class="active"><a href="/">Home</a></li>
+            <li><a href="/about.html">About</a></li>
+            <li><a href="/contact.html">Contact</a></li>
+          </ul>
+        </div><!--/.nav-collapse -->
+      </div>
+    </div>
+
+    <div class="container">
+
+      <div class="starter-template">
+        <h1>Bootstrap starter template</h1>
+        <p class="lead">Use this document as a way to quickly start any new project.<br> All you get is this text and a mostly barebones HTML document.</p>
+      </div>
+
+    </div><!-- /.container -->
+
+
+    <!-- Bootstrap core JavaScript
+    ================================================== -->
+    <!-- Placed at the end of the document so the pages load faster -->
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+    <script src="http://getbootstrap.com/dist/js/bootstrap.min.js"></script>
+  </body>
+</html>

data/test_site/contact.html

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="description" content="">
+    <meta name="author" content="">
+
+    <title>Contact</title>
+
+    <!-- Bootstrap core CSS -->
+    <link href="http://getbootstrap.com/dist/css/bootstrap.min.css" rel="stylesheet">
+
+    <!-- Custom styles for this template -->
+    <link href="http://getbootstrap.com/examples/starter-template/starter-template.css" rel="stylesheet">
+
+    <!-- Just for debugging purposes. Don't actually copy this line! -->
+    <!--[if lt IE 9]><script src="../../assets/js/ie8-responsive-file-warning.js"></script><![endif]-->
+
+    <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+    <!--[if lt IE 9]>
+      <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+      <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+    <![endif]-->
+  </head>
+
+  <body>
+
+    <div class="navbar navbar-inverse navbar-fixed-top" role="navigation">
+      <div class="container">
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <a class="navbar-brand" href="#">Project name</a>
+        </div>
+        <div class="collapse navbar-collapse">
+          <ul class="nav navbar-nav">
+            <li class="active"><a href="/">Home</a></li>
+            <li><a href="/about.html">About</a></li>
+            <li><a href="/contact.html">Contact</a></li>
+          </ul>
+        </div><!--/.nav-collapse -->
+      </div>
+    </div>
+
+    <div class="container">
+
+      <div class="starter-template">
+        <h1>Bootstrap starter template</h1>
+        <p class="lead">Use this document as a way to quickly start any new project.<br> All you get is this text and a mostly barebones HTML document.</p>
+      </div>
+
+    </div><!-- /.container -->
+
+
+    <!-- Bootstrap core JavaScript
+    ================================================== -->
+    <!-- Placed at the end of the document so the pages load faster -->
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+    <script src="http://getbootstrap.com/dist/js/bootstrap.min.js"></script>
+  </body>
+</html>

data/test_site/css_only.html

@@ -0,0 +1 @@
+<link rel="stylesheet" type="text/css" href="style.css">

data/test_site/image_only.html

@@ -0,0 +1 @@
+<img src='test.png'>

data/test_site/index.html

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="description" content="">
+    <meta name="author" content="">
+
+    <title>Home</title>
+
+    <!-- Bootstrap core CSS -->
+    <link href="http://getbootstrap.com/dist/css/bootstrap.min.css" rel="stylesheet">
+
+    <!-- Custom styles for this template -->
+    <link href="http://getbootstrap.com/examples/starter-template/starter-template.css" rel="stylesheet">
+
+    <!-- Just for debugging purposes. Don't actually copy this line! -->
+    <!--[if lt IE 9]><script src="../../assets/js/ie8-responsive-file-warning.js"></script><![endif]-->
+
+    <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+    <!--[if lt IE 9]>
+      <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+      <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+    <![endif]-->
+  </head>
+
+  <body>
+
+    <div class="navbar navbar-inverse navbar-fixed-top" role="navigation">
+      <div class="container">
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <a class="navbar-brand" href="#">Project name</a>
+        </div>
+        <div class="collapse navbar-collapse">
+          <ul class="nav navbar-nav">
+            <li class="active"><a href="/">Home</a></li>
+            <li><a href="/about.html">About</a></li>
+            <li><a href="/contact.html">Contact</a></li>
+          </ul>
+        </div><!--/.nav-collapse -->
+      </div>
+    </div>
+
+    <div class="container">
+
+      <div class="starter-template">
+        <h1>Bootstrap starter template</h1>
+        <p class="lead">Use this document as a way to quickly start any new project.<br> All you get is this text and a mostly barebones HTML document.</p>
+      </div>
+
+    </div><!-- /.container -->
+
+
+    <!-- Bootstrap core JavaScript
+    ================================================== -->
+    <!-- Placed at the end of the document so the pages load faster -->
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+    <script src="http://getbootstrap.com/dist/js/bootstrap.min.js"></script>
+  </body>
+</html>

data/test_site/script_only.html

@@ -0,0 +1 @@
+<script src='stuffs.js'></script>

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: spidermech
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Caleb Albritton
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+description: Does things
+email:
+- ithinkincode@gmail.com
+executables:
+- spidermech
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/spidermech
+- lib/spidermech.rb
+- spec/spec_helper.rb
+- spec/spidermech_spec.rb
+- spidermech.gemspec
+- test_site/about.html
+- test_site/contact.html
+- test_site/css_only.html
+- test_site/empty.html
+- test_site/image_only.html
+- test_site/index.html
+- test_site/script_only.html
+homepage: http://github.com/C0deMaver1ck/spidermech
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: Single URL crawler.
+test_files:
+- spec/spec_helper.rb
+- spec/spidermech_spec.rb