webspinne 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.csv

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in webspinne.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,24 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+
+  # Rails example
+  watch(%r{^app/(.+)\.rb$})                           { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^app/(.*)(\.erb|\.haml)$})                 { |m| "spec/#{m[1]}#{m[2]}_spec.rb" }
+  watch(%r{^app/controllers/(.+)_(controller)\.rb$})  { |m| ["spec/routing/#{m[1]}_routing_spec.rb", "spec/#{m[2]}s/#{m[1]}_#{m[2]}_spec.rb", "spec/acceptance/#{m[1]}_spec.rb"] }
+  watch(%r{^spec/support/(.+)\.rb$})                  { "spec" }
+  watch('config/routes.rb')                           { "spec/routing" }
+  watch('app/controllers/application_controller.rb')  { "spec/controllers" }
+
+  # Capybara features specs
+  watch(%r{^app/views/(.+)/.*\.(erb|haml)$})          { |m| "spec/features/#{m[1]}_spec.rb" }
+
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$})   { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'spec/acceptance' }
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Jens Bissinger
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# Webspinne
+
+Webspinne analyzes websites by counting their pages.
+
+Named after [http://de.wikipedia.org/wiki/Webspinnen](http://de.wikipedia.org/wiki/Webspinnen).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'webspinne'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install webspinne
+
+## Usage
+
+From the commandline:
+
+    $ webspinne http://www.wikipedia.org 1000
+
+This will make webspinne go to Wikipedia website and make up to 1,000 web requests to follow the links and sublinks found on the pages. The links found will automatically be reported in a file called `www.wikipedia.org.csv`.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## License
+
+Copyright (c) 2012 Jens Bissinger. See [LICENSE.txt](LICENSE.txt)

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/webspinne

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'webspinne'
+
+# parameters
+uri         = ARGV[0] || 'http://www.google.de'
+max_follows = (ARGV[1] || 10).to_i
+
+# visit
+visit = Webspinne.visit uri, max_follows
+
+puts "=== FINISHED ==="
+
+# stats
+visit.index.tap do |i|
+  puts "links (total)       = #{i.size}"
+  puts "visited / unvisited = #{i.visited_size} / #{i.unvisited_size}" 
+  puts "onsite / offsite    = #{i.onsite_size} / #{i.offsite_size}"
+end
+
+# csv report
+report = Webspinne.csv_report(visit.index)
+report.to_file(visit.site.hostname + '.csv')

data/examples/dummy_site.rb

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+
+require 'sinatra'
+
+def html
+  %Q{<!DOCTYPE html>
+    <html>
+      <head><title>dummy</title></head>
+      <body>#{yield}</body>
+    </html>}
+end
+
+def link_to uri
+  "<a href='#{uri}'>#{uri}</a><br/>"
+end
+
+get '/' do
+  html do
+    %Q{#{link_to '/foo'}}
+  end
+end
+
+get '/foo' do
+  html do
+    [ link_to('/foo'),
+      link_to('/bar'),
+      link_to('/baz'),
+      link_to('http://foo.bar.baz') ].join
+  end
+end
+
+get '/bar' do
+  html do
+    (
+      [ link_to('/foo'),
+        link_to('http://subdomain.localhost:4567/') ] +
+      15.times.map{|i| link_to "/#{i}"}
+    ).join
+  end
+end
+
+10.times do |i|
+  get "/#{i}" do
+    html do
+      [ link_to('/foo'),
+        link_to('/bar') ].join
+    end
+  end
+end

data/lib/webspinne.rb

@@ -0,0 +1,20 @@
+require "webspinne/version"
+require "mechanize"
+
+module Webspinne
+  class DoubleVisitError < StandardError; end
+  
+  autoload :Visit, "webspinne/visit"
+  autoload :Index, "webspinne/index"
+  autoload :Site, "webspinne/site"
+  autoload :Link, "webspinne/link"
+  autoload :CSVReport, "webspinne/csv_report"
+
+  def self.visit uri, max_follows
+    Visit.new(uri).run max_follows
+  end
+
+  def self.csv_report index
+    CSVReport.new(index)
+  end
+end

data/lib/webspinne/csv_report.rb

@@ -0,0 +1,14 @@
+require 'csv'
+
+module Webspinne
+  class CSVReport < Struct.new(:index)
+    def to_file path
+      CSV.open(path, "wb") do |csv|
+        csv << ['uri', 'onsite', 'visited']
+        index.each do |link|
+          csv << [link.uri, link.onsite?, link.visited?]
+        end
+      end
+    end
+  end
+end

data/lib/webspinne/index.rb

@@ -0,0 +1,44 @@
+module Webspinne
+  class Index
+    include Enumerable
+
+    def initialize
+      @list = {}
+    end
+
+    def each &block
+      @list.each do |uri, link|
+        block.call link
+      end
+    end
+
+    # defensive set
+    def << link
+      @list[link.uri] ||= link
+    end
+
+    def size
+      @list.size
+    end
+
+    def next_unvisited_onsite_link
+      find { |link| !link.visited? && link.onsite? }
+    end
+
+    def visited_size
+      find_all { |link| link.visited? }.size
+    end
+
+    def unvisited_size
+      size - visited_size
+    end
+
+    def onsite_size
+      find_all { |link| link.onsite? }.size
+    end
+
+    def offsite_size
+      size - onsite_size
+    end
+  end
+end

data/lib/webspinne/link.rb

@@ -0,0 +1,31 @@
+module Webspinne
+  class Link
+    attr_reader :uri, :onsite
+
+    def initialize uri, onsite=false
+      @uri = uri
+      @onsite = onsite
+      @visited = false
+    end
+
+    def visited!
+      if visited?
+        raise DoubleVisitError, uri
+      else
+        @visited = true
+      end
+    end
+
+    def visited?
+      @visited == true
+    end
+
+    def onsite?
+      onsite == true
+    end
+
+    def offsite?
+      !onsite?
+    end
+  end
+end

data/lib/webspinne/site.rb

@@ -0,0 +1,15 @@
+module Webspinne
+  class Site < Struct.new(:uri)
+    def onsite? test_uri
+      hostname == hostname(test_uri) || hostname(test_uri) == nil
+    end
+
+    def offsite? test_uri
+      !onsite?(test_uri)
+    end
+
+    def hostname test_uri=uri
+      URI(test_uri).hostname
+    end
+  end
+end

data/lib/webspinne/version.rb

@@ -0,0 +1,3 @@
+module Webspinne
+  VERSION = "0.0.1"
+end

data/lib/webspinne/visit.rb

@@ -0,0 +1,58 @@
+module Webspinne
+  class Visit
+    attr_reader :site, :agent, :index
+
+    def initialize(uri)
+      @site = Site.new(uri)
+    end
+
+    def run max_follows
+      @agent = Mechanize.new
+      @index = Index.new
+
+      puts "visiting #{site.uri} - following max. #{max_follows} links"
+
+      plan_visit site.uri
+      max_follows.times do
+        if link = index.next_unvisited_onsite_link
+          if exec_visit(link)
+            print '.'
+          else
+            print 'f'
+          end
+        else
+          # no more unvisited onsite links
+          break
+        end
+      end
+
+      puts
+
+      self
+    end
+
+    def exec_visit link
+      link.visited!
+      page = agent.get(link.uri)
+      
+      # e.g. images have no links
+      if page.respond_to? :links
+        page.links.each do |link|
+          if link.uri.to_s != ''
+            plan_visit link.uri.to_s
+          end
+        end
+        true
+      else
+        false
+      end
+    rescue => e
+      puts e.class
+      false
+    end
+
+    def plan_visit uri
+      index << Link.new(uri, site.onsite?(uri))
+    end
+  end
+end

data/spec/fixtures/vcr/localhost.yml

@@ -0,0 +1,297 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://localhost:4567/
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - ! '*/*'
+      User-Agent:
+      - Mechanize/2.5.1 Ruby/1.9.3p327 (http://github.com/tenderlove/mechanize/)
+      Accept-Encoding:
+      - gzip,deflate,identity
+      Accept-Charset:
+      - ISO-8859-1,utf-8;q=0.7,*;q=0.7
+      Accept-Language:
+      - en-us,en;q=0.5
+      Host:
+      - localhost:4567
+      Connection:
+      - keep-alive
+      Keep-Alive:
+      - 300
+  response:
+    status:
+      code: 200
+      message: ! 'OK '
+    headers:
+      X-Frame-Options:
+      - sameorigin
+      X-Xss-Protection:
+      - 1; mode=block
+      Content-Type:
+      - text/html;charset=utf-8
+      Content-Length:
+      - '126'
+      Server:
+      - WEBrick/1.3.1 (Ruby/1.9.3/2012-11-10)
+      Date:
+      - Thu, 06 Dec 2012 18:10:20 GMT
+      Connection:
+      - Keep-Alive
+    body:
+      encoding: US-ASCII
+      string: ! "<!DOCTYPE html>\n    <html>\n      <head><title>dummy</title></head>\n
+        \     <body><a href='/foo'>/foo</a><br/></body>\n    </html>"
+    http_version: 
+  recorded_at: Thu, 06 Dec 2012 18:10:20 GMT
+- request:
+    method: get
+    uri: http://localhost:4567/foo
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - ! '*/*'
+      User-Agent:
+      - Mechanize/2.5.1 Ruby/1.9.3p327 (http://github.com/tenderlove/mechanize/)
+      Accept-Encoding:
+      - gzip,deflate,identity
+      Accept-Charset:
+      - ISO-8859-1,utf-8;q=0.7,*;q=0.7
+      Accept-Language:
+      - en-us,en;q=0.5
+      Host:
+      - localhost:4567
+      Referer:
+      - !ruby/object:URI::HTTP
+        scheme: http
+        user: 
+        password: 
+        host: localhost
+        port: 4567
+        path: /
+        query: 
+        opaque: 
+        registry: 
+        fragment: 
+        parser: 
+      Connection:
+      - keep-alive
+      Keep-Alive:
+      - 300
+  response:
+    status:
+      code: 200
+      message: ! 'OK '
+    headers:
+      X-Frame-Options:
+      - sameorigin
+      X-Xss-Protection:
+      - 1; mode=block
+      Content-Type:
+      - text/html;charset=utf-8
+      Content-Length:
+      - '238'
+      Server:
+      - WEBrick/1.3.1 (Ruby/1.9.3/2012-11-10)
+      Date:
+      - Thu, 06 Dec 2012 18:10:20 GMT
+      Connection:
+      - Keep-Alive
+    body:
+      encoding: US-ASCII
+      string: ! "<!DOCTYPE html>\n    <html>\n      <head><title>dummy</title></head>\n
+        \     <body><a href='/foo'>/foo</a><br/><a href='/bar'>/bar</a><br/><a href='/baz'>/baz</a><br/><a
+        href='http://foo.bar.baz'>http://foo.bar.baz</a><br/></body>\n    </html>"
+    http_version: 
+  recorded_at: Thu, 06 Dec 2012 18:10:20 GMT
+- request:
+    method: get
+    uri: http://localhost:4567/bar
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - ! '*/*'
+      User-Agent:
+      - Mechanize/2.5.1 Ruby/1.9.3p327 (http://github.com/tenderlove/mechanize/)
+      Accept-Encoding:
+      - gzip,deflate,identity
+      Accept-Charset:
+      - ISO-8859-1,utf-8;q=0.7,*;q=0.7
+      Accept-Language:
+      - en-us,en;q=0.5
+      Host:
+      - localhost:4567
+      Referer:
+      - !ruby/object:URI::HTTP
+        scheme: http
+        user: 
+        password: 
+        host: localhost
+        port: 4567
+        path: /foo
+        query: 
+        opaque: 
+        registry: 
+        fragment: 
+        parser: 
+      Connection:
+      - keep-alive
+      Keep-Alive:
+      - 300
+  response:
+    status:
+      code: 200
+      message: ! 'OK '
+    headers:
+      X-Frame-Options:
+      - sameorigin
+      X-Xss-Protection:
+      - 1; mode=block
+      Content-Type:
+      - text/html;charset=utf-8
+      Content-Length:
+      - '580'
+      Server:
+      - WEBrick/1.3.1 (Ruby/1.9.3/2012-11-10)
+      Date:
+      - Thu, 06 Dec 2012 18:10:20 GMT
+      Connection:
+      - Keep-Alive
+    body:
+      encoding: US-ASCII
+      string: ! "<!DOCTYPE html>\n    <html>\n      <head><title>dummy</title></head>\n
+        \     <body><a href='/foo'>/foo</a><br/><a href='http://subdomain.localhost:4567/'>http://subdomain.localhost:4567/</a><br/><a
+        href='/0'>/0</a><br/><a href='/1'>/1</a><br/><a href='/2'>/2</a><br/><a href='/3'>/3</a><br/><a
+        href='/4'>/4</a><br/><a href='/5'>/5</a><br/><a href='/6'>/6</a><br/><a href='/7'>/7</a><br/><a
+        href='/8'>/8</a><br/><a href='/9'>/9</a><br/><a href='/10'>/10</a><br/><a
+        href='/11'>/11</a><br/><a href='/12'>/12</a><br/><a href='/13'>/13</a><br/><a
+        href='/14'>/14</a><br/></body>\n    </html>"
+    http_version: 
+  recorded_at: Thu, 06 Dec 2012 18:10:20 GMT
+- request:
+    method: get
+    uri: http://localhost:4567/baz
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - ! '*/*'
+      User-Agent:
+      - Mechanize/2.5.1 Ruby/1.9.3p327 (http://github.com/tenderlove/mechanize/)
+      Accept-Encoding:
+      - gzip,deflate,identity
+      Accept-Charset:
+      - ISO-8859-1,utf-8;q=0.7,*;q=0.7
+      Accept-Language:
+      - en-us,en;q=0.5
+      Host:
+      - localhost:4567
+      Referer:
+      - &70162820747780 !ruby/object:URI::HTTP
+        scheme: http
+        user: 
+        password: 
+        host: localhost
+        port: 4567
+        path: /bar
+        query: 
+        opaque: 
+        registry: 
+        fragment: 
+        parser: 
+      Connection:
+      - keep-alive
+      Keep-Alive:
+      - 300
+  response:
+    status:
+      code: 404
+      message: ! 'Not Found '
+    headers:
+      X-Frame-Options:
+      - sameorigin
+      X-Xss-Protection:
+      - 1; mode=block
+      Content-Type:
+      - text/html;charset=utf-8
+      X-Cascade:
+      - pass
+      Content-Length:
+      - '439'
+      Server:
+      - WEBrick/1.3.1 (Ruby/1.9.3/2012-11-10)
+      Date:
+      - Thu, 06 Dec 2012 18:10:20 GMT
+      Connection:
+      - Keep-Alive
+    body:
+      encoding: US-ASCII
+      string: ! "<!DOCTYPE html>\n<html>\n<head>\n  <style type=\"text/css\">\n  body
+        { text-align:center;font-family:helvetica,arial;font-size:22px;\n    color:#888;margin:20px}\n
+        \ #c {margin:0 auto;width:500px;text-align:left}\n  </style>\n</head>\n<body>\n
+        \ <h2>Sinatra doesn&rsquo;t know this ditty.</h2>\n  <img src='http://localhost:4567/__sinatra__/404.png'>\n
+        \ <div id=\"c\">\n    Try this:\n    <pre>get '/baz' do\n  \"Hello World\"\nend</pre>\n
+        \ </div>\n</body>\n</html>\n"
+    http_version: 
+  recorded_at: Thu, 06 Dec 2012 18:10:20 GMT
+- request:
+    method: get
+    uri: http://localhost:4567/0
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - ! '*/*'
+      User-Agent:
+      - Mechanize/2.5.1 Ruby/1.9.3p327 (http://github.com/tenderlove/mechanize/)
+      Accept-Encoding:
+      - gzip,deflate,identity
+      Accept-Charset:
+      - ISO-8859-1,utf-8;q=0.7,*;q=0.7
+      Accept-Language:
+      - en-us,en;q=0.5
+      Host:
+      - localhost:4567
+      Referer:
+      - *70162820747780
+      Connection:
+      - keep-alive
+      Keep-Alive:
+      - 300
+  response:
+    status:
+      code: 200
+      message: ! 'OK '
+    headers:
+      X-Frame-Options:
+      - sameorigin
+      X-Xss-Protection:
+      - 1; mode=block
+      Content-Type:
+      - text/html;charset=utf-8
+      Content-Length:
+      - '154'
+      Server:
+      - WEBrick/1.3.1 (Ruby/1.9.3/2012-11-10)
+      Date:
+      - Thu, 06 Dec 2012 18:10:20 GMT
+      Connection:
+      - Keep-Alive
+    body:
+      encoding: US-ASCII
+      string: ! "<!DOCTYPE html>\n    <html>\n      <head><title>dummy</title></head>\n
+        \     <body><a href='/foo'>/foo</a><br/><a href='/bar'>/bar</a><br/></body>\n
+        \   </html>"
+    http_version: 
+  recorded_at: Thu, 06 Dec 2012 18:10:20 GMT
+recorded_with: VCR 2.3.0

data/spec/lib/webspinne/index_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+describe Webspinne::Index do
+  subject { described_class.new }
+
+  its(:size) { should == 0}
+  its(:visited_size) { should == 0}
+
+  context 'with onsite-link' do
+    before { subject << Webspinne::Link.new('foo', true) }
+
+    its(:size) { should == 1 }
+    its(:next_unvisited_onsite_link) { should be_instance_of(Webspinne::Link) }
+  
+    context 'and the same link again' do
+      before { subject << Webspinne::Link.new('foo', true) }
+      its(:size) { should == 1 }
+    end
+
+    context 'and another link' do
+      before { subject << Webspinne::Link.new('bar', true) }
+      its(:size) { should == 2 }
+    end
+
+    context 'visited' do
+      before { subject.next_unvisited_onsite_link.visited! }
+      its(:visited_size) { should == 1 }
+    end
+  end
+
+  context 'with offsite-link' do
+    before { subject << Webspinne::Link.new('foo', false) }
+
+    its(:size) { should == 1 }
+    its(:next_unvisited_onsite_link) { should be_nil }
+  end
+
+end

data/spec/lib/webspinne/site_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe Webspinne::Site do
+  subject { described_class.new 'http://foobar.com' }
+
+  it 'returns hostname' do
+    subject.hostname.should == 'foobar.com'
+    subject.hostname('http://www.foobar.com').should == 'www.foobar.com'
+    subject.hostname('http://baccio.com').should == 'baccio.com'
+    subject.hostname('lalal').should == nil
+  end
+
+  shared_examples_for 'onsite' do |uri|
+    specify { subject.onsite?(uri).should be_true }
+    specify { subject.offsite?(uri).should be_false }
+  end
+
+  shared_examples_for 'offsite' do |uri|
+    specify { subject.onsite?(uri).should be_false }
+    specify { subject.offsite?(uri).should be_true }
+  end
+
+  # onsite
+  %w[ http://foobar.com
+      http://foobar.com/
+      http://foobar.com/x ].each do |uri|
+    context uri do
+      it_behaves_like 'onsite', uri
+    end
+  end
+
+  # offsite
+  %w[ http://baccio.com
+      http://subdomain.foobar.com ].each do |uri|
+    context uri do
+      it_behaves_like 'offsite', uri
+    end
+  end
+end

data/spec/lib/webspinne_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe Webspinne do
+  it 'visits a site' do
+    VCR.use_cassette('localhost') do
+      visit = described_class.visit 'http://localhost:4567', 5
+      visit.index.size.should           == 21   # links
+      visit.index.visited_size.should   == 5    # visitied_links
+      visit.index.unvisited_size.should == 16   # unvisited links
+      visit.index.onsite_size.should    == 19   # onsite links
+      visit.index.offsite_size.should   == 2    # offsite links
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,28 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end
+
+require 'vcr'
+require 'webmock'
+VCR.configure do |c|
+  c.cassette_library_dir = File.expand_path('../../spec/fixtures/vcr', __FILE__)
+  c.hook_into :webmock # or :fakeweb
+end
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'webspinne'

data/webspinne.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'webspinne/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "webspinne"
+  gem.version       = Webspinne::VERSION
+  gem.authors       = ["Jens Bissinger"]
+  gem.email         = ["mail@jens-bissinger.de"]
+  gem.description   = %q{Webspinne analyzes websites by counting their pages.}
+  gem.summary       = %q{Uses the mechanize gem to open a website and collects all links and sublinks.}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency "mechanize"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "guard-rspec"
+  gem.add_development_dependency "vcr"
+  gem.add_development_dependency "webmock"
+  gem.add_development_dependency "sinatra"
+end

metadata

@@ -0,0 +1,171 @@
+--- !ruby/object:Gem::Specification
+name: webspinne
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jens Bissinger
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Webspinne analyzes websites by counting their pages.
+email:
+- mail@jens-bissinger.de
+executables:
+- webspinne
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/webspinne
+- examples/dummy_site.rb
+- fonic.de.csv
+- lib/webspinne.rb
+- lib/webspinne/csv_report.rb
+- lib/webspinne/index.rb
+- lib/webspinne/link.rb
+- lib/webspinne/site.rb
+- lib/webspinne/version.rb
+- lib/webspinne/visit.rb
+- localhost.csv
+- spec/fixtures/vcr/localhost.yml
+- spec/lib/webspinne/index_spec.rb
+- spec/lib/webspinne/site_spec.rb
+- spec/lib/webspinne_spec.rb
+- spec/spec_helper.rb
+- webspinne.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Uses the mechanize gem to open a website and collects all links and sublinks.
+test_files:
+- spec/fixtures/vcr/localhost.yml
+- spec/lib/webspinne/index_spec.rb
+- spec/lib/webspinne/site_spec.rb
+- spec/lib/webspinne_spec.rb
+- spec/spec_helper.rb