spidercrawl 0.3.9

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1cbd732cf3ba32a6a670cf77b39350eb36577d82
+  data.tar.gz: bb74e55ea3561de5b270978705d5852c3051fbaf
+SHA512:
+  metadata.gz: 432c2150504fc990558fe00091a6965dda060212ae8427603fc19002ee8a7c5f6c3f65e2019700d3b58fdc282801f45df82f37e329db43118fc98edc6dc1afff
+  data.tar.gz: 18b71483b43c9abbcf012850865b26e99e0cf853b15954f18d0f92cf35d62fc01e31e7deb1b78161df34ee7029875031884d27a8bc4597ecc7881c426fc64bdb

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+.DS_Store

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in spidercrawl.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Belson Heng
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,78 @@
+# SpiderCrawl
+
+A ruby gem that can crawl a domain and let you have information about the pages it visits. 
+
+With the help of Nokogiri, SpiderCrawl will parse each page and return you its title, links, css, words, and many many more! You can also customize what you want to do before & after each fetch request.
+
+Long story short - Feed an URL to SpiderCrawl and it will crawl + scrape the content for you. 
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'spidercrawl'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install spidercrawl
+
+## Usage
+
+Start crawling a domain by calling __Spiderman.shoot__(*url*) and it will return you a list of pages it has crawled and scraped:
+
+    pages = Spiderman.shoot('http://forums.hardwarezone.com.sg/hwm-magazine-publication-38/')
+
+To include a pattern matching for each page:
+
+    pages = Spiderman.shoot('http://forums.hardwarezone.com.sg/hwm-magazine-publication-38/',
+                            :pattern => Regexp.new('^http:\/\/forums\.hardwarezone\.com\.sg\/hwm-magazine-publication-38\/?(.*\.html)?$')
+
+Access the following scraped data:
+
+    pages.each do |page|
+      page.url              #URL of the page
+      page.scheme           #Scheme of the page (http, https, etc.)
+      page.host             #Hostname of the page
+      page.base_url         #Root URL of the page
+      page.doc              #Nokogiri document
+      page.headers          #Response headers for the page
+      page.title            #Title of the page
+      page.links            #Every link found in the page, returned as an array
+      page.internal_links   #Only internal links returned as an array
+      page.external_links   #Only external links returned as an array
+      page.emails           #Every email found in the page, returned as an array
+      page.images           #Every img found in the page, returned as an array
+      page.words            #Every word that appeared in the page, returned as an array
+      page.css              #CSS scripts used in the page, returned as an array
+      page.content          #Contents of the HTML document in string
+      page.content_type     #Content type of the page
+      page.text             #Any text found in the page without HTML tags
+      page.response_code    #HTTP response code of the page
+      page.response_time    #HTTP response time of the page
+      page.crawled_time     #The time when this page is crawled/fetched, returned as milliseconds since epoch
+    end
+
+## Dependencies
+
++ Colorize
++ Curb
++ Nokogiri
++ Typhoeus
+
+## Contributing
+
+1. Fork it ( https://github.com/belsonheng/spidercrawl/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## License
+
+SpiderCrawl is released under the [MIT license](https://github.com/belsonheng/spidercrawl/blob/master/LICENSE.txt).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/lib/spidercrawl.rb

@@ -0,0 +1,10 @@
+require "spidercrawl/spider_worker"
+
+class Spiderman
+  def self.shoot(urls, options, &block)
+    spiderman = Spidercrawl::SpiderWorker.new(urls, options)
+    yield spiderman if block_given?
+    return spiderman.parallel_crawl if options[:parallel] == true
+    return spiderman.crawl unless options[:parallel]
+  end
+end

data/lib/spidercrawl/page.rb

@@ -0,0 +1,187 @@
+require 'nokogiri'
+
+module Spidercrawl
+  # Parses the content with Nokogiri
+  class Page
+
+    attr_reader :location, :response_time
+    attr_accessor :crawled_time
+
+    def initialize(url, options = {})
+      @url = url
+      @code = options[:response_code]
+      @headers = options[:response_head]
+      @location = options[:redirect_url]
+      @body = options[:response_body]
+      @response_time = options[:response_time]
+      @crawled_time = options[:crawled_time]
+    end
+
+    #
+    # Return the url of the page
+    #
+    def url
+      @url.to_s
+    end
+
+    #
+    # Return the url scheme of the page (e.g. http, https, etc.)
+    #
+    def scheme
+      @url.scheme
+    end
+
+    #
+    # Return the url host of the page
+    #
+    def host
+      @url.host
+    end
+
+    #
+    # Return the base url of the page
+    #
+    def base_url
+      @base_url = "#{scheme}://#{host}"
+    end
+
+    #
+    # Return the Nokogiri html document
+    #
+    def doc
+      @document = Nokogiri::HTML(@body)
+      rescue Exception => e
+        puts e.inspect
+        puts e.backtrace
+    end
+
+    #
+    # Return the headers of the page
+    #
+    def headers
+      puts @headers
+    end
+    
+    #
+    # Return the title of the page
+    #
+    def title
+      @title = doc.css('head title').inner_text
+    end
+
+    #
+    # Return the entire links found in the page; exclude empty links
+    #
+    def links
+      @links = doc.css('a').map { |link| link['href'].to_s }.uniq.delete_if { |href| href.empty? }.map { |url| absolutify(url.strip) }
+    end
+
+    #
+    # Return the internal links found in the page
+    #
+    def internal_links
+      @internal_links = links.select { |link| URI.parse(link).host == host } rescue nil
+    end
+
+    #
+    # Return the external links found in the page
+    #
+    def external_links
+      @external_links = links.select { |link| URI.parse(link).host != host } rescue nil
+    end
+
+    #
+    # Return any emails found in the page
+    #
+    def emails
+      @body.match(/[\w.!#\$%+-]+@[\w-]+(?:\.[\w-]+)+/)
+    end
+
+    #
+    # Return all images found in the page
+    #
+    def images
+      @images = doc.css('img').map { |img| img['src'].to_s }.uniq.delete_if { |src| src.empty? }.map { |url| absolutify(url.strip) }
+    end
+
+    #
+    # Return all words found in the page
+    #
+    def words
+      @words = text.split(/[^a-zA-Z]/).delete_if { |word| word.empty? }
+    end
+
+    #
+    # Return css scripts of the page
+    #
+    def css
+      @css = doc.search("[@type='text/css']")
+    end
+
+    def meta_keywords
+    end
+
+    def meta_descriptions
+    end
+
+    #
+    # Return html content as a string
+    #
+    def content
+      @body.to_s
+    end
+
+    #
+    # Return the content type of the page
+    #
+    def content_type
+      doc.at("meta[@http-equiv='Content-Type']")['content']
+    end
+
+    # 
+    # Return plain text of the page without html tags
+    #
+    def text
+      temp_doc = doc
+      temp_doc.css('script, noscript, style, link').each { |node| node.remove }
+      @text = temp_doc.css('body').text.split("\n").collect { |line| line.strip }.join("\n")
+    end
+
+    #
+    # Return the response code
+    #
+    def response_code
+      @code
+    end
+
+    #
+    # Return true if page not found 
+    #
+    def not_found?
+      @code == 404
+    end
+
+    #
+    # Return true if page is fetched successfully
+    #
+    def success?
+      @code == 200
+    end
+
+    #
+    # Return true if page is redirected
+    #
+    def redirect?
+      (300..307).include?(@code)
+    end
+
+    #
+    # Return the absolute url
+    #
+    private
+    def absolutify(page_url)
+      return URI.escape(page_url) if page_url =~ /^\w*\:/i
+      return base_url + URI.escape(page_url)
+    end
+  end
+end

data/lib/spidercrawl/request.rb

@@ -0,0 +1,155 @@
+require 'spidercrawl/page'
+require 'spidercrawl/user_agents'
+require 'net/http'
+require 'curb'
+require 'colorize'
+require 'typhoeus'
+
+module Spidercrawl
+  # Makes the request to the targeted website
+  class Request
+
+    attr_accessor :uri
+
+    def initialize(url, options = {})
+      @uri = URI.parse(url)
+      @threads = options[:threads]
+      @timeout = options[:timeout]
+
+      @http = Net::HTTP.new(@uri.host, @uri.port) do |http|
+        http.open_timeout = @timeout # in seconds
+        http.read_timeout = @timeout # in seconds
+      end
+
+      @c = Curl::Easy.new(@uri.to_s) do |curl|
+        curl.headers['User-Agent'] = UserAgents.random
+      end
+    end
+
+    #
+    # Fetch a page from the given url using libcurl
+    #
+    def curl
+      puts "fetching #{@uri.to_s}".green.on_black
+      start_time = Time.now
+      begin
+        c = @c
+        c.url = @uri.to_s
+        c.perform
+        end_time = Time.now
+        case c.response_code
+        when 200 then
+          page = Page.new(@uri, response_code: c.response_code,
+                                response_head: c.header_str,
+                                response_body: c.body_str,
+                                response_time: ((end_time-start_time)*1000).round,
+                                crawled_time: (Time.now.to_f*1000).to_i)
+        when 300..307 then
+          page = Page.new(@uri, response_code: c.response_code,
+                                response_head: c.header_str,
+                                response_body: c.body_str,
+                                response_time: ((end_time-start_time)*1000).round,
+                                redirect_url:  c.redirect_url)
+        when 404 then
+          page = Page.new(@uri, response_code: c.response_code,
+                                response_time: ((end_time-start_time)*1000).round)
+        end
+      rescue Exception => e
+        puts e.inspect
+        puts e.backtrace
+      end
+    end
+
+    #
+    # Fetch a page from the given url using net/http
+    #
+    def fetch
+      puts "fetching #{@uri.to_s}".green.on_black
+      start_time = Time.now
+      begin
+        request = Net::HTTP::Get.new(@uri.request_uri)
+        request["User-Agent"] = UserAgents.random
+        response = @http.request(request) 
+        end_time = Time.now
+        case response
+        when Net::HTTPSuccess then
+          page = Page.new(@uri, response_code: response.code.to_i,
+                                response_head: response.instance_variable_get("@header"),
+                                response_body: response.body,
+                                response_time: (end_time-start_time).to_f,
+                                crawled_time: (Time.now.to_f*1000).to_i)
+        when Net::HTTPRedirection then
+          page = Page.new(@uri, response_code: response.code.to_i,
+                                response_head: response.instance_variable_get("@header"),
+                                response_body: response.body,
+                                response_time: (end_time-start_time).to_f,
+                                redirect_url:  response['location'])
+        when Net::HTTPNotFound then
+          page = Page.new(@uri, response_code: response.code.to_i,
+                                response_time: (end_time-start_time).to_f)
+        end
+      rescue Exception => e
+        puts e.inspect
+        puts e.backtrace
+      end
+    end
+  end
+
+  # Makes parallel requests to the targeted website using typhoeus and hydra
+  class ParallelRequest
+    
+    attr_accessor :urls
+
+    def initialize(urls, options = {})
+      @urls = urls
+      @threads = options[:threads]
+      @timeout = options[:timeout]
+    end
+
+    #
+    # Fetch page(s) from the given url(s)
+    #
+    def fetch
+      hydra = Typhoeus::Hydra.new(:max_concurrency => @threads)
+      page, pages = nil, []
+
+      @urls.each do |url|
+        request = Typhoeus::Request.new(url, :timeout => @timeout, :followlocation => false, :headers => {"Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Cache-Control" => "no-cache", "Pragma" => "no-cache", "User-Agent" => UserAgents.random})
+        request.on_complete do |response|
+          uri = URI(url)
+          if response.success?
+            puts "fetching #{url}".green.on_black
+            page = Page.new(uri, response_code: response.code,
+                                 response_head: response.headers,
+                                 response_body: response.body,
+                                 response_time: response.time*1000,
+                                 crawled_time: (Time.now.to_f*1000).to_i)
+          elsif (300..307).include?(response.code)
+            puts "fetching #{url}".green.on_black
+            puts "### #{response.code} ### redirect to #{response.headers['Location']}".white.on_black
+            page = Page.new(uri, response_code: response.code,
+                                 response_head: response.headers,
+                                 response_body: response.body,
+                                 response_time: response.time*1000,
+                                 redirect_url:  response.headers['Location'])
+          elsif 404 == response.code
+            puts "fetching #{url}".green.on_black
+            puts "### #{response.code} ### not found #{url}".magenta.on_black
+            page = Page.new(uri, response_code: response.code,
+                                 response_time: response.time*1000)
+          else
+            puts "fetching #{url}".green.on_black
+            puts "### #{response.code} ### failed #{url}".magenta.on_black
+            puts "### Time: #{response.time} ### #{response.return_message}".magenta.on_black
+            page = Page.new(uri, response_code: response.code,
+                                 response_time: response.time*1000)
+          end
+          pages << page
+        end
+        hydra.queue(request)
+      end
+      hydra.run
+      return pages
+    end
+  end
+end

data/lib/spidercrawl/spider_worker.rb

@@ -0,0 +1,203 @@
+require 'spidercrawl/request'
+require 'spidercrawl/page'
+require 'colorize'
+
+module Spidercrawl
+  # Start working hard
+  class SpiderWorker
+
+    attr_reader :page
+
+    def initialize(url, options = {})
+      @url = url
+      #@headers = options[:headers]
+      @delay = options[:delay] ? options[:delay] : 0 # default 0 seconds
+      @threads = options[:threads] ? options[:threads] : 10 # default 10 threads
+      @timeout = options[:timeout] ? options[:timeout] : 20 # default 20 seconds
+      @allow_redirections = options[:allow_redirections]
+      @max_pages = options[:max_pages]
+      @pattern = options[:pattern]
+      @setup = nil
+      @teardown = nil
+      @redirect = nil
+      @success = nil
+      @failure = nil
+    end
+
+    def crawl
+      link_queue = Queue.new
+      pages, visited_links = [], []
+      link_queue << @url
+
+      spider_worker = Request.new(@url, :threads => @threads, :timeout => @timeout)
+
+      begin
+        url = link_queue.pop
+        next if visited_links.include?(url) || (@pattern && url !~ @pattern)
+
+        start_time = Time.now
+        response = @setup.yield url unless @setup.nil?
+        end_time = Time.now
+        
+        spider_worker.uri = URI.parse(url)
+        page = (response ? setup_page(URI.parse(url), response, ((end_time - start_time).to_f*1000).to_i) : spider_worker.curl)
+        visited_links << url
+
+        if page.success? || page.redirect? then
+          while page.redirect?
+            puts ("### redirect to #{page.location}" + (visited_links.include?(page.location) ? " which we have already visited!" : "")).white.on_black
+            break if visited_links.include?(page.location)
+            
+            start_time = Time.now
+            response = @redirect.yield page.location unless @redirect.nil?
+            end_time = Time.now
+
+            spider_worker.uri = URI.parse(page.location)
+            page = (response ? setup_page(URI.parse(page.location), response, ((end_time - start_time).to_f*1000).to_i) : spider_worker.curl)
+            visited_links << page.url
+          end
+          if !visited_links.include?(page.location)
+            pages << page unless page.content == ""
+            page.internal_links.each do |link|
+              if !visited_links.include?(link) 
+                link_queue << link if @pattern && link =~ @pattern
+                link_queue << link unless @pattern
+              end
+            end unless page.internal_links.nil?
+            @teardown.yield page unless @teardown.nil?
+            sleep @delay
+          end
+        elsif page.not_found? then
+          puts "page not found"
+        end
+      end until link_queue.empty?
+      pages
+    end
+
+    def parallel_crawl
+      link_queue = Queue.new
+      pages, visited_links = [], []
+      link_queue << @url
+
+      spider_workers = ParallelRequest.new([@url], :threads => @threads, :timeout => @timeout)
+
+      begin
+        urls = []
+        while !link_queue.empty?
+          url = link_queue.pop
+          next if visited_links.include?(url) || (@pattern && url !~ @pattern)
+          visited_links << url
+
+          start_time = Time.now
+          response = @setup.yield url unless @setup.nil?
+          end_time = Time.now
+          
+          if response then
+            pages << (page = setup_page(URI.parse(url), response, ((end_time - start_time).to_f*1000).to_i))
+            @teardown.yield page unless @teardown.nil?
+
+            page.internal_links.each do |link| # queue up internal links for crawling 
+              if !visited_links.include?(link) 
+                link_queue << link if @pattern && link =~ @pattern
+                link_queue << link unless @pattern
+              end
+            end unless page.internal_links.nil?
+          else # queue up url for crawling 
+            urls << url
+            puts "queue: #{url}"
+          end
+        end
+
+        spider_workers.urls = urls
+        responses = spider_workers.fetch
+
+        responses.each do |page|
+          if (503..504).include?(page.response_code) then
+            link_queue << page.url
+          elsif page.success? || page.redirect? then
+            response = nil
+            if page.redirect? then
+              puts ("### redirect to #{page.location}" + (visited_links.include?(page.location) ? " which we have already visited!" : "")).white.on_black
+              unless visited_links.include?(page.location) || (@pattern && page.location !~ @pattern)
+                start_time = Time.now
+                response = @redirect.yield page.location unless @redirect.nil?
+                end_time = Time.now
+
+                if response then
+                  page = setup_page(URI.parse(page.location), response, ((end_time - start_time).to_f*1000).to_i)
+                  visited_links << page.url        
+                else
+                  puts "queue: #{page.location}"
+                  link_queue << page.location
+                end
+              else
+                puts "discard: #{page.location}"
+              end
+            end
+            if page.success? || response then
+              pages << page unless page.content == ""
+              page.internal_links.each do |link| # queue up internal links for crawling
+                if !visited_links.include?(link) 
+                  link_queue << link if @pattern && link =~ @pattern
+                  link_queue << link unless @pattern
+                end
+              end unless page.internal_links.nil?
+              page.crawled_time = (Time.now.to_f*1000).to_i
+              @teardown.yield page unless @teardown.nil?
+            end
+          elsif page.not_found? then
+            puts "page not found"
+          end
+        end
+      end until link_queue.empty?
+      pages
+    end
+
+    #
+    # Code block for before fetch
+    #
+    def before_fetch(&block)
+      @setup = block if block
+    end
+
+    #
+    # Code block for after fetch
+    #
+    def after_fetch(&block)
+      @teardown = block if block
+    end
+
+    #
+    # Code block for on redirect
+    #
+    def on_redirect(&block)
+      @redirect = block if block
+    end
+
+    #
+    # Code block for on success
+    #
+    def on_success(&block)
+      @success = block if block
+    end
+
+    #
+    # Code block for on failure
+    #
+    def on_failure(&block)
+      @failure = block if block
+    end
+
+    #
+    # Setup page based on given response
+    #
+    private
+    def setup_page(uri, response, response_time)
+      page = Page.new(uri, response_code: response.code.to_i,
+                           response_head: response.instance_variable_get("@header"),
+                           response_body: response.body,
+                           response_time: response_time,
+                           crawled_time: (Time.now.to_f*1000).to_i)
+    end
+  end
+end

data/lib/spidercrawl/user_agents.rb

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2013 Charles H Martin, PhD
+#  
+
+class UserAgents
+  # Random agents
+  def self.random
+    case rand(20)
+    when 0
+      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:#{10+rand(10)}.#{rand(10)}) Gecko/20#{10+rand(3)}#{1000+rand(3)*100+rand(28)} Firefox/20.0"
+    when 1
+      "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.#{10+rand(10)}) Gecko/20#{10+rand(3)}#{1000+rand(3)*100+rand(28)} Ubuntu/10.10 (maverick) Firefox/3.6.#{14+rand(5)}"
+    when 2
+      ver = "#{400+rand(99)}.#{10+rand(75)}"
+      "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/#{ver} (KHTML, like Gecko) Chrome/12.0.#{700+rand(90)}.#{100+rand(200)} Safari/#{ver}"
+    when 3
+      ver = "#{400+rand(99)}.#{rand(9)}"
+      "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/#{ver} (KHTML, like Gecko) Chrome/13.0.#{700+rand(90)}.#{100+rand(200)} Safari/#{ver}"
+    when 4
+      "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:5.0) Gecko/20#{10+rand(3)}#{1000+rand(3)*100+rand(28)} Firefox/#{4+rand(1)}.0"
+    when 5
+      "Mozilla/4.0 (compatible; MSIE 8.#{rand(6)}; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.#{50000+rand(7000)}; .NET CLR 3.5.#{30000+rand(8000)}; .NET CLR 3.0.#{30000+rand(8000)}; Media Center PC 6.0; .NET4.0C; .NET4.0E; MS-RTC LM 8; Zune 4.#{6+rand(3)})"
+    end
+  end
+end

data/lib/spidercrawl/version.rb

@@ -0,0 +1,3 @@
+module Spidercrawl
+  VERSION = "0.3.9"
+end

data/spidercrawl.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'spidercrawl/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "spidercrawl"
+  spec.version       = Spidercrawl::VERSION
+  spec.authors       = ["Belson Heng"]
+  spec.email         = ["belsonheng@gmail.com"]
+  spec.summary       = %q{A ruby gem that can crawl a domain and let you have information about the pages it visits.}
+  spec.description   = %q{With the help of Nokogiri, SpiderCrawl will parse each page and return you its title, links, css, words, and many many more! You can also customize what you want to do before & after each fetch request.}
+  spec.homepage      = "http://github.com/belsonheng/spidercrawl"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'nokogiri', '~> 1.6'
+  spec.add_dependency "curb"
+  spec.add_dependency "colorize"
+  spec.add_dependency "typhoeus"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: spidercrawl
+version: !ruby/object:Gem::Version
+  version: 0.3.9
+platform: ruby
+authors:
+- Belson Heng
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: curb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: With the help of Nokogiri, SpiderCrawl will parse each page and return
+  you its title, links, css, words, and many many more! You can also customize what
+  you want to do before & after each fetch request.
+email:
+- belsonheng@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/spidercrawl.rb
+- lib/spidercrawl/page.rb
+- lib/spidercrawl/request.rb
+- lib/spidercrawl/spider_worker.rb
+- lib/spidercrawl/user_agents.rb
+- lib/spidercrawl/version.rb
+- spidercrawl.gemspec
+homepage: http://github.com/belsonheng/spidercrawl
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: A ruby gem that can crawl a domain and let you have information about the
+  pages it visits.
+test_files: []