rubyretriever 0.0.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e4e4773d62ec74a55bb3d9afc78622229e7db559
+  data.tar.gz: a2186b11bb3eabdec8c312e0b3365961c97805de
+SHA512:
+  metadata.gz: ae392a910a3c7a6f2b3f9097d82978ba35ba413da46f4e5ce30b9e728bac2fb40f61c3a110ea59b9f59d75d79b0fcf85a8a783597aeaccd31c833b2c3753bd4d
+  data.tar.gz: 72fe87613059ccae6022c65ad5db70445cc4470028d081e918ffb9dfc122adb5dd8c9967abe2622a8daaeea59d878340183597a73768f32dd42679b7b159d64d

data/LICENSE

@@ -0,0 +1,20 @@
+2014 (c) Joseph Michael Norton - 'Joe Norton' - SoftwareByJoe.com
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/rr

@@ -0,0 +1,80 @@
+#! /usr/bin/env ruby
+require_relative('../lib/retriever.rb')
+options = {}
+ optparse = OptionParser.new do|opts|
+   # Set a banner, displayed at the top
+   # of the help screen.
+   opts.banner = "Usage: rr [options] Target_URL"
+ 
+    options[:filename] = nil
+   opts.on( '-o', '--out FILENAME', 'Dump output to selected filename' ) do|filename|
+     options[:filename] = filename
+   end
+   # Define the options, and what they do
+   options[:verbose] = false
+   opts.on( '-v', '--verbose', 'Output more information' ) do
+     options[:verbose] = true
+   end
+
+   options[:progress] = false
+   opts.on( '-p', '--progressbar', 'Output more information' ) do
+     options[:progress] = true
+   end
+ 
+  options[:sitemap] = false
+   opts.on( '-s', '--sitemap', 'Crawl site and output sitemap' ) do
+     options[:sitemap] = true
+   end
+ 
+  options[:fileharvest] = false
+   opts.on( '-f', '--files', 'Crawl site and collect links for files found' ) do
+     options[:fileharvest] = true
+   end
+
+  options[:maxpages] = false
+   opts.on( '-l', '--limit PAGE_LIMIT_#', 'set a max on the total number of crawled pages' ) do |maxpages|
+     options[:maxpages] = maxpages
+   end
+
+  options[:file_ext] = false
+   opts.on( '-e', '--ext FILE_EXTENSION', 'set a file extension to look for on crawled pages' ) do |file_ext|
+     options[:file_ext] = file_ext
+   end
+
+   options[:autodown] = false
+   opts.on( '-a', '--autodown', 'Automatically download all files of filetype located' ) do
+     options[:autodown] = true
+   end
+ 
+   # This displays the help screen, all programs are
+   # assumed to have this option.
+   opts.on( '-h', '--help', 'Display this screen' ) do
+     puts opts
+     exit
+   end
+ end
+ 
+ optparse.parse!
+if ARGV[0].nil?
+	abort("###Missing Required Argument\nUsage: rr [mode] [options] Target_URL")
+end
+
+ARGV.each do|q|
+  if options[:verbose]
+    puts "###############################"
+    puts "### [RubyRetriever]"
+    puts "### Creating Sitemap" if options[:sitemap]
+    puts "### Performing File Harvest" if options[:fileharvest]
+    puts "### Searching for file extension: #{options[:file_ext]} pages" if (options[:file_ext])
+    puts "### Writting output to filename: #{options[:filename]}" if options[:filename]
+    puts "### Being verbose"
+    puts "### Stopping after #{options[:maxpages]} pages" if options[:maxpages]
+  end
+  puts "###############################"
+  puts "### [RubyRetriever] go fetch #{q}"
+  test = Retriever::FetchFiles.new(q, options) if options[:fileharvest]
+  test = Retriever::FetchSitemap.new(q, options)  if options[:sitemap]
+  puts "### [RubyRetriever] is done."
+  puts "###############################"
+  puts
+end

data/lib/retriever.rb

@@ -0,0 +1,19 @@
+##################################################################
+#####RubyRetriever -- web crawler and file harvester
+#####created by Joe Norton
+#####http://softwarebyjoe.com
+##LICENSING: GNU GPLv3  License##################################
+#! usr/bin/ruby
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+require 'em-synchrony/fiber_iterator'
+require 'ruby-progressbar'
+require 'open-uri'
+require 'optparse'
+require 'uri'
+require 'csv'
+require 'bloomfilter-rb'
+
+require 'retriever/fetch'
+require 'retriever/fetchfiles'
+require 'retriever/fetchsitemap'

data/lib/retriever/fetch.rb

@@ -0,0 +1,196 @@
+module Retriever
+	class Fetch
+		attr_reader :target, :host, :host_re, :maxPages
+		#constants
+		HTTP_RE = Regexp.new(/^http/i).freeze
+		HREF_CONTENTS_RE = Regexp.new(/\shref=['|"]([^\s][a-z0-9\.\/\:\-\%\+\?\!\=\&\,\:\;\~\_]+)['|"][\s|\W]/ix).freeze
+		NONPAGE_EXT_RE = Regexp.new(/\.(?:css|js|png|gif|jpg|mp4|wmv|flv|mp3|wav|doc|txt|ico)/ix).freeze
+		SINGLE_SLASH_RE = Regexp.new(/^\/{1}[^\/]/).freeze
+		DOUBLE_SLASH_RE = Regexp.new(/^\/{2}[^\/]/).freeze
+		NO_SLASH_PAGE_RE = Regexp.new(/^[a-z0-9\-\_\=\?\.]+\z/ix).freeze
+		DUB_DUB_DUB_DOT_RE = Regexp.new(/^www\./i).freeze
+
+		def initialize(url,options)
+			new_uri = URI(url)
+			@target = new_uri.to_s
+			@host = new_uri.host
+			#OPTIONS
+			@prgrss = options[:progress] ? options[:progress] : false
+			@maxPages = options[:maxpages] ? options[:maxpages].to_i : 100
+			@v= options[:verbose] ? true : false
+			@output=options[:filename] ? options[:filename] : false
+			@fh = options[:fileharvest] ? true : false
+			@s = options[:sitemap] ? true : false
+			@file_ext = options[:file_ext] ? options[:file_ext] : false
+			@autodown = options[:autodown] ? true : false
+			#
+			@host_re = Regexp.new(host).freeze
+			if @fh
+				errlog("Please provide a FILETYPE. It is required for file harvest mode.") if !@file_ext
+				tempExtStr = "."+@file_ext+'\z'
+				@file_re = Regexp.new(tempExtStr).freeze
+			else
+				errlog("Cannot AUTODOWNLOAD when not in FILEHARVEST MODE") if @autodown #when FH is not true, and autodown is true
+			end
+			if @prgrss
+				errlog("CANNOT RUN VERBOSE & PROGRESSBAR AT SAME TIME, CHOOSE ONE, -v or -p") if @v #verbose & progressbar conflict
+				prgressVars = {
+					:title => "Pages Crawled",
+					:starting_at => 1,
+					:total => @maxPages,
+					:format => '%a |%b>%i| %c/%C %t',
+				}
+				@progressbar = ProgressBar.create(prgressVars)
+			end
+			@already_crawled = BloomFilter::Native.new(:size => 1000000, :hashes => 5, :seed => 1, :bucket => 8, :raise => false)
+			@already_crawled.insert(@target)
+		end
+		def errlog(msg)
+			raise "ERROR: #{msg}"
+		end
+		def lg(msg)
+			puts "### #{msg}" if @v
+		end
+		def dump(data)
+			puts "###############################"
+			if @s
+				puts "#{@target} Sitemap"
+				puts "Page Count: #{data.size}"
+			elsif @fh
+				puts "Target URL: #{@target}"
+				puts "Filetype: #{@file_ext}"
+				puts "File Count: #{data.size}"
+			else
+				puts "ERROR"
+			end
+			puts "###############################"
+			puts data
+			puts "###############################"
+			puts
+		end
+		def write(data)
+			if @output
+				CSV.open("#{@output}.csv", "w") do |csv|
+				  data.each do |entry|
+				  	csv << [entry]
+				  end
+				end
+				puts "###############################"
+				puts "File Created: #{filename}.csv"
+				puts "Object Count: #{data.size}"
+				puts "###############################"
+				puts
+			end
+		end
+		def fetchPage(url)
+			resp = false
+			EM.synchrony do
+				begin
+					resp = EventMachine::HttpRequest.new(url).get
+				rescue StandardError => e
+					#puts e.message + " ## " + url
+					#the trap abrt is nescessary to handle the SSL error
+					#for some ungodly reason it's the only way I found to handle it
+					trap("ABRT"){
+						puts "#{url} failed SSL Certification Verification"
+					}
+					return false
+				end
+				lg("URL Crawled: #{url}")
+		    	EventMachine.stop
+			end
+			if resp.response == ""
+				errlog("Domain is not working. Try the non-WWW version.")
+			end
+			return resp.response.encode('UTF-8', :invalid => :replace, :undef => :replace) #.force_encoding('UTF-8') #ran into issues with some sites without forcing UTF8 encoding, and also issues with it. Not sure atm.
+		end
+		#recieves page source as string
+		#returns array of unique href links
+		def fetchLinks(doc)
+			return false if !doc
+			linkArray = []
+			doc.scan(HREF_CONTENTS_RE) do |arr|  #filter some malformed URLS that come in, this is meant to be a loose filter to catch all reasonable HREF attributes.
+				link = arr[0]
+				if (!(HTTP_RE =~ link))
+					if (DUB_DUB_DUB_DOT_RE =~ link)
+						link = "http://#{link}"
+					elsif SINGLE_SLASH_RE =~ link #link uses relative path
+						link = "http://#{@host}"+link #appending hostname to relative paths
+					elsif DOUBLE_SLASH_RE =~ link #link begins with '//' (maybe a messed up link?)
+						link = "http:#{link}" #appending current url to relative paths
+					elsif (NO_SLASH_PAGE_RE =~ link) #link uses relative path with no slashes at all, people actually this - imagine that.
+						link = "http://#{@host}"+"/"+link #appending hostname and slashy to create full paths
+					else
+						next
+					end
+				end
+				linkArray.push(link)
+			end
+			linkArray.uniq!
+		end
+		def parseInternalLinks(all_links)
+			if all_links
+				all_links.select{ |linky| (@host_re =~ linky && (!(NONPAGE_EXT_RE =~linky)))}
+			else
+				return false
+			end
+		end
+		def async_crawl_and_collect()
+			while (@already_crawled.size < @maxPages)
+				if @linkStack.empty?
+					if @prgrss
+						@progressbar.log("Can't find any more links. Site might be completely mapped.")
+					else
+						lg("Can't find any more links. Site might be completely mapped.")
+					end
+					break;
+				end
+				#puts "New loop"
+				#puts @linkStack
+				new_links_arr = self.asyncGetWave()
+				next if (new_links_arr.nil? || new_links_arr.empty?)
+				new_link_arr = new_links_arr-@linkStack#set operations to see are these in our previous visited pages arr?
+				@linkStack.concat(new_links_arr)
+				@sitemap.concat(new_links_arr) if @s
+			end
+		end
+		def asyncGetWave() #send a new wave of GET requests, using current @linkStack
+			new_stuff = []
+			EM.synchrony do
+				lenny = 0
+			    concurrency = 10
+			    EM::Synchrony::FiberIterator.new(@linkStack, concurrency).each do |url|
+			    	next if (@already_crawled.size >= @maxPages)
+			    	if @already_crawled.include?(url)
+			    		@linkStack.delete(url)
+			    		next
+			    	else
+			    		@already_crawled.insert(url)
+			    	end
+			    	resp = EventMachine::HttpRequest.new(url).get
+					lg("URL Crawled: #{url}")
+					if @prgrss
+						@progressbar.increment if @already_crawled.size < @maxPages
+					end
+					new_links_arr = self.fetchLinks(resp.response)
+					if new_links_arr
+						lg("#{new_links_arr.size} new links found")
+						internal_links_arr = self.parseInternalLinks(new_links_arr)
+						new_stuff.push(internal_links_arr)
+						if @fh
+							filez = self.parseFiles(new_links_arr)
+							@fileStack.concat(filez) if !filez.empty?
+							lg("#{filez.size} files found")
+						end
+					end
+			    end
+			    new_stuff = new_stuff.flatten # all completed requests
+			    EventMachine.stop
+			end
+			new_stuff.uniq!
+		end
+		def parseFiles(all_links)
+			all_links.select{ |linky| (@file_re =~ linky)}
+		end
+	end
+end

data/lib/retriever/fetchfiles.rb

@@ -0,0 +1,70 @@
+module Retriever
+	class FetchFiles < Fetch
+		attr_reader :fileStack
+		def initialize(url,options)
+			super
+			@fileStack = []
+			all_links = self.fetchLinks(fetchPage(@target))
+			@linkStack = self.parseInternalLinks(all_links)
+			self.lg("#{@linkStack.size-1} new links found")
+
+			tempFileCollection = self.parseFiles(all_links)
+			@fileStack.concat(tempFileCollection) if tempFileCollection.size>0
+			self.lg("#{@fileStack.size} new files found")
+			errlog("Bad URL -- #{@target}") if !@linkStack
+
+			@linkStack.delete(@target) if @linkStack.include?(@target)
+			@linkStack = @linkStack.take(@maxPages) if (@linkStack.size+1 > @maxPages)
+
+			self.async_crawl_and_collect()
+
+			@fileStack.sort_by! {|x| x.length}
+			@fileStack.uniq!
+
+			self.dump(self.fileStack)
+			self.write(@output,self.fileStack) if @output
+			self.autodownload()
+		end
+		def download_file(path)
+			arr = path.split('/')
+			shortname = arr.pop
+			puts "Initiating Download to: #{'/rr-downloads/' + shortname}"
+			File.open(shortname, "wb") do |saved_file|
+			  # the following "open" is provided by open-uri
+			  open(path) do |read_file|
+			    saved_file.write(read_file.read)
+			  end
+			end
+			puts "	SUCCESS: Download Complete"
+		end
+		def autodownload()
+			lenny = @fileStack.count
+			puts "###################"
+			puts "### Initiating Autodownload..."
+			puts "###################"
+			puts "#{lenny} - #{@file_ext}'s Located"
+			puts "###################"
+			if File::directory?("rr-downloads")
+			 Dir.chdir("rr-downloads")
+			else
+			puts "creating rr-downloads Directory"
+			 Dir.mkdir("rr-downloads")
+			 Dir.chdir("rr-downloads")
+			end
+			file_counter = 0
+			@fileStack.each do |entry|
+				begin	
+					self.download_file(entry)
+					file_counter+=1
+					lg("		File [#{file_counter} of #{lenny}]")
+					puts
+				rescue StandardError => e
+					puts "ERROR: failed to download - #{entry}"
+					puts e.message
+					puts
+				end
+			end
+			Dir.chdir("..")
+		end
+	end
+end

data/lib/retriever/fetchsitemap.rb

@@ -0,0 +1,25 @@
+module Retriever
+	class FetchSitemap < Fetch
+		attr_reader :sitemap
+		def initialize(url,options)
+			super
+			@sitemap = [@target]
+			@linkStack = self.parseInternalLinks(self.fetchLinks(fetchPage(@target)))
+			self.lg("#{@linkStack.size-1} new links found")
+			errlog("Bad URL -- #{@target}") if !@linkStack
+
+			@linkStack.delete(@target) if @linkStack.include?(@target)
+			@linkStack = @linkStack.take(@maxPages) if (@linkStack.size+1 > @maxPages)
+			@sitemap.concat(@linkStack)
+
+			self.async_crawl_and_collect()
+
+			@sitemap.sort_by!	 {|x| x.length} if @sitemap.size>1
+			@sitemap.uniq!
+			@sitemap = @sitemap.take(@maxPages) if (@sitemap.size+1 > @maxPages)
+
+			self.dump(self.sitemap)
+			self.write(@output,self.sitemap) if @output
+		end
+	end
+end

data/lib/retriever/version.rb

@@ -0,0 +1,3 @@
+module Retriever
+  VERSION = '0.0.8'
+end

data/readme.md

@@ -0,0 +1,79 @@
+RubyRetriever  [![Gem Version](https://badge.fury.io/rb/RubyRetriever.svg)](http://badge.fury.io/rb/RubyRetriever)  
+==============
+
+Now an official RubyGem! --make sure to use camel-casing--    
+```sh
+gem install RubyRetriever  
+```  
+
+Update (5/25):  
+ Version 0.06 - Switches to using a Bloom Filter to keep track of past 'visited pages'. I saw this in [Arachnid] (https://github.com/dchuk/Arachnid) and realized it's a much better idea for performance and implemented it immediately. Hat tip [dchuk] (https://github.com/dchuk/)  
+
+About
+=====
+
+RubyRetriever is a Web Crawler, Site Mapper, File Harvester & Autodownloader, and all around nice buddy to have around.  
+Soon to add some high level scraping options.  
+
+RubyRetriever uses aynchronous HTTP requests, thanks to eventmachine and Synchrony fibers, to crawl webpages *very quickly*.  
+
+This is the 2nd or 3rd reincarnation of the RubyRetriever autodownloader project. It started out as a executable autodownloader, intended for malware research. From there it has morphed to become a more well-rounded web-crawler and general purpose file harvesting utility.  
+
+RubyRetriever does NOT respect robots.txt, and RubyRetriever currently - by default - launches up to 10 parallel GET requests at once. This is a feature, do not abuse it. Use at own risk.
+
+  
+HOW IT WORKS 
+-----------
+```sh
+gem install RubyRetriever  
+rr [MODE] [OPTIONS] Target_URL  
+```  
+   
+ **Site Mapper**  
+```sh
+rr --sitemap --progress --limit 1000 --output cnet http://www.cnet.com
+```  
+OR -- SAME COMMAND  
+```sh
+rr -s -p -l 1000 -o cnet http://www.cnet.com
+```  
+  
+This would go to http://www.cnet.com and map it until it crawled a max of 1,000 pages, and then it would write it out to a csv named cnet.  
+  
+ **File Harvesting**  
+```sh
+rr --files --ext pdf --progress --limit 1000 --output hubspot http://www.hubspot.com
+```  
+OR -- SAME COMMAND  
+```sh
+rr -f -e pdf -p -l 1000 -o hubspot http://www.hubspot.com
+```  
+  
+This would go to http://www.hubspot.com and crawl it looking for filetype:PDF until it crawled a max of 1,000 pages, and then it would write out a list of filepaths to a csv named hubspot, and then it would go ahead and try and download each of those files to a new 'rr-downloads' folder  
+  
+
+command-line arguments
+-----------------------
+Usage: rr [MODE] [OPTIONS] Target_URL  
+
+Where MODE FLAG is either:  
+	-s, --sitemap  
+	-f, --files
+  
+and OPTIONS is the applicable:  
+    -o, --out FILENAME                  *Dump output to selected filename*  
+    -p, --progress						*Outputs a progressbar*  
+    -v, --verbose                       *Output more information*  
+    -l, --limit PAGE_LIMIT_#            *set a max on the total number of crawled pages*  
+    -e, --ext FILE_EXTENSION            *set a file extension to look for on crawled pages*  
+    -h, --help                          *Display this screen*  
+  
+Current Requirements
+------------ 
+em-synchrony  
+ruby-progressbar  
+bloomfilter-rb  
+
+License
+-------  
+See included 'LICENSE' file. It's the MIT license.

data/spec/retriever_spec.rb

@@ -0,0 +1,65 @@
+require_relative '../lib/retriever'
+
+r = Retriever::Fetch.new("http://www.cnet.com/reviews/",{:file_ext => "exe",:maxpages => "100"})
+test_html = "<a href='www.cnet.com/download.exe'>download</a>
+http://www.google.com 
+<a href='/test.html'>test</a>
+<a href='http://www.cnet.com/products/gadgets#view-comments'>gadgets comments</a>
+<a href='http://www.cnet.com/products/gadgets/' data-vanity-rewritten='true'></a>
+<a href='http://www.cnet.com/products/gadgets/'>gadgets </a>
+ <a href='http://www.yahoo.com/test/'>yahoo</a> 
+ test.com
+ <link rel='stylesheet' id='gforms_reset_css-css'  href='http://www.cnet.com/wp-content/plugins/gravityforms/css/formreset.css?ver=1.7.12' type='text/css' media='all' />
+ <a href='cpage_18'>about</a>"
+
+doc = r.fetchPage(r.target)
+links_collection = r.fetchLinks(test_html)
+filtered_links = r.parseInternalLinks(links_collection)
+file_list = r.parseFiles(links_collection)
+
+describe "Fetch" do
+
+	describe "#new" do
+		it "sets target, host, and max page vars" do
+			expect(r.target).to eq("http://www.cnet.com/reviews/")
+			expect(r.host).to eq("www.cnet.com")
+			expect(r.maxPages).to eq(100)
+		end
+	end
+
+	describe "#fetchPage" do
+		it "opens URL and returns source as String" do
+			expect(doc.class).to eq(String)
+		end
+	end
+
+	describe "#fetchLinks" do
+		it "collects all unique href links on the page" do
+			expect(links_collection).to have(6).items
+		end
+		it "returns relative urls with full path based on hostname" do
+			expect(links_collection).to include("http://www.cnet.com/test.html","http://www.cnet.com/cpage_18")
+		end
+	end
+
+	describe "#parseInternalLinks" do
+		it "filters links by host" do
+			filtered_links.each do |link| 
+				expect(link).to include(r.host)
+			end
+		end
+		it "filters out 'unvisitable' URLS like JS, Stylesheets, Images" do
+			filtered_links.each do |link| 
+				expect(link).to_not (include(".css",".js",".png",".gif",".jpg"))
+			end
+		end
+	end
+	describe "#parseFiles" do
+		it "filters links by filetype" do
+			file_list.each do |link|
+				expect(link).to include(".exe")
+			end
+		end
+	end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,153 @@
+--- !ruby/object:Gem::Specification
+name: rubyretriever
+version: !ruby/object:Gem::Version
+  version: 0.0.8
+platform: ruby
+authors:
+- Joe Norton
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: em-synchrony
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bloomfilter-rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.14'
+description: General purpose web crawler, site mapper, and file harvester
+email:
+- joe@softwarebyjoe.com
+executables:
+- rr
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- bin/rr
+- lib/retriever.rb
+- lib/retriever/fetch.rb
+- lib/retriever/fetchfiles.rb
+- lib/retriever/fetchsitemap.rb
+- lib/retriever/version.rb
+- readme.md
+- spec/retriever_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/joenorton/rubyretriever
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.8.6
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: rubyretriever
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Ruby Web Crawler & File Harvester
+test_files: []