arachnidish 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cbba3f96eda4cf0065a0b7d65c373bbac54b55bf
+  data.tar.gz: ebc7dd1e5ae6432c83534f94f77057b0cb2716f3
+SHA512:
+  metadata.gz: 3cf72eae18e2c7c9c43a8405f9ad0dc88f131df092d35fb550348cea3e3a84bae4f37bc666e7683c0a459ab19b09cbda380c9fe859b70edfa5870500fa838099
+  data.tar.gz: b7e0f862dd41359c9e994c8b6c4f993f0b95e0a0feeb3afe62d3aa9e08697c140d63948b21be57cbcd615aab5895a4a94e5fbaa0657b900f6ebcf674941e6675

data/lib/arachnidish.rb

@@ -0,0 +1,174 @@
+# encoding: utf-8
+
+require 'typhoeus'
+require 'bloomfilter-rb'
+require 'nokogiri'
+require 'domainatrix'
+require 'uri'
+
+class Arachnidish
+
+	def initialize(url, options = {})
+		@start_url = url
+		@domain = parse_domain(url)
+
+		@split_url_at_hash = options[:split_url_at_hash] ? options[:split_url_at_hash] : false
+		@exclude_urls_with_hash = options[:exclude_urls_with_hash] ? options[:exclude_urls_with_hash] : false
+		@exclude_urls_with_extensions = options[:exclude_urls_with_extensions] ? options[:exclude_urls_with_extensions] : false
+		@proxy_list = options[:proxy_list] ? options[:proxy_list] : false
+
+		@debug = options[:debug] ? options[:debug] : false
+	end
+
+	def crawl(options = {})
+
+		#defaults to 1 thread so people don't do a stupid amount of crawling on unsuspecting domains
+		threads = options[:threads] ? options[:threads] : 1
+		#defaults to -1 so it will always keep running until it runs out of urls
+		max_urls = options[:max_urls] ? options[:max_urls] : nil
+
+		@hydra = Typhoeus::Hydra.new(:max_concurrency => threads)
+		@global_visited = BloomFilter::Native.new(:size => 1000000, :hashes => 5, :seed => 1, :bucket => 8, :raise => false)
+		@global_queue = []
+
+		@global_queue << @start_url
+
+		while(@global_queue.size > 0 && (max_urls.nil? || @global_visited.size.to_i < max_urls))
+			temp_queue = @global_queue
+
+			temp_queue.each do |q|
+
+				begin
+					ip,port,user,pass = grab_proxy
+
+					request = Typhoeus::Request.new(q, :timeout => 10000, :follow_location => true) if ip == nil
+					request = Typhoeus::Request.new(q, :timeout => 10000, :follow_location => true, :proxy => "#{ip}:#{port}") if ip != nil && user == nil
+					request = Typhoeus::Request.new(q, :timeout => 10000, :follow_location => true, :proxy => "#{ip}:#{port}", :proxy_username => user, :proxy_password => pass) if user != nil
+
+					request.on_complete do |response|
+
+						yield response
+
+						links = Nokogiri::HTML.parse(response.body).xpath('.//a/@href')
+
+						links.each do |link|
+							if(internal_link?(link, response.effective_url) && !@global_visited.include?(make_absolute(link, response.effective_url)) && no_hash_in_url?(link) && ignore_extensions(link))
+
+								sanitized_link = sanitize_link(split_url_at_hash(link))
+								if(sanitized_link)
+
+									absolute_link = make_absolute(sanitized_link, response.effective_url)
+									if(absolute_link)
+										@global_queue << absolute_link
+									end
+								end
+							end
+						end
+
+					end
+
+					@hydra.queue request
+
+				rescue URI::InvalidURIError, NoMethodError => e
+					puts "Exception caught: #{e}" if @debug == true
+				end
+
+				@global_visited.insert(q)
+				@global_queue.delete(q)
+
+			end
+
+			@hydra.run
+
+		end
+
+	end
+
+	def grab_proxy
+
+		return nil unless @proxy_list
+
+		return @proxy_list.sample.split(':')
+
+	end
+
+	def parse_domain(url)
+		puts "Parsing URL: #{url}" if @debug
+
+		begin
+			parsed_domain = Domainatrix.parse(url)
+
+			if(parsed_domain.subdomain != "")
+				parsed_domain.subdomain + '.' + parsed_domain.domain + '.' + parsed_domain.public_suffix
+			else
+				parsed_domain.domain + '.' + parsed_domain.public_suffix
+			end
+		rescue NoMethodError, Addressable::URI::InvalidURIError => e
+			puts "URL Parsing Exception (#{url}): #{e}"
+			return nil
+		end
+	end
+
+	def internal_link?(url, effective_url)
+
+		absolute_url = make_absolute(url, effective_url)
+
+		parsed_url = parse_domain(absolute_url)
+		if(@domain == parsed_url)
+			return true
+		else
+			return false
+		end
+	end
+
+	def split_url_at_hash(url)
+		return url.to_s unless @split_url_at_hash
+
+		return url.to_s.split('#')[0]
+
+	end
+
+	def no_hash_in_url?(url)
+		return true unless @exclude_urls_with_hash
+
+		if(url.to_s.scan(/#/).size > 0)
+			return false
+		else
+			return true
+		end
+	end
+
+	def ignore_extensions(url)
+		return true if url.to_s.length == 0
+		return true unless @exclude_urls_with_extensions
+
+		not_found = true
+
+		@exclude_urls_with_extensions.each do |e|
+			if(url.to_s.length > e.size && url.to_s[-e.size .. -1].downcase == e.to_s.downcase)
+				not_found = false
+				puts "#{e} Found At URL: #{url}" if @debug
+			end
+		end
+
+		return not_found
+	end
+
+	def sanitize_link(url)
+		begin
+			return url.gsub(/\s+/, "%20")
+		rescue
+			return false
+		end
+	end
+
+	def make_absolute( href, root )
+
+		begin
+	  		URI.parse(root).merge(URI.parse(split_url_at_hash(href.to_s.gsub(/\s+/, "%20")))).to_s
+	  	rescue URI::InvalidURIError, URI::InvalidComponentError => e
+	  		return false
+	  	end
+	end
+
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: arachnidish
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- csphere
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+- !ruby/object:Gem::Dependency
+  name: bloomfilter-rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: domainatrix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Arachnidish is a web crawler that relies on Bloom Filters to efficiently
+  store visited urls and Typhoeus to avoid the overhead of Mechanize when crawling
+  every page on a domain.
+email: cspheregreen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/arachnidish.rb
+homepage: https://github.com/csphere/Arachnid
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: Extremely fast and efficient domain spider, that isn't quite so strict
+test_files: []
+has_rdoc: