pantopoda 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2f6ca88cbcd2df8c738d31bea859f65165dd9253
+  data.tar.gz: 1b32bc531837a4343788fc8c5a5941a77fac4ce7
+SHA512:
+  metadata.gz: 93f04b120ea1b9c7615aa3b275b6eae758a781743d0e7cc2ecb1c74fc1e5bb6f76d0dba40ebd9518474400efab5b2bcaa967a0023530e44a5c5d799f21eed000
+  data.tar.gz: ee84ff6889c09c993290a996fe6a173011873661076f76717a4b6b7d43b730edff30a32fe37ffd6252f686f8b39df8f298120d607e596dfd407b6784ecd71839

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pantopoda.gemspec
+gemspec
+
+gem 'domainatrix', :source => 'http://gemcutter.org'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Gabriel Lim
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# Pantopoda
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pantopoda'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pantopoda
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/pantopoda/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/lib/pantopoda/version.rb

@@ -0,0 +1,3 @@
+module Pantopoda
+  VERSION = "0.0.1"
+end

data/lib/pantopoda.rb

@@ -0,0 +1,141 @@
+require "pantopoda/version"
+require 'typhoeus'
+require 'bloomfilter-rb'
+require 'nokogiri'
+require 'domainatrix'
+require 'uri'
+
+class Pantopoda
+	def initialize(url, options = {})
+		@start_url = url
+		@domain = parse_domain(url)
+
+		@split_url_at_hash = options[:split_url_at_hash] ? options[:split_url_at_hash] : false
+		@exclude_urls_with_hash = options[:exclude_urls_with_hash] ? options[:exclude_urls_with_hash] : false
+		@exclude_urls_with_extensions = options[:exclude_urls_with_extensions] ? options[:exclude_urls_with_extensions] : false
+	end
+
+	def crawl(options = {})
+		# Defaults to 1 thread, so we won't do a crazy amount of crawling on domains
+		threads = options[:threads] ? options[:threads] : 1
+
+		# Defaults to -1 so it will always keep running until it runs out of urls
+		max_urls = options[:max_urls] ? options[:max_urls] : nil
+
+		@hydra = Typheous::Hydra.new(:max_concurrency => threads)
+		@global_visited = BloomFilter::Native.new(:size => 1000000, :hashes => 5, :seed => 1, :bucket => 8, :raise => false)
+		@global_queue = []
+
+		@global_queue << @start_url
+
+		while(@global_queue.size > 0 && (max_urls.nil? || @global_visited.size.to_i < max_urls))
+			temp_queue = @global_queue
+			temp_queue.each do |q|
+				begin
+					ip,port,user,pass = nil
+
+					request = Typheous::Request.new(q, :timeout => 10000, :follow_location => true) if ip == nil
+					request.on_complete do |response|
+						yield response
+						links = Nokogiri::HTML.parse(response.body).xpath('.//a/@href')
+						links.each do |link|
+							if (internal_link?(link, response.effective_url) && !@global_visited.include?(make_absolute(link, response.effective_url)) && no_hash_in_url?(link) && ignore_extensions(link))
+								sanitized_link = sanitized_link(split_url_at_hash(link))
+								if (sanitized_link)
+									absolute_link = make_absolute(sanitized_link, response.effective_url)
+									if absolute_link
+										@global_queue << absolute_link
+									end
+								end
+							end
+						end
+					end
+
+					@hydra.queue request
+
+				rescue URI::InvalidURIError, NoMethodError => e
+					puts "Exception caught: #{e}" if @debug == true
+				end
+
+				@global_visited.insert(q)
+				@global_queue.delete(q)
+			end
+
+			@hydra.run
+		end
+	end
+
+	def parse_domain(url)
+		puts "Parsing URL: #{url}"
+
+		begin
+			parsed_domain = Domainatrix.parse(url)
+			if (parsed_domain.subdomain != "")
+				parsed_domain.subdomain + '.' + parsed_domain + '.' + parse_domain.public_suffix
+			else
+				parse_domain.domain + '.' + parsed_domain.public_suffix
+			end
+
+		rescue NoMethodError, Addressable::URI::InvalidURIError => e
+			puts "URL Parsing Exception (#{url}) : #{e}"
+			return nil
+		end
+	end
+
+	def internal_link?(url, effective_url)
+		absolute_url = make_absolute(url, effective_url)
+		parsed_url = parse_domain(absolute_url)
+		if (@domain == parsed_url)
+			return true
+		else
+			return false
+		end
+	end
+
+	def split_url_at_hash(url)
+		return url.to_s unless @split_url_at_hash
+		return url.to_s.split('#')[0]
+	end
+
+	def no_hash_in_url?(url)
+		return true unless @exclude_urls_with_hash
+
+		if(url.to_s.scan(/#/).size > 0)
+			return false
+		else
+			return true
+		end
+	end
+
+	def ignore_extensions(url)
+		return true if url.to_s.length == 0
+		return true unless @exclude_urls_with_extensions
+
+		not_found = true
+
+		@exclude_urls_with_extensions.each do |e|
+			if(url.to_s.length > e.size && url.to_s[-e.size .. -1].downcase == e.to_s.downcase)
+				not_found = false
+				puts "#{e} Found At URL: #{url}"
+			end
+		end
+
+		return not_found
+	end
+
+	def sanitize_link(url)
+		begin
+			return url.gsub(/\s+/, "%20")
+		rescue
+			return false
+		end
+	end
+
+	def make_absolute(href, root)
+		begin
+	  		URI.parse(root).merge(URI.parse(split_url_at_hash(href.to_s.gsub(/\s+/, "%20")))).to_s
+	  	rescue URI::InvalidURIError, URI::InvalidComponentError => e
+	  		return false
+	  	end
+	end
+end

data/pantopoda.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pantopoda/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "pantopoda"
+  spec.version       = Pantopoda::VERSION
+  spec.authors       = ["Gabriel Lim"]
+  spec.email         = ["gabriel@saleswhale.io"]
+  spec.summary       =  "Efficient domain web spider."
+  spec.description   =  "Pantopoda is a web crawler that visits all links on a given domain that's fast and effective."
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_dependency('typhoeus',    '0.7.2')
+  spec.add_dependency('bloomfilter-rb',    '2.1.1')
+  spec.add_dependency('nokogiri',    '1.6.2')
+  spec.add_dependency('domainatrix')
+
+end

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification
+name: pantopoda
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Gabriel Lim
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.2
+- !ruby/object:Gem::Dependency
+  name: bloomfilter-rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.2
+- !ruby/object:Gem::Dependency
+  name: domainatrix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Pantopoda is a web crawler that visits all links on a given domain that's
+  fast and effective.
+email:
+- gabriel@saleswhale.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/pantopoda.rb
+- lib/pantopoda/version.rb
+- pantopoda.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Efficient domain web spider.
+test_files: []