sengi 0.1.0

data/lib/sengi/crawler_worker.rb

@@ -0,0 +1,16 @@
+
+module TheFox
+	module Sengi
+		
+		class CrawlerWorker
+			@queue = :crawler
+			
+			def self.perform(url, options)
+				crawler = Crawler.new(url, options)
+				crawler.go
+			end
+			
+		end
+		
+	end
+end

data/lib/sengi/uri.rb

@@ -0,0 +1,288 @@
+
+require 'uri'
+require 'digest'
+
+module TheFox
+	module Sengi
+		
+		class Uri
+			
+			URI_CLASSES = [URI::Generic, URI::HTTP, URI::HTTPS]
+			
+			def initialize(url)
+				@uri = nil
+				@hash = nil
+				@is_blacklisted = nil
+				@is_ignored = nil
+				@is_ignored_reason = 'nil'
+				@hash_id_key_name = nil
+				@id = nil
+				@key_name = nil
+				@domain_nowww = nil
+				@domain_nowww_hash = nil
+				@domain_original_hash = nil
+				@domain_hash_id_key_name = nil
+				@domain_id = nil
+				@domain_key_name = nil
+				@request_id = nil
+				@request_key_name = nil
+				@response_id = nil
+				@response_key_name = nil
+				@response_size = 0
+				@response_content_type = ''
+				
+				begin
+					@uri = URI(url)
+				rescue Exception => e
+					@uri = nil
+				end
+				
+				validate
+				if is_valid?
+					append_slash
+					host_downcase
+					remove_fragment
+					domain_setup
+					
+					@uri_class = @uri.class
+					@hash = Digest::SHA256.hexdigest(to_s)
+					@hash_id_key_name = "urls:id:#{@hash}"
+				end
+			end
+			
+			def is_valid?
+				!@uri.nil?
+			end
+			
+			def ruri
+				@uri
+			end
+			
+			def is_blacklisted=(is_blacklisted)
+				@is_blacklisted = is_blacklisted
+			end
+			
+			def is_blacklisted
+				@is_blacklisted
+			end
+			
+			def is_ignored=(is_ignored)
+				@is_ignored = is_ignored
+			end
+			
+			def is_ignored
+				@is_ignored
+			end
+			
+			def is_ignored_reason=(is_ignored_reason)
+				@is_ignored_reason = is_ignored_reason
+			end
+			
+			def is_ignored_reason
+				@is_ignored_reason
+			end
+			
+			# def hash_id_key_name=(hash_id_key_name)
+			# 	@hash_id_key_name = hash_id_key_name
+			# end
+			
+			def hash_id_key_name
+				@hash_id_key_name
+			end
+			
+			def id=(id)
+				@id = id
+				@key_name = "urls:#{@id}"
+			end
+			
+			def id
+				@id
+			end
+			
+			# def key_name=(key_name)
+			# 	@key_name = key_name
+			# end
+			
+			def key_name
+				@key_name
+			end
+			
+			def domain_nowww
+				@domain_nowww
+			end
+			
+			def domain_nowww_hash
+				@domain_nowww_hash
+			end
+			
+			def domain_original_hash
+				@domain_original_hash
+			end
+			
+			def domain_hash_id_key_name
+				@domain_hash_id_key_name
+			end
+			
+			def domain_id=(domain_id)
+				@domain_id = domain_id
+				@domain_key_name = "domains:#{@domain_id}"
+			end
+			
+			def domain_id
+				@domain_id
+			end
+			
+			def domain_key_name
+				@domain_key_name
+			end
+			
+			def request_id=(request_id)
+				@request_id = request_id
+				@request_key_name = "requests:#{@request_id}"
+			end
+			
+			def request_id
+				@request_id
+			end
+			
+			def request_key_name
+				@request_key_name
+			end
+			
+			def response_id=(response_id)
+				@response_id = response_id
+				@response_key_name = "responses:#{@response_id}"
+			end
+			
+			def response_id
+				@response_id
+			end
+			
+			def response_key_name
+				@response_key_name
+			end
+			
+			def response_size=(response_size)
+				@response_size = response_size.to_s
+			end
+			
+			def response_size
+				@response_size
+			end
+			
+			def response_content_type=(response_content_type)
+				@response_content_type = response_content_type.to_s
+			end
+			
+			def response_content_type
+				@response_content_type
+			end
+			
+			def to_s
+				"#{@uri}"
+			end
+			
+			def to_hash
+				@hash
+			end
+			
+			def to_http
+				http_uri = @uri.clone
+				http_uri.scheme = 'http'
+				http_uri
+			end
+			
+			def weight(ref_uri = nil)
+				is_subdomain = false
+				
+				if !@uri.host.nil? && !ref_uri.nil? && !ref_uri.ruri.host.nil?
+					#puts "#{@uri.host}"
+					#puts "#{ref_uri.ruri.host}"
+					
+					a_ss = @uri.host[ref_uri.ruri.host]
+					#puts "a: '#{a_ss}'"
+					
+					if a_ss.nil?
+						b_ss = ref_uri.ruri.host[@uri.host]
+						#puts "b: '#{b_ss}'"
+						
+						if !b_ss.nil?
+							is_subdomain = true
+						end
+					else
+						is_subdomain = true
+					end
+				end
+				
+				if false
+				elsif @uri_class == URI::Generic then return 100
+				elsif @uri_class == URI::HTTP
+					if is_subdomain
+						return 200
+					end
+					return 250
+				elsif @uri_class == URI::HTTPS then return 290
+				end
+				return 999
+			end
+			
+			def join(suburi)
+				self.class.new(URI.join(@uri, suburi.ruri).to_s)
+			end
+			
+			def is_relative?(uri = nil)
+				@uri_class == URI::Generic ||
+				(!uri.nil? && uri.ruri.host == @uri.host)
+			end
+			
+			private
+			
+			def validate
+				if is_valid?
+					s = to_s.downcase
+					#puts "s '#{s[0..3]}'"
+					if s[0..10] == 'javascript:' ||
+						s[0..3] == 'tel:'
+						@uri = nil
+					end
+				end
+				
+				if is_valid? && !URI_CLASSES.include?(@uri.class)
+					@uri = nil
+				end
+			end
+			
+			def append_slash
+				url = to_s
+				
+				#puts "url: '#{@url}'"
+				#puts "request uri: '#{@uri.request_uri}'"
+				#puts "class: '#{@uri.class}'"
+				
+				if @uri.class == URI::HTTP && @uri.request_uri == '/' && url[-1] != '/'
+					@uri = URI("#{url}/")
+				end
+			end
+			
+			def host_downcase
+				if @uri.class != URI::Generic
+					@uri.host = @uri.host.downcase
+				end
+			end
+			
+			def remove_fragment
+				@uri.fragment = nil
+			end
+			
+			def domain_setup
+				if !@uri.nil? && !@uri.host.nil?
+					@domain_nowww = @uri.host.sub(/^www\./, '')
+					@domain_nowww_hash = Digest::SHA256.hexdigest(@domain_nowww)
+					@domain_original_hash = Digest::SHA256.hexdigest(@uri.host)
+					@domain_hash_id_key_name = "domains:id:#{@domain_nowww_hash}"
+				end
+			end
+		end
+		
+	end
+end

data/lib/sengi/version.rb

@@ -0,0 +1,17 @@
+
+module TheFox
+	module Sengi
+		VERSION = '0.1.0'
+		DATE = '2016-05-07'
+		HOMEPAGE = 'https://github.com/TheFox/sengi'
+		
+		#HTTP_USER_AGENT = "Sengi SearchENGIne/#{VERSION}"
+		HTTP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.108 Safari/537.36'
+		
+		HTTP_REFERER = 'https://www.google.com/'
+		
+		URL_DELAY = 2
+		URL_SEPARATE_DELAY = 5
+		URL_RESCHEDULE = 300
+	end
+end

data/sengi.gemspec

@@ -0,0 +1,37 @@
+# coding: UTF-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'sengi/version'
+
+Gem::Specification.new do |spec|
+	spec.name          = 'sengi'
+	spec.version       = TheFox::Sengi::VERSION
+	spec.date          = TheFox::Sengi::DATE
+	spec.author        = 'Christian Mayer'
+	spec.email         = 'christian@fox21.at'
+	
+	spec.summary       = %q{Sengi Web Crawler}
+	spec.description   = %q{A web crawler using Ruby and Redis.}
+	spec.homepage      = TheFox::Sengi::HOMEPAGE
+	spec.license       = 'GPL-3.0'
+	
+	spec.files         = `git ls-files -z`.split("\x0").reject{ |f| f.match(%r{^(test|spec|features)/}) }
+	spec.bindir        = 'bin'
+	spec.executables   = []
+	spec.require_paths = ['lib']
+	spec.required_ruby_version = '>=2.1.0'
+	
+	spec.add_development_dependency 'minitest', '~>5.8'
+	
+	spec.add_dependency 'activesupport', '~>4.2'
+	spec.add_dependency 'redis', '~>3.2'
+	spec.add_dependency 'hiredis', '~>0.6'
+	spec.add_dependency 'resque', '~>1.26'
+	spec.add_dependency 'resque-scheduler', '~>4.1'
+	spec.add_dependency 'nokogiri', '~>1.6'
+	spec.add_dependency 'cookiejar', '~>0.3'
+	
+	spec.add_dependency 'thefox-ext', '~>1.4'
+end

data/sengi.sublime-project

@@ -0,0 +1,10 @@
+{
+	"folders":[
+		{
+			"path": ".",
+			"name": "Sengi",
+			"folder_exclude_patterns": [ ],
+			"file_exclude_patterns": [ ]
+		}
+	]
+}

data/tests/tc_crawler.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'minitest/autorun'
+require 'sengi'
+
+
+class TestCrawler < MiniTest::Test
+	def test_base
+		crawler = TheFox::Sengi::Crawler.new(nil, 0, 0)
+		
+		assert_equal('TheFox::Sengi::Crawler', crawler.class.to_s)
+	end
+end
+	

data/tests/tc_uri.rb

@@ -0,0 +1,140 @@
+#!/usr/bin/env ruby
+
+require 'minitest/autorun'
+require 'sengi'
+
+
+class TestUri < MiniTest::Test
+	def test_base
+		uri = TheFox::Sengi::Uri.new('http://example.com')
+		
+		assert_equal('TheFox::Sengi::Uri', uri.class.to_s)
+		assert_equal('URI::HTTP', uri.ruri.class.to_s)
+	end
+	
+	def test_string
+		uri = TheFox::Sengi::Uri.new('http://example.com')
+		assert_equal('http://example.com/', "#{uri}")
+		assert_equal('http://example.com/', uri.to_s)
+		
+		uri = TheFox::Sengi::Uri.new('http://example.com/')
+		assert_equal('http://example.com/', uri.to_s)
+		
+		uri = TheFox::Sengi::Uri.new('http://example.com/subdir1/')
+		assert_equal('http://example.com/subdir1/', uri.to_s)
+		
+		uri = TheFox::Sengi::Uri.new('http://example.com/subdir2')
+		assert_equal('http://example.com/subdir2', uri.to_s)
+		
+		uri = TheFox::Sengi::Uri.new('http://example.com/subdir2.html')
+		assert_equal('http://example.com/subdir2.html', uri.to_s)
+		
+		uri = TheFox::Sengi::Uri.new('/subdir2.html')
+		assert_equal('/subdir2.html', uri.to_s)
+	end
+	
+	def test_hash
+		uri = TheFox::Sengi::Uri.new('http://www.example.com/index.html')
+		
+		assert_equal('b1ae8ba07f44d280254af4d1db914de03ce87b027e1c291ffcb9211c7712c9d1', uri.to_hash)
+	end
+	
+	def test_id
+		uri = TheFox::Sengi::Uri.new('http://www.example.com/index.html')
+		
+		uri.id = 21
+		assert_equal(21, uri.id)
+		assert_equal('urls:21', uri.key_name)
+		
+		uri.id = 24
+		assert_equal(24, uri.id)
+		assert_equal('urls:24', uri.key_name)
+	end
+	
+	def test_valid
+		uri = TheFox::Sengi::Uri.new('http://example.com')
+		assert_equal(true, uri.is_valid?)
+		
+		uri = TheFox::Sengi::Uri.new('javascript:alert(1);')
+		assert_equal(false, uri.is_valid?)
+		
+		uri = TheFox::Sengi::Uri.new('tel:+43501234567890')
+		assert_equal(false, uri.is_valid?)
+	end
+	
+	def test_to_http
+		uri_http = TheFox::Sengi::Uri.new('http://www.example.com/index.html')
+		uri_https = TheFox::Sengi::Uri.new('https://www.example.com/index.html')
+		assert_equal(uri_http.to_http.to_s, uri_https.to_http.to_s)
+	end
+	
+	def test_weight
+		uri = TheFox::Sengi::Uri.new('http://www.example1.com/index.html')
+		
+		suburi = TheFox::Sengi::Uri.new('test.html')
+		assert_equal(100, suburi.weight(uri))
+		
+		suburi = TheFox::Sengi::Uri.new('http://sub.www.example1.com')
+		assert_equal(200, suburi.weight(uri))
+		
+		suburi = TheFox::Sengi::Uri.new('http://sub.example1.com')
+		assert_equal(250, suburi.weight(uri))
+		
+		suburi = TheFox::Sengi::Uri.new('http://www.example2.com')
+		assert_equal(250, suburi.weight(uri))
+		
+		suburi = TheFox::Sengi::Uri.new('https://www.example2.com')
+		assert_equal(290, suburi.weight(uri))
+	end
+	
+	def test_join
+		uri1 = TheFox::Sengi::Uri.new('http://www.example.com')
+		uri2 = TheFox::Sengi::Uri.new('index.html')
+		uri3 = uri1.join(uri2)
+		assert_equal('http://www.example.com/', uri1.to_s)
+		assert_equal('index.html', uri2.to_s)
+		assert_equal('http://www.example.com/index.html', uri3.to_s)
+		
+		uri1 = TheFox::Sengi::Uri.new('http://www.example.com/test1')
+		uri2 = TheFox::Sengi::Uri.new('../test2.html')
+		uri3 = uri1.join(uri2)
+		assert_equal('http://www.example.com/test1', uri1.to_s)
+		assert_equal('../test2.html', uri2.to_s)
+		assert_equal('http://www.example.com/test2.html', uri3.to_s)
+		
+		uri1 = TheFox::Sengi::Uri.new('http://www.example1.com/test1.html')
+		uri2 = TheFox::Sengi::Uri.new('http://www.example2.com/test2.html')
+		uri3 = uri1.join(uri2)
+		assert_equal('http://www.example1.com/test1.html', uri1.to_s)
+		assert_equal('http://www.example2.com/test2.html', uri2.to_s)
+		assert_equal('http://www.example2.com/test2.html', uri3.to_s)
+	end
+	
+	def test_is_relative
+		uri1 = TheFox::Sengi::Uri.new('index1.html')
+		uri2 = TheFox::Sengi::Uri.new('index2.html')
+		assert_equal(true, uri2.is_relative?(uri1))
+		
+		uri1 = TheFox::Sengi::Uri.new('http://www.example.com')
+		uri2 = TheFox::Sengi::Uri.new('index.html')
+		assert_equal(true, uri2.is_relative?(uri1))
+		
+		uri1 = TheFox::Sengi::Uri.new('http://www.example1.com/index.html')
+		uri2 = TheFox::Sengi::Uri.new('http://www.example2.com/index.html')
+		assert_equal(false, uri2.is_relative?(uri1))
+	end
+	
+	def test_host_downcase
+		uri = TheFox::Sengi::Uri.new('http://www.EXAMPLE.com/Index.html')
+		
+		assert_equal('http://www.example.com/Index.html', uri.to_s)
+	end
+	
+	def test_fragment
+		uri = TheFox::Sengi::Uri.new('http://example.com/index.html#test')
+		assert_equal('http://example.com/index.html', uri.to_s)
+		
+		uri = TheFox::Sengi::Uri.new('index.html#test')
+		assert_equal('index.html', uri.to_s)
+	end
+end