scraptory 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a7ecdad23a953fc93146eb88a01ea5216d47dfac
+  data.tar.gz: 2289f29c61f00c6db5abf6fef5a71d7d9c0b3fa5
+SHA512:
+  metadata.gz: de07b7725d9b99217e0c9896649579f88276b504c60f51deb79f7ddb320420b1de5971e704fcacca5fbe292d13579fe04141f1bde3aa38b9fb3e92b971762cbe
+  data.tar.gz: 052389b0c605fce0ba5fc4bdee75cad29d3f6fd8f6be2d9a5ba7a9114f0a22851773f876fd5173f21561fda842810faaf7ad8e67fc2ce810e212f73070f02021

data/lib/proxy.rb

@@ -0,0 +1,48 @@
+# encoding: UTF-8
+require "tor"
+require "typhoeus"
+require "useragents"
+
+class Proxy
+	attr_reader :type
+	attr_reader :timeout
+
+	def initialize(host="localhost",port=8080,options={},credentials={})
+		@host 				= host
+		@port 				= port
+		@type 				= options[:type] || "http"
+		@timeout 			= options[:timeout].to_i || 10
+		@credentials 	= credentials
+		@config				= options
+	end
+
+	def url
+		return @host.to_s+":"+@port.to_s
+	end
+
+	def change_ip
+		if @config[:tor]
+			
+			if @node.nil?
+				@node = Tor::Controller.new(:host => @credentials[:telnet_host], :port => @credentials[:telnet_port])
+			end
+
+			if !@node.authenticated?
+				@node.authenticate(@credentials[:telnet_passwd])
+			end
+
+			@node.signal("NEWNYM")
+		end
+	end
+
+	def get_ip
+		Typhoeus::Config.user_agent = UserAgents.rand()
+
+		response = Typhoeus::Request.new("http://checkip.amazonaws.com/", 
+			timeout: 		@timeout,
+			proxy: 			self.url,
+			proxytype: 	@type).run
+		
+		return response.response_body.gsub("\n", '').strip
+	end
+end

data/lib/scraptory.rb

@@ -0,0 +1,184 @@
+# encoding: UTF-8
+
+# Requests
+require 'typhoeus'
+require 'useragents'
+# For file/dir manipulation
+require 'fileutils'
+# Logs
+require 'logger'
+# For Tor
+require 'net/telnet'
+require_relative "proxy"
+
+class Scraptory
+	attr_reader :config
+	@@chg_ip_fater_nfails = 10
+	@@default_hydra_timeout = 10
+	@@default_tor_timeout = 10
+	@@default_nthreads = 1
+	@@default_err_before_chg_ip = 100
+	@@count_connect_errors = 0
+
+	def initialize(config={})
+		@proxies = []
+		@proxy_cursor=-1
+
+		set_config(config)
+	end
+
+	def set_config(config={})
+		# Use debug output
+		if !config.has_key?("debug")
+			config["debug"] = false
+		end
+
+		# If debug_file is set and doesn't exists, we create it
+		if config.has_key?("debug_file") and !File.exist?(config["debug_file"])
+			config["debug"] = true
+			FileUtils.touch(config["debug_file"])
+			@logger = Logger.new(config["debug_file"])
+		elsif config["debug"]
+			@logger = Logger.new(STDOUT)
+		end
+
+		# If the param nthreads exists and is an integer it is created. Else it is set to 1
+		if config.has_key?("nthreads") and config["nthreads"].is_a? Integer
+			@hydra = Typhoeus::Hydra.new(max_concurrency: config["nthreads"].to_i)
+		else
+			@hydra = Typhoeus::Hydra.new(max_concurrency: @@default_nthreads)
+		end
+
+		# If wrong data are in timeout config, we set it to default
+		if !config.has_key?("timeout") or config["timeout"].to_i < 1
+			config["timeout"] = @@default_hydra_timeout
+		end
+
+		if !config.has_key?("retry_on_error")
+			config["retry_on_error"] = false
+		end
+
+		# Switch between proxies and clear connection
+		if config["use_clearconnection"].nil?
+			config["use_clearconnection"] = false
+		end
+
+
+		if !config["err_before_chg_ip"].nil? or config["err_before_chg_ip"].to_i < 1
+			config["err_before_chg_ip"] = @@default_err_before_chg_ip
+		end
+
+		@config = config
+	end
+
+	def queue(url,callback)
+		request = build_request(url)
+
+		request.on_complete do |response|
+			on_request_complete(response,request,callback)
+		end
+		
+		@hydra.queue(request)
+	end
+
+	def queues(urls=Array.new,callback)
+		urls.each do |url|
+			queue(url,callback)
+		end
+	end
+
+	def build_request(url)
+		Typhoeus::Config.user_agent = UserAgents.rand()
+
+		# proxy_cursor is set to 1 when no using any proxy
+		if not @proxies.any? and @proxy_cursor > -1
+			proxy = @proxies[@proxy_cursor]
+			return Typhoeus::Request.new(url,
+				:timeout 		=> proxy.timeout,
+				:proxy 			=> proxy.url,
+				:proxytype 	=> proxy.type)
+		else
+			return Typhoeus::Request.new(url, :timeout => @config['timeout'])
+		end
+	end
+
+	def on_request_complete(response,request,callback)
+		error_msg = "Unknow error"
+		add_to_queue = false
+
+		if response.success?
+			# SUCCESS
+			callback.call(response)
+		elsif response.timed_out?
+			error_msg = "Timed out ("+request.url+")"
+			add_to_queue = true
+		elsif response.code == 404
+			error_msg = "404 Page not found ("+request.url+")"
+			add_to_queue = false
+
+		elsif response.code == 301 or response.code == 302
+			error_msg = "301/302 Redirection not followed ("+request.url+")"
+			add_to_queue = false
+
+		elsif response.code == 0
+			# Could not get an http response, something's wrong.
+			error_msg = "Could not get an http response, something's wrong ("+request.url+") : "+response.return_message
+			add_to_queue = true
+		else
+			# Received a non-successful http response.
+			error_msg = "Received a non-successful http response ("+request.url+") : "+response.code.to_s
+			add_to_queue = true
+		end
+
+		self._debug(error_msg)
+
+		if add_to_queue
+			@@count_connect_errors = @@count_connect_errors + 1
+
+			if @@count_connect_errors > @config['err_before_chg_ip']
+				self._debug("Changing Proxy","info")
+
+				@proxies[@proxy_cursor].change_ip()
+
+				if @proxy_cursor == @proxies.length - 1 and @config["use_clearconnection"]
+					@proxy_cursor = -1
+				elsif @proxy_cursor == @proxies.length - 1 and !@config["use_clearconnection"]
+					@proxy_cursor = 0
+				else
+					@proxy_cursor = @proxy_cursor + 1
+				end
+
+				@@count_connect_errors = 0
+			end
+
+			if @config['retry_on_error']
+				@hydra.queue(request)
+			end
+
+		end
+	end
+
+	def scrap()
+		@hydra.run
+	end
+
+	def add_proxy(proxy)
+		@proxies << proxy
+	end
+
+	def _debug(msg,lvl="warn")
+		if @config["debug"]
+			if lvl == "warn"
+				@logger.warn msg
+			elsif lvl == "info"
+				@logger.info msg
+			elsif lvl == "debug"
+				@logger.debug msg
+			elsif lvl == "error"
+				@logger.error msg
+			else
+				@logger.warn msg
+			end
+		end
+	end
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: scraptory
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- AlexMili
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: useragents
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.4
+- !ruby/object:Gem::Dependency
+  name: tor2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+description: A simple scraping gem using tor
+email: /
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/scraptory.rb
+- lib/proxy.rb
+homepage: https://github.com/AlexMili/scraptory
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14.1
+signing_key: 
+specification_version: 4
+summary: Scraper over Tor in ruby
+test_files: []