reid 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OTBmYTkyYzZjYzU3OTZiYTJhYzViYTZkN2M4ZWMyZWJjODUzOGUzYw==
+  data.tar.gz: !binary |-
+    YjhhZTBjZGQyOWU3M2VkNjU1ZjA3NGZhYWE1ZDgyN2I2NTNjYjFjOA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    N2MyMzRjYTNkYzBhMTFkNWEwNGM0YzRmYWFmZTc4MzkyZWEyMWUyZWJjNGY4
+    Mzg1YzRhYWYwZThiYzk0ZWZkODI5Y2Q1Y2IxZGE4MmQwOTJmODc2Zjc1ZDNl
+    NjdmMjM4NjI4MWI0MGY1YzllZTVjMzViNDk0YThhM2JmMTE4ZmY=
+  data.tar.gz: !binary |-
+    MTZkYzdkMmZhNTEyNDRiNjU3M2MwZjQzYTEyNDEzNzVmYjI1ZWY4Y2I5NmY5
+    NGE1MGI2NmY1ODZmNjhjNWIxMmVlNGU0OTY2ZGZjMmRjYmE0NjJiMzhmNDE0
+    YzhjMDgxODAxNTk4NjNlMzNjMmRmNTljOGJiZjUyYjM1ZjAxNmY=

data/lib/reid.rb

@@ -0,0 +1,5 @@
+require 'nokogiri'
+require 'open-uri'
+
+require 'reid/reid'
+require 'reid/requester.rb'

data/lib/reid/reid.rb

@@ -0,0 +1,31 @@
+require 'nokogiri'
+
+class Reid
+	def initialize(requester_options = {})
+		@requester = Requester.new(requester_options)
+	end
+
+	def scrape_doc doc, operations
+		record = {}
+		operations.each do |e|
+			if e[3] == :xpath
+				e[0].call(doc.xpath(e[1]), record)
+			else
+				e[0].call(doc.css(e[1]), record)
+			end
+		end
+		return record
+	end
+
+	def scrape_page url, operations
+		return scrape_doc(@requester.request(url), operations)
+	end
+
+	def crawl url_crawler, operations, store_function
+		doc = nil
+		while(url = url_crawler.next(doc))
+			doc = @requester.request url
+			store_function.call(scrape_doc(doc, operations))
+		end
+	end
+end

data/lib/reid/requester.rb

@@ -0,0 +1,43 @@
+require 'nokogiri'
+require 'open-uri'
+
+class Requester
+	def initialize(options = {})
+		@min_request_interval=(options[:min_request_interval] || false)
+		@error_log = (options[:error_log] || false)
+		@max_backoff_time = (options[:max_backoff_time] || false)
+		@initial_delay = (options[:intial_delay] || 1.0)
+		@multiplicand = (options[:multiplicand] || 1.3)
+		@previous_request = (@min_request_interval && Time.now)
+	end
+
+	def request(url, time = false)
+		if !time then if @min_request_interval then damper end end
+		begin
+			return Nokogiri::HTML(open(url))
+		rescue Exception => e
+			if @error_record
+				error_record = {:time => Time.now, :message =>e.message}
+				@error_log.insert(error_record)
+			end
+			unless time
+				sleep @intial_delay
+				return request url, @initial_delay * @multiplicand
+			else
+				if (!@max_backoff_time || (time < @max_backoff_time))
+					sleep time
+					return request url, time * @multiplicand
+				else
+					raise "Problem with request. Max backoff time exceided."
+				end
+			end
+		end
+	end
+
+	def damper()
+		if	(Time.now - @previous_request < @min_request_interval)
+			sleep(@min_request_interval - (Time.now - @previous_request))
+		end
+		@previous_request = Time.now
+	end
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: reid
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dan Breczinski
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+description: Reid is a simple tool for crawling web pages and throttling requests
+email: pt2323@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/reid.rb
+- lib/reid/reid.rb
+- lib/reid/requester.rb
+homepage: https://github.com/danpaul/reid
+licenses:
+- MIT (http://opensource.org/licenses/MIT)
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.5
+signing_key: 
+specification_version: 4
+summary: Reid is a gem to help structure web scraping.
+test_files: []