flyerhzm-regexp_crawler 0.4.0 → 0.5.0

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.5.0

data/lib/regexp_crawler/crawler.rb

@@ -1,6 +1,6 @@
 module RegexpCrawler
   class Crawler
-    attr_accessor :start_page, :continue_regexp, :named_captures, :model, :save_method
+    attr_accessor :start_page, :continue_regexp, :named_captures, :model, :save_method, :headers, :encoding
 
     def initialize(options = {})
       @start_page = options[:start_page]
@@ -9,6 +9,8 @@ module RegexpCrawler
       @named_captures = options[:named_captures]
       @model = options[:model]
       @save_method = options[:save_method]
+      @headers = options[:headers]
+      @encoding = options[:encoding]
     end
 
     def capture_regexp=(regexp)
@@ -29,32 +31,34 @@ module RegexpCrawler
 
     private
       def parse_page(uri)
-        response = Net::HTTP.get_response(uri)
+        response = Net::HTTP.start(uri.host, uri.port) do |http|
+          http.get(uri.request_uri, headers)
+        end
         parse_response(response, uri)
       end
 
       def parse_response(response, uri)
+        response_body = Iconv.iconv("UTF-8//IGNORE", "#{encoding}//IGNORE", response.body).first if encoding
         if response.is_a? Net::HTTPSuccess
           if continue_regexp
-            response.body.scan(continue_regexp).each do |page|
+            response_body.scan(continue_regexp).each do |page|
               page = page.first if page.is_a? Array
               continue_uri = page.start_with?(uri.scheme) ? URI.parse(page) : URI.join(uri.scheme + '://' + uri.host, page)
               @pages << continue_uri unless @captured_pages.include?(continue_uri) or @pages.include?(continue_uri)
             end 
           end
-          md = @capture_regexp.match(response.body)
+          md = @capture_regexp.match(response_body)
           if md
             captures = md.captures if md
             result = {}
             captures.each_index do |i|
               result[named_captures[i].to_sym] = captures[i]
             end
-            url = "#{uri.scheme}://#{uri.host}#{uri.path}"
             if @save_method
-              ret = @save_method.call(result, url)
+              ret = @save_method.call(result, uri.to_s)
               @stop = true if ret == false
             else
-              @results << {@model.downcase.to_sym => result, :page => url}
+              @results << {@model.downcase.to_sym => result, :page => uri.to_s}
             end
           end
         elsif response.is_a? Net::HTTPRedirection

data/regexp_crawler.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{regexp_crawler}
-  s.version = "0.4.0"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Richard Huang"]

data/spec/regexp_crawler_spec.rb

@@ -43,7 +43,6 @@ describe RegexpCrawler::Crawler do
       crawl.continue_regexp = %r{(?:http://complex.com/)?nested\d.html}
       crawl.capture_regexp = %r{<div class="title">(.*?)</div>.*<div class="date">(.*?)</div>.*<div class="body">(.*?)</div>}m
       crawl.named_captures = ['title', 'date', 'body']
-      crawl.model = 'post'
       my_results = []
       crawl.save_method = Proc.new {|result, page| my_results << result}
       results = crawl.start
@@ -57,7 +56,6 @@ describe RegexpCrawler::Crawler do
       crawl.continue_regexp = %r{(?:http://complex.com/)?nested\d.html}
       crawl.capture_regexp = %r{<div class="title">(.*?)</div>.*<div class="date">(.*?)</div>.*<div class="body">(.*?)</div>}m
       crawl.named_captures = ['title', 'date', 'body']
-      crawl.model = 'post'
       stop_page = "http://complex.com/nested1.html"
       parse_pages = []
       crawl.save_method = Proc.new do |result, page| 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: flyerhzm-regexp_crawler
 version: !ruby/object:Gem::Version 
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Richard Huang
@@ -41,6 +41,7 @@ files:
 - spec/spec_helper.rb
 has_rdoc: false
 homepage: ""
+licenses: 
 post_install_message: 
 rdoc_options: 
 - --charset=UTF-8
@@ -61,7 +62,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.2.0
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
 summary: RegexpCrawler is a Ruby library for crawl data from website using regular expression.