flyerhzm-regexp_crawler 0.8.2 → 0.9.0

data/README.textile

@@ -31,6 +31,7 @@ options is a hash
 * <code>:headers</code>, optional, a hash to define http headers
 * <code>:encoding</code>, optional, a string of the coding of crawled page, the results will be converted to utf8
 * <code>:need_parse</code>, optional, a proc if parsing the page by regexp or not, the proc accept two parameters, first is the crawled website uri, second is the response body of crawled page
+* <code>:logger</code>, optional, true for logging to STDOUT, or a Logger object for logging to that logger
 
 If the crawler define :model no :save_method, the RegexpCrawler::Crawler#start will return an array of results, such as
 <pre><code>
@@ -41,7 +42,7 @@ If the crawler define :model no :save_method, the RegexpCrawler::Crawler#start w
 
 h2. Example
 
-a script to synchronize your github projects except fork projects
+a script to synchronize your github projects except fork projects, please check <code>example/github_projects.rb</code>
 
 <pre><code>
 require 'rubygems'
@@ -49,8 +50,8 @@ require 'regexp_crawler'
 
 crawler = RegexpCrawler::Crawler.new(
   :start_page => "http://github.com/flyerhzm",
-  :continue_regexp => %r{<div class="title"><b><a href="(/flyerhzm/.*?/tree)">}m,
-  :capture_regexp => %r{<a href="http://github.com/flyerhzm/.*?/tree">(.*?)</a>.*<span id="repository_description".*?>(.*?)</span>.*(<div class="(?:wikistyle|plain)">.*?</div>)</div>}m,
+  :continue_regexp => %r{<div class="title"><b><a href="(/flyerhzm/.*?)">}m,
+  :capture_regexp => %r{<a href="http://github.com/flyerhzm/[^/"]*?(?:/tree)?">(.*?)</a>.*<span id="repository_description".*?>(.*?)</span>.*(<div class="(?:wikistyle|plain)">.*?</div>)</div>}m,
   :named_captures => ['title', 'description', 'body'],
   :save_method => Proc.new do |result, page|
     puts '============================='
@@ -60,10 +61,10 @@ crawler = RegexpCrawler::Crawler.new(
     puts result[:body][0..100] + "..."
   end, 
   :need_parse => Proc.new do |page, response_body|
-    !response_body.index "Fork of"
+    page =~ %r{http://github.com/flyerhzm/\w+} && !response_body.index(/Fork of.*?<a href=".*?">/)
   end) 
 crawler.start
-</code></pre>
+</pre></code>
 
 The results are as follows:
 <pre><code>

data/VERSION

@@ -1 +1 @@
-0.8.2
+0.9.0

data/lib/regexp_crawler.rb

@@ -1,6 +1,7 @@
 require 'net/http'
 require 'uri'
 require 'iconv'
+require 'logger'
 require 'regexp_crawler/http'
 require 'regexp_crawler/crawler'
 

data/lib/regexp_crawler/crawler.rb

@@ -12,6 +12,7 @@ module RegexpCrawler
       @headers = options[:headers]
       @encoding = options[:encoding]
       @need_parse = options[:need_parse]
+      @logger = options[:logger] == true ? Logger.new(STDOUT) : options[:logger]
     end
 
     def capture_regexp=(regexp)
@@ -32,6 +33,7 @@ module RegexpCrawler
 
     private
       def parse_page(uri)
+        @logger.debug "crawling page: #{uri.to_s}" if @logger
         response = Net::HTTP.get_response_with_headers(uri, @headers)
         parse_response(response, uri)
       end
@@ -49,8 +51,10 @@ module RegexpCrawler
       def parse_response(response, uri)
         response_body = encoding.nil? ? response.body : Iconv.iconv("UTF-8//IGNORE", "#{encoding}//IGNORE", response.body).first
         if response.is_a? Net::HTTPSuccess
+          @logger.debug "crawling success: #{uri.to_s}" if @logger
           if continue_regexp
             response_body.scan(continue_regexp).each do |page|
+              @logger.debug "continue_page: #{page}" if @logger
               page = page.compact.first if page.is_a? Array
               continue_uri = continue_uri(uri, page)
               @pages << continue_uri unless @captured_pages.include?(continue_uri) or @pages.include?(continue_uri)
@@ -59,6 +63,7 @@ module RegexpCrawler
           if @need_parse.nil? or @need_parse.call(uri.to_s, response_body)
             md = @capture_regexp.match(response_body)
             if md
+              @logger.debug "response body captured" if @logger
               captures = md.captures
               result = {}
               captures.each_index do |i|
@@ -73,8 +78,10 @@ module RegexpCrawler
             end
           end
         elsif response.is_a? Net::HTTPRedirection
+          @logger.debug "crawling redirect: #{response['location']}" if @logger
           parse_page(URI.parse(response['location']))
         else
+          @logger.debug "crawling nothing: #{uri.to_s}" if @logger
           # do nothing
         end
       end

data/regexp_crawler.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{regexp_crawler}
-  s.version = "0.8.2"
+  s.version = "0.9.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Richard Huang"]
@@ -20,7 +20,6 @@ Gem::Specification.new do |s|
     "LICENSE",
      "README.textile",
      "Rakefile",
-     "TODO",
      "VERSION",
      "example/github_projects.rb",
      "init.rb",

data/spec/regexp_crawler_spec.rb

@@ -5,7 +5,7 @@ describe RegexpCrawler::Crawler do
     it 'should parse data according to regexp' do
       success_page('/resources/simple.html', 'http://simple.com/')
 
-      crawl = RegexpCrawler::Crawler.new(:start_page => 'http://simple.com/', :capture_regexp => %r{<div class="title">(.*?)</div>.*<div class="date">(.*?)</div>.*<div class="body">(.*?)</div>}m, :named_captures => ['title', 'date', 'body'], :model => 'post')
+      crawl = RegexpCrawler::Crawler.new(:start_page => 'http://simple.com/', :capture_regexp => %r{<div class="title">(.*?)</div>.*<div class="date">(.*?)</div>.*<div class="body">(.*?)</div>}m, :named_captures => ['title', 'date', 'body'], :model => 'post', :logger => true)
       results = crawl.start
       results.size.should == 1
       results.first[:post][:title].should == 'test'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: flyerhzm-regexp_crawler
 version: !ruby/object:Gem::Version 
-  version: 0.8.2
+  version: 0.9.0
 platform: ruby
 authors: 
 - Richard Huang
@@ -26,7 +26,6 @@ files:
 - LICENSE
 - README.textile
 - Rakefile
-- TODO
 - VERSION
 - example/github_projects.rb
 - init.rb