flyerhzm-regexp_crawler 0.1.0 → 0.2.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Richard Huang (flyerhzm@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -1 +1,20 @@
-A crawler which use regrex expression to catch data.
+RegexpCrawler
+============
+
+RegexpCrawler is a crawler which use regrex expression to catch data.
+
+
+Install
+=======
+
+gem sources -a http://gems.github.com
+gem install flyerhzm-regexp_crawler
+
+
+Usage
+=====
+
+>> crawler = RegexpCrawler::Crawler.new(:start_page => "http://www.tijee.com/tags/64-google-face-questions/posts", :continue_regexp => %r{"(/posts/\d+-[^#]*?)"}, :capture_regexp => %r{<h2 class='title'><a.*?>(.*?)</a></h2>.*?<div class='body'>(.*?)</div>}m, :named_captures => ['title', 'body'], :model => Post)
+>> crawler.start
+
+=>[{:page=>"http://www.tijee.com/posts/327-google-face-questions-many-companies-will-ask-oh", :model=>#<Post id: nil, title: "Google面试题（很多公司都会问的哦）", body: "\n内容摘要：几星期前，一个朋友接受...", created_at: nil, updated_at: nil, verify: false>}, {:page=>"http://www.tijee.com/posts/328-java-surface-together-with-the-google-test", :model=>#<Post id: nil, title: "google的一道JAVA面试题", body: "\n内容摘要：有一个整数n,写一个函数f(n...", created_at: nil, updated_at: nil, verify: false>}]

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/lib/regexp_crawler/crawler.rb

@@ -16,9 +16,11 @@ module RegexpCrawler
 
     def start
       results = []
-      @pages = [@start_page]
+      @captured_pages = []
+      @pages = [URI.parse(@start_page)]
       while !@pages.empty?
-        uri = URI.parse(@pages.shift)
+        uri = @pages.shift
+        @captured_pages << uri
         result = parse_page(uri)
         results << result if result
       end
@@ -33,8 +35,9 @@ module RegexpCrawler
     def parse_response(response, uri)
       if response.is_a? Net::HTTPSuccess
         response.body.scan(continue_regexp).each do |page|
-          url = page.start_with?(uri.scheme) ? page : "#{uri.scheme}://#{uri.host}/#{page}"
-          @pages << url
+          page = page.first if page.is_a? Array
+          continue_uri = page.start_with?(uri.scheme) ? URI.parse(page) : URI.join(uri.scheme + '://' + uri.host, page)
+          @pages << continue_uri unless @captured_pages.include?(continue_uri) or @pages.include?(continue_uri)
         end if continue_regexp
         md = @capture_regexp.match(response.body)
         if md

data/regexp_crawler.gemspec

@@ -0,0 +1,54 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{regexp_crawler}
+  s.version = "0.2.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Richard Huang"]
+  s.date = %q{2009-08-02}
+  s.description = %q{RegexpCrawler is a Ruby library for crawl data from website using regular expression.}
+  s.email = %q{flyerhzm@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "README",
+     "Rakefile",
+     "TODO",
+     "VERSION",
+     "init.rb",
+     "lib/regexp_crawler.rb",
+     "lib/regexp_crawler/crawler.rb",
+     "regexp_crawler.gemspec",
+     "spec/regexp_crawler_spec.rb",
+     "spec/resources/complex.html",
+     "spec/resources/nested1.html",
+     "spec/resources/nested2.html",
+     "spec/resources/simple.html",
+     "spec/spec.opts",
+     "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{RegexpCrawler is a Ruby library for crawl data from website using regular expression.}
+  s.test_files = [
+    "spec/spec_helper.rb",
+     "spec/regexp_crawler_spec.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: flyerhzm-regexp_crawler
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Richard Huang
@@ -32,6 +32,7 @@ files:
 - init.rb
 - lib/regexp_crawler.rb
 - lib/regexp_crawler/crawler.rb
+- regexp_crawler.gemspec
 - spec/regexp_crawler_spec.rb
 - spec/resources/complex.html
 - spec/resources/nested1.html