graboid 0.3.0 → 0.3.2

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.3.2

data/graboid.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{graboid}
-  s.version = "0.3.0"
+  s.version = "0.3.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Christopher Burnett"]
@@ -14,8 +14,7 @@ Gem::Specification.new do |s|
   s.email = %q{signalstatic@gmail.com}
   s.extra_rdoc_files = [
     "LICENSE",
-     "README.mdown",
-     "README.mdown.orig"
+     "README.mdown"
   ]
   s.files = [
     ".document",
@@ -29,6 +28,8 @@ Gem::Specification.new do |s|
      "lib/graboid/entity.rb",
      "spec/fixtures/graboid.jpg",
      "spec/fixtures/posts.html",
+     "spec/fixtures/server.rb",
+     "spec/fixtures/views/posts.erb",
      "spec/graboid/entity_spec.rb",
      "spec/graboid_spec.rb",
      "spec/spec.opts",
@@ -40,7 +41,8 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.7}
   s.summary = %q{web scraping made easy}
   s.test_files = [
-    "spec/graboid/entity_spec.rb",
+    "spec/fixtures/server.rb",
+     "spec/graboid/entity_spec.rb",
      "spec/graboid_spec.rb",
      "spec/spec_helper.rb"
   ]

data/lib/graboid/entity.rb

@@ -3,8 +3,9 @@ module Graboid
     
     def self.included klass
       klass.class_eval do
-        extend ClassMethods
+        extend  ClassMethods
         include InstanceMethods
+        
         write_inheritable_attribute(:attribute_map, {}) if attribute_map.nil?
       end
     end
@@ -43,7 +44,15 @@ module Graboid
       end
       
       def doc
-        Nokogiri::HTML read_source
+        eval "Nokogiri::#{self.mode.to_s.upcase}(read_source)"
+      end
+      
+      def collection
+        @collection ||= []
+      end
+      
+      def collection=(col)
+        @collection = col
       end
       
       def attribute_map
@@ -57,7 +66,8 @@ module Graboid
       def hash_map fragment
         attribute_map.inject({}) do |extracted_hash, at| 
           selector, processor       = at.last[:selector], at.last[:processor]
-          extracted_hash[at.first]  = processor.nil? ? fragment.css(selector).first.text : processor.call(fragment.css(selector).first) rescue ""
+          node_collection           = self.mode == :html ? fragment.css(selector) : fragment.xpath(selector)
+          extracted_hash[at.first]  = processor.nil? ? node_collection.first.text : processor.call(node_collection.first) rescue ""
 
           extracted_hash
         end
@@ -65,15 +75,15 @@ module Graboid
       
       def all_fragments
         return page_fragments if @pager.nil?
-        old_source  = self.source
-        @collection = []
+        old_source = self.source
         while next_page?
-          @frags = page_fragments
-          @collection += @frags
+          self.collection += page_fragments
+          run_before_paginate_callbacks
           paginate
+          run_after_paginate_callbacks
         end
         self.source = old_source
-        @collection
+        self.collection
       end
       
       def paginate
@@ -83,7 +93,11 @@ module Graboid
       end
       
       def next_page?
-        (current_page <= max_pages-1)
+        if max_pages.zero?
+          return true unless @pager.call(doc).nil?
+        else
+          current_page <= max_pages-1
+        end
       end
       
       def page_fragments
@@ -91,13 +105,20 @@ module Graboid
       end
       
       def all opts={}
+        reset_context
         self.max_pages = opts[:max_pages] if opts[:max_pages].present?
         all_fragments.collect{ |frag| extract_instance(frag) }
       end
       
+      def reset_context
+        self.collection   = []
+        self.current_page = 0
+        self.max_pages    = 0
+      end
+      
       def read_source
         case self.source
-          when /^http:\/\//
+          when /^http[s]?:\/\//
             open self.source
           when String
             self.source
@@ -108,6 +129,15 @@ module Graboid
         @pager = block
       end
       
+      def mode
+        @mode ||= :html
+      end
+      
+      def mode=(m)
+        raise ArgumentError unless [:html, :xml].include?(m)
+        @mode = m
+      end
+      
       def max_pages
         @max_pages ||= 0
       end
@@ -124,6 +154,21 @@ module Graboid
         @current_page = num
       end
       
+      instance_eval do
+        [:before, :after].each do |prefix|
+          [:paginate, :extract].each do |suffix|
+            method_name = "#{prefix}_#{suffix}"
+            define_method method_name.to_sym do |&block|
+              instance_variable_set "@#{method_name}", block
+            end
+            define_method "run_#{method_name}_callbacks" do
+              ivar = instance_variable_get("@#{method_name}")
+              self.class_eval { ivar.call } unless ivar.nil?
+            end
+          end
+        end 
+      end     
+      
     end # ClassMethods
     
     module InstanceMethods

data/spec/fixtures/server.rb

@@ -0,0 +1,8 @@
+%w{rubygems sinatra}.each {|f| require f }
+
+get "/posts" do
+  @total_pages  = 8
+  @page         = params[:page].to_i || 1
+  @limit        = 2 
+  erb :posts
+end

data/spec/fixtures/views/posts.erb

@@ -0,0 +1,37 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
+   "http://www.w3.org/TR/html4/strict.dtd">
+
+<html lang="en">
+<head>
+	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+	<title>posts</title>
+	<meta name="generator" content="TextMate http://macromates.com/">
+	<meta name="author" content="Posterous">
+	<!-- Date: 2010-06-10 -->
+</head>
+<body>
+
+<% @limit.times do |num| %>
+
+  <div class="post" id="<%= num  + (@page*@limit)-1 %>">
+    
+    <p class="title">Post <%= num + (@page*@limit)-1 %></p>
+    
+    <p class="body">Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor 
+      incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation 
+      ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit 
+      in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat 
+      non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </p>
+    <span class="author">Someone Awesome (06/11/2010)</span>
+    
+  </div>
+  
+<% end %>
+
+<% if @page < @total_pages %>
+  <a class="next" href="/posts?page=<%= @page.next %>">next</a>
+<% end %>
+
+</body>
+</html>

data/spec/graboid/entity_spec.rb

@@ -1,11 +1,5 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
-class Post
-  include Graboid::Entity
-  
-  selector '.post'
-end
-
 describe Graboid::Entity do
   describe "#source" do
     describe "when url" do
@@ -112,12 +106,6 @@ describe Graboid::Entity do
   
   describe "#all_fragments" do
     before(:each) do
-      
-      class WorkingPost
-        include Graboid::Entity
-        selector '.post'
-        set :body
-      end
   
       WorkingPost.source = POSTS_HTML_STR
       @fragments  = WorkingPost.all_fragments
@@ -136,17 +124,7 @@ describe Graboid::Entity do
   describe "#extract_instance" do
     
     before(:each) do
-      class WorkingPost
-        include Graboid::Entity
-        selector '.post'
-        set :title
-        set :body
-        set :author
-        set :date, :selector => '.author', :processor => lambda {|frag| frag.text.match(/\((.*)\)/)[1] }
-      end
-      
       @instance = WorkingPost.extract_instance(POST_FRAGMENT)
-      
     end
     
     it "should return a WorkingPost instance" do
@@ -165,17 +143,7 @@ describe Graboid::Entity do
   
   describe "#all" do
     before(:each) do
-      class WorkingPost
-        include Graboid::Entity
-        selector '.post'
-        set :title
-        set :body
-        set :author
-        set :date, :selector => '.author', :processor => lambda {|frag| frag.text.match(/\((.*)\)/)[1] }
-      end
-      
       WorkingPost.source = POSTS_HTML_STR
-      
     end
     
     it "should return 2 WorkingPosts" do
@@ -196,32 +164,42 @@ describe Graboid::Entity do
     end
   end
   
+  describe "#mode" do
+    it "should be html by default" do
+      WorkingPost.mode.should == :html
+    end
+    it "should throw an error for invalid values" do
+      lambda {
+        WorkingPost.mode = :derp
+      }.should raise_error ArgumentError
+    end
+    it "should change to :xml" do
+      WorkingPost.mode = :xml
+      WorkingPost.mode.should == :xml
+    end
+  end  
+  
   describe "#pager" do
-    before(:each) do
-
-      class RedditEntry
-        include Graboid::Entity
-
-        selector '.entry'
-
-        set :title
-        set :domain, :selector => '.domain a'
-        set :link,   :selector => '.title' do |entry| 
-          entry.css('a').first['href'] 
-        end
-        
-        pager do |doc|
-          doc.css('p.nextprev a').select{|a| a.text =~ /next/i  }.first['href']
-        end
-
+    describe "with a limit" do
+      before(:each) do
+        PostWithPager.source = 'http://localhost:9393/posts'
+        @posts = PostWithPager.all(:max_pages => 3)
+      end
+      it "should get 2 posts" do
+        @posts.length.should == 6
       end
-      RedditEntry.source = 'http://reddit.com'
-      @posts = RedditEntry.all(:max_pages => 2)
     end
-    it "should get 70 posts" do
-      @posts.length.should == 70
+    
+    describe "without a limit" do
+      before(:each) do
+        PostWithPager.source = 'http://localhost:9393/posts'
+        @posts = PostWithPager.all
+      end
+      it "should get 2 posts" do
+        @posts.length.should == 16
+      end
     end
+    
   end
-  
-  
+    
 end

data/spec/spec_helper.rb

@@ -8,7 +8,50 @@ Spec::Runner.configure do |config|
   
 end
 
-file_path       = File.expand_path(File.dirname(__FILE__)+'/fixtures/posts.html')
-POSTS_HTML_STR  = File.read(file_path){|f| f.read }
-d = Nokogiri::HTML(POSTS_HTML_STR)
-POST_FRAGMENT   = Nokogiri::HTML::fragment(d.css('.post').first.to_html)
+FIXTURE_PATH    = File.expand_path(File.dirname(__FILE__)+'/fixtures/posts.html')
+POSTS_HTML_STR  = File.read(FIXTURE_PATH){|f| f.read }
+POST_DOC        = Nokogiri::HTML(POSTS_HTML_STR)
+POST_FRAGMENT   = Nokogiri::HTML::fragment(POST_DOC.css('.post').first.to_html)
+
+class Post
+  include Graboid::Entity
+  
+  selector '.post'
+end
+
+class WorkingPost
+  include Graboid::Entity
+  
+  selector '.post'
+  
+  set :title
+  set :body
+  set :author
+  set :date, :selector => '.author' do |elm| 
+    elm.text.match(/\((.*)\)/)[1] 
+  end
+end
+
+class PostWithPager
+  include Graboid::Entity
+
+  selector '.post'
+  
+  set :title
+  set :body
+  set :author
+  set :date, :selector => '.author' do |elm| 
+    elm.text.match(/\((.*)\)/)[1] 
+  end
+
+  pager do |doc|
+    link = 'http://localhost:9393'+doc.css('a.next').first['href'] rescue nil
+    #puts link.inspect
+    link
+  end
+
+  before_paginate do
+    puts "page: #{self.source}"
+  end
+
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: graboid
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 23
   prerelease: false
   segments: 
   - 0
   - 3
-  - 0
-  version: 0.3.0
+  - 2
+  version: 0.3.2
 platform: ruby
 authors: 
 - Christopher Burnett
@@ -71,7 +71,6 @@ extensions: []
 extra_rdoc_files: 
 - LICENSE
 - README.mdown
-- README.mdown.orig
 files: 
 - .document
 - .gitignore
@@ -84,11 +83,12 @@ files:
 - lib/graboid/entity.rb
 - spec/fixtures/graboid.jpg
 - spec/fixtures/posts.html
+- spec/fixtures/server.rb
+- spec/fixtures/views/posts.erb
 - spec/graboid/entity_spec.rb
 - spec/graboid_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-- README.mdown.orig
 has_rdoc: true
 homepage: http://github.com/twoism/graboid
 licenses: []
@@ -124,6 +124,7 @@ signing_key:
 specification_version: 3
 summary: web scraping made easy
 test_files: 
+- spec/fixtures/server.rb
 - spec/graboid/entity_spec.rb
 - spec/graboid_spec.rb
 - spec/spec_helper.rb

data/README.mdown.orig

@@ -1,61 +0,0 @@
-### Graboid ###
-
-![Graboid](http://github.com/twoism/graboid/raw/master/spec/fixtures/graboid.jpg "Graboid")
-
-  Simply awesome web scraping. Better docs later. See specs.
-
-### Installation ###
-
-
-    gem install nokogiri graboid
-
-
-### Usage ###
-
-    %w{rubygems graboid}.each { |f| require f }
-
-    class RedditEntry
-      include Graboid::Entity
-
-      selector '.entry'
-<<<<<<< HEAD
-
-      set :title
-      set :domain, :selector => '.domain a'
-      set :link,   :selector => '.title' do |entry| 
-=======
-      
-      field :title
-      field :domain, :selector => '.domain a'
-      field :link,    :selector => '.title' do |entry| 
->>>>>>> ea3c69202c1af78378fd4fb4b2d9ccd2098bc9d8
-        entry.css('a').first['href'] 
-      end
-      
-      pager do |doc|
-        doc.css('p.nextprev a').select{|a| a.text =~ /next/i  }.first['href']
-      end
-
-    end
-
-    RedditEntry.source = 'http://reddit.com'
-
-    RedditEntry.all(:max_pages => 2).each do |p| 
-      puts "title: #{p.title}"
-      puts "domain: #{p.domain}"
-      puts "link: #{p.link}"
-    end
-
-##Note on Patches/Pull Requests
- 
-* Fork the project.
-* Make your feature addition or bug fix.
-* Add tests for it. This is important so I don't break it in a
-  future version unintentionally.
-* Commit, do not mess with rakefile, version, or history.
-  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
-* Send me a pull request. Bonus points for topic branches.
-
-## Copyright
-
-Copyright (c) 2010 Christopher Burnett. See LICENSE for details.