graboid 0.3.2 → 0.3.3

data/README.mdown

@@ -21,22 +21,30 @@
 
       set :title
       set :domain, :selector => '.domain a'
+      
       set :link,   :selector => '.title' do |entry| 
         entry.css('a').first['href'] 
       end
-      
+
       pager do |doc|
         doc.css('p.nextprev a').select{|a| a.text =~ /next/i  }.first['href']
       end
 
+      before_paginate do
+        puts "opening page: #{self.source}"
+        puts "collection size: #{self.collection.length}"
+        puts "#{"*"*100}"
+      end
+
     end
 
     RedditEntry.source = 'http://reddit.com'
 
-    RedditEntry.all(:max_pages => 2).each do |p| 
+    RedditEntry.all(:max_pages => 5).each do |p| 
       puts "title: #{p.title}"
       puts "domain: #{p.domain}"
       puts "link: #{p.link}"
+      puts "#{"*"*100}"
     end
 
 ##Note on Patches/Pull Requests

data/VERSION

@@ -1 +1 @@
-0.3.2
+0.3.3

data/examples/active_rain_post.rb

@@ -0,0 +1,36 @@
+dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+require File.join(dir, 'graboid')
+
+class ActiveRainPost
+  include Graboid::Entity
+
+  root   '.blog_entry'
+
+  field :title, :selector => 'h2'
+
+  field :body, :selector => 'div' do |elm|
+    elm.css('p').collect(&:to_html)
+  end
+  
+  pager do |doc|
+    "http://activerain.com" + doc.css('.pagination a').select{|a| a.text =~ /Next/i }.first['href'] rescue nil
+  end
+  
+  before_paginate do
+    # logging for fun
+    puts "opening page: #{self.source}"
+    puts "collection size: #{self.collection.length}"
+    puts "*"*100
+  end
+
+end
+
+ActiveRainPost.source = 'http://activerain.com/blogs/elizabethweintraub'
+@posts  = ActiveRainPost.all
+
+@posts.each do |post|
+  puts "#{post.title}"
+  puts "*"*100
+end
+
+puts "total: #{@posts.length}"

data/examples/live_journal_post.rb

@@ -0,0 +1,38 @@
+dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+require File.join(dir, 'graboid')
+
+class LiveJournalPost
+  include Graboid::Entity
+
+  root '.entrybox'
+
+  field :title,         :selector => '.caption a'
+  field :body,          :selector => 'td[@colspan="2"]'
+  field :comment_link,  :selector => '.caption a' do |elm|
+    elm['href']
+  end
+  
+  pager do |doc|
+    doc.css('a').select{|a| a.text =~ /earlier/i }.first['href'] rescue nil
+  end
+  
+  before_paginate do
+    # logging for fun
+    puts "opening page: #{self.source}"
+    puts "collection size: #{self.collection.length}"
+    puts "*"*100
+  end
+
+end
+
+LiveJournalPost.source = 'http://zeroplate.livejournal.com/'
+@posts  = LiveJournalPost.all(:max_pages => 3)
+
+@posts.each do |post|
+  puts "#{post.title}"
+  puts "#{post.comment_link}"
+  puts "#{post.body}"
+  puts "*"*100
+end
+
+puts "total: #{@posts.length}"

data/examples/ning_post.rb

@@ -0,0 +1,56 @@
+%w{rubygems graboid}.each {|f| require f }
+
+class NingPost
+  include Graboid::Entity
+
+  selector 'div.xg_blog .xg_module_body'
+
+  set :title do |elm|
+    elm.text.match(/^\s*(.*)$\s*/).captures.first
+  end
+
+  set :pub_date, :selector => 'p[class=small]' do |elm| 
+    elm.text.match(/on (.* \d+, \d{4})/)[1] 
+  end
+
+  set :comment_link, :selector => 'p[class=small]' do |elm|
+    elm.css('a').select {|n| n['href'] =~ /comments/ }.first['href'] rescue nil
+  end
+  
+  set :link, :selector => '.title' do |elm|
+    elm.css('a').last["href"]
+  end
+
+  set :body, :selector => '.title' do |elm|
+    # ning's list page only has an excerpt of the body. No biggie,
+    # we'll just go grab it.
+    show_url = elm.css('a').last["href"]
+    Nokogiri::HTML(open(show_url)).css('.postbody').to_html
+  end
+  
+  pager do |doc|
+    doc.css('.pagination a').select{|a| a.text =~ /previous/i }.first['href'] rescue nil
+  end
+  
+  before_paginate do
+    # clearing empty rows. ning has shit markup 
+    # and very few relevant class names.
+    self.collection.delete_if {|post| post.css('h3').length == 0 }
+    
+    # logging for fun
+    puts "opening page: #{self.source}"
+    puts "collection size: #{self.collection.length}"
+    puts "*"*100
+  end
+
+end
+
+NingPost.source = 'http://cuwebd.ning.com/profiles/blog/list'
+@posts          = NingPost.all(:max_pages => 1)
+
+@posts.each do |post|
+  puts "#{post.pub_date} -- #{post.title}"
+  puts "*"*100
+end
+
+puts "total: #{@posts.length}"

data/graboid.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{graboid}
-  s.version = "0.3.2"
+  s.version = "0.3.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Christopher Burnett"]
-  s.date = %q{2010-06-14}
+  s.date = %q{2010-06-15}
   s.description = %q{web scraping made easier}
   s.email = %q{signalstatic@gmail.com}
   s.extra_rdoc_files = [
@@ -23,6 +23,9 @@ Gem::Specification.new do |s|
      "README.mdown",
      "Rakefile",
      "VERSION",
+     "examples/active_rain_post.rb",
+     "examples/live_journal_post.rb",
+     "examples/ning_post.rb",
      "graboid.gemspec",
      "lib/graboid.rb",
      "lib/graboid/entity.rb",
@@ -44,7 +47,10 @@ Gem::Specification.new do |s|
     "spec/fixtures/server.rb",
      "spec/graboid/entity_spec.rb",
      "spec/graboid_spec.rb",
-     "spec/spec_helper.rb"
+     "spec/spec_helper.rb",
+     "examples/active_rain_post.rb",
+     "examples/live_journal_post.rb",
+     "examples/ning_post.rb"
   ]
 
   if s.respond_to? :specification_version then

data/lib/graboid.rb

@@ -1,5 +1,9 @@
 %w{rubygems nokogiri open-uri active_support}.each { |f| require f }
 
+dir = Pathname(__FILE__).dirname.expand_path
+
+require dir + 'graboid/entity'
+
+
 module Graboid
-  require 'graboid/entity'
 end

data/lib/graboid/entity.rb

@@ -67,7 +67,7 @@ module Graboid
         attribute_map.inject({}) do |extracted_hash, at| 
           selector, processor       = at.last[:selector], at.last[:processor]
           node_collection           = self.mode == :html ? fragment.css(selector) : fragment.xpath(selector)
-          extracted_hash[at.first]  = processor.nil? ? node_collection.first.text : processor.call(node_collection.first) rescue ""
+          extracted_hash[at.first]  = processor.nil? ? node_collection.first.inner_html : processor.call(node_collection.first) rescue ""
 
           extracted_hash
         end

data/spec/spec_helper.rb

@@ -45,9 +45,7 @@ class PostWithPager
   end
 
   pager do |doc|
-    link = 'http://localhost:9393'+doc.css('a.next').first['href'] rescue nil
-    #puts link.inspect
-    link
+    'http://localhost:9393'+doc.css('a.next').first['href'] rescue nil
   end
 
   before_paginate do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: graboid
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 21
   prerelease: false
   segments: 
   - 0
   - 3
-  - 2
-  version: 0.3.2
+  - 3
+  version: 0.3.3
 platform: ruby
 authors: 
 - Christopher Burnett
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-06-14 00:00:00 -07:00
+date: 2010-06-15 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -78,6 +78,9 @@ files:
 - README.mdown
 - Rakefile
 - VERSION
+- examples/active_rain_post.rb
+- examples/live_journal_post.rb
+- examples/ning_post.rb
 - graboid.gemspec
 - lib/graboid.rb
 - lib/graboid/entity.rb
@@ -128,3 +131,6 @@ test_files:
 - spec/graboid/entity_spec.rb
 - spec/graboid_spec.rb
 - spec/spec_helper.rb
+- examples/active_rain_post.rb
+- examples/live_journal_post.rb
+- examples/ning_post.rb