graboid 0.3.5 → 0.4.0

data/Rakefile

@@ -12,7 +12,6 @@ begin
     gem.authors = ["Christopher Burnett"]
     gem.add_development_dependency "rspec", ">= 1.2.9"
     gem.add_dependency "nokogiri"
-    gem.add_dependency "activesupport"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-0.3.5
+0.4.0

data/graboid.gemspec

@@ -1,61 +1,57 @@
 # Generated by jeweler
 # DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
   s.name = %q{graboid}
-  s.version = "0.3.5"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Christopher Burnett"]
-  s.date = %q{2010-07-09}
+  s.date = %q{2011-02-22}
   s.description = %q{web scraping made easier}
   s.email = %q{signalstatic@gmail.com}
   s.extra_rdoc_files = [
     "LICENSE",
-     "README.mdown"
+    "README.mdown"
   ]
   s.files = [
     ".document",
-     ".gitignore",
-     "LICENSE",
-     "README.mdown",
-     "Rakefile",
-     "VERSION",
-     "examples/active_rain_post.rb",
-     "examples/live_journal_post.rb",
-     "examples/ning_post.rb",
-     "graboid.gemspec",
-     "lib/graboid.rb",
-     "lib/graboid/entity.rb",
-     "lib/graboid/scraper.rb",
-     "spec/fixtures/graboid.jpg",
-     "spec/fixtures/posts.html",
-     "spec/fixtures/server.rb",
-     "spec/fixtures/views/posts.erb",
-     "spec/graboid/entity_spec.rb",
-     "spec/graboid/scraper_spec.rb",
-     "spec/graboid_spec.rb",
-     "spec/spec.opts",
-     "spec/spec_helper.rb"
+    "LICENSE",
+    "README.mdown",
+    "Rakefile",
+    "VERSION",
+    "examples/active_rain_post.rb",
+    "examples/live_journal_post.rb",
+    "examples/ning_post.rb",
+    "graboid.gemspec",
+    "lib/graboid.rb",
+    "lib/graboid/entity.rb",
+    "lib/graboid/scraper.rb",
+    "spec/fixtures/graboid.jpg",
+    "spec/fixtures/posts.html",
+    "spec/fixtures/server.rb",
+    "spec/fixtures/views/posts.erb",
+    "spec/graboid/entity_spec.rb",
+    "spec/graboid/scraper_spec.rb",
+    "spec/graboid_spec.rb",
+    "spec/spec.opts",
+    "spec/spec_helper.rb"
   ]
   s.homepage = %q{http://github.com/twoism/graboid}
-  s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.3.7}
   s.summary = %q{web scraping made easy}
   s.test_files = [
+    "examples/active_rain_post.rb",
+    "examples/live_journal_post.rb",
+    "examples/ning_post.rb",
     "spec/fixtures/server.rb",
-     "spec/graboid/entity_spec.rb",
-     "spec/graboid/scraper_spec.rb",
-     "spec/graboid_spec.rb",
-     "spec/spec_helper.rb",
-     "examples/active_rain_post.rb",
-     "examples/live_journal_post.rb",
-     "examples/ning_post.rb",
-     "examples/reddit_post.rb",
-     "examples/tumblr_post.rb"
+    "spec/graboid/entity_spec.rb",
+    "spec/graboid/scraper_spec.rb",
+    "spec/graboid_spec.rb",
+    "spec/spec_helper.rb"
   ]
 
   if s.respond_to? :specification_version then
@@ -65,16 +61,13 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
-      s.add_runtime_dependency(%q<activesupport>, [">= 0"])
     else
       s.add_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_dependency(%q<nokogiri>, [">= 0"])
-      s.add_dependency(%q<activesupport>, [">= 0"])
     end
   else
     s.add_dependency(%q<rspec>, [">= 1.2.9"])
     s.add_dependency(%q<nokogiri>, [">= 0"])
-    s.add_dependency(%q<activesupport>, [">= 0"])
   end
 end
 

data/lib/graboid.rb

@@ -1,10 +1,20 @@
-%w{rubygems nokogiri open-uri active_support ostruct}.each { |f| require f }
+%w{rubygems nokogiri open-uri ostruct}.each { |f| require f }
 
 dir = Pathname(__FILE__).dirname.expand_path
 
 require dir + 'graboid/entity'
 require dir + 'graboid/scraper'
 
+class String
+  def underscore
+    self.to_s.gsub(/::/, '/').
+      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+      gsub(/([a-z\d])([A-Z])/,'\1_\2').
+      tr("-", "_").
+      downcase
+  end
+end
+
 module Graboid
   extend self
   
@@ -15,4 +25,4 @@ module Graboid
   def user_agent=(agent)
     @user_agent = agent
   end
-end
+end

data/lib/graboid/entity.rb

@@ -1,68 +1,87 @@
 module Graboid
   module Entity
-    
+
     def self.included klass
       klass.class_eval do
         extend  ClassMethods
         include InstanceMethods
-        warn "Deprecation Warning! Graboid::Entity - This module has been deprecated. See Graboid::Scraper."
-        write_inheritable_attribute(:attribute_map, {}) if attribute_map.nil?
+
+        inherited_attributes :attribute_map
+        @attribute_map = {}
       end
     end
-    
+
     module ClassMethods
-      
+
+      def inherited_attributes(*args)
+        @inherited_attributes ||= [:inherited_attributes]
+        @inherited_attributes += args
+        args.each do |arg|
+          class_eval %(
+            class << self; attr_accessor :#{arg} end
+          )
+        end
+        @inherited_attributes
+      end
+
+      def inherited(subclass)
+        @inherited_attributes.each do |inheritable_attribute|
+          instance_var = "@#{inheritable_attribute}"
+          subclass.instance_variable_set(instance_var, instance_variable_get(instance_var))
+        end
+      end
+
       def source
         @source
       end
-      
+
       def source=(src)
         @source = src
       end
-      
+
       def set name, opts={}, &block
-        opts.merge!(:selector   => ".#{name}")  unless opts[:selector].present?
+        opts.merge!(:selector   => ".#{name}")  if opts[:selector].nil?
         opts.merge!(:processor  => block)       if block_given?
-        
+
         attribute_map[name] = opts
       end
-      
+
       alias_method :field, :set
-      
+
       def selector selector
         @root_selector = selector
       end
-            
+
       alias_method :root, :selector
 
       def root_selector
         @root_selector || inferred_selector
       end
-      
+
       def inferred_selector
         @inferred_selector ||= ".#{self.to_s.underscore}"
       end
-      
+
       def doc
         eval "Nokogiri::#{self.mode.to_s.upcase}(read_source)"
       end
-      
+
       def collection
         @collection ||= []
       end
-      
+
       def collection=(col)
         @collection = col
       end
-      
+
       def attribute_map
         read_inheritable_attribute :attribute_map
       end
-      
+
       def extract_instance fragment
         new(hash_map(fragment))
       end
-      
+
       def hash_map fragment
         attribute_map.inject({}) do |extracted_hash, at| 
           selector, processor       = at.last[:selector], at.last[:processor]
@@ -72,7 +91,7 @@ module Graboid
           extracted_hash
         end
       end
-      
+
       def all_fragments
         return page_fragments if @pager.nil?
         old_source = self.source
@@ -85,13 +104,13 @@ module Graboid
         self.source = old_source
         self.collection
       end
-      
+
       def paginate
         next_page_url = @pager.call(doc) rescue nil
         self.source   = next_page_url
         self.current_page += 1
       end
-      
+
       def next_page?
         if max_pages.zero?
           return true unless @pager.call(doc).nil?
@@ -99,23 +118,23 @@ module Graboid
           current_page <= max_pages-1
         end
       end
-      
+
       def page_fragments
         doc.css(root_selector)
       end
-      
+
       def all opts={}
         reset_context
-        self.max_pages = opts[:max_pages] if opts[:max_pages].present?
+        self.max_pages = opts[:max_pages] unless opts[:max_pages].nil?
         all_fragments.collect{ |frag| extract_instance(frag) }
       end
-      
+
       def reset_context
         self.collection   = []
         self.current_page = 0
         self.max_pages    = 0
       end
-      
+
       def read_source
         case self.source
           when /^http[s]?:\/\//
@@ -124,36 +143,36 @@ module Graboid
             self.source
         end
       end
-      
+
       def pager &block
         @pager = block
       end
-      
+
       def mode
         @mode ||= :html
       end
-      
+
       def mode=(m)
         raise ArgumentError unless [:html, :xml].include?(m)
         @mode = m
       end
-      
+
       def max_pages
         @max_pages ||= 0
       end
-      
+
       def max_pages=num
         @max_pages = num
       end
-      
+
       def current_page
         @current_page ||= 0
       end
-      
+
       def current_page=num
         @current_page = num
       end
-      
+
       instance_eval do
         [:before, :after].each do |prefix|
           [:paginate, :extract].each do |suffix|
@@ -163,29 +182,29 @@ module Graboid
             end
             define_method "run_#{method_name}_callbacks" do
               ivar = instance_variable_get("@#{method_name}")
-              self.class_eval { ivar.call } unless ivar.nil?
+              class_eval { ivar.call } unless ivar.nil?
             end
           end
-        end 
-      end     
-      
+        end
+      end
+
     end # ClassMethods
-    
+
     module InstanceMethods
-      
+
       def initialize opts={}
         opts.each do |k,v|
-          self.class_eval do
+          self.class.class_eval do
             define_method k do
               v
             end
           end
         end
       end
-      
+
       def attribute_map
         self.class.attribute_map
       end
     end # InstanceMethods
   end
-end
+end

data/lib/graboid/scraper.rb

@@ -1,19 +1,35 @@
 module Graboid
+  
   module Scraper
     def self.included klass
       klass.class_eval do
         extend  ClassMethods
         include InstanceMethods
-        
-        write_inheritable_attribute(:attribute_map, {}) if attribute_map.nil?
-        write_inheritable_attribute(:callbacks, {})     if callbacks.nil?
+
+        inherited_attributes :attribute_map, :callbacks
+        @attribute_map  = {}
+        @callbacks      = {}
       end
     end
-    
+
     module ClassMethods
-      
-      def attribute_map
-        read_inheritable_attribute :attribute_map
+
+      def inherited_attributes(*args)
+        @inherited_attributes ||= [:inherited_attributes]
+        @inherited_attributes += args
+        args.each do |arg|
+          class_eval %(
+            class << self; attr_accessor :#{arg} end
+          )
+        end
+        @inherited_attributes
+      end
+
+      def inherited(subclass)
+        @inherited_attributes.each do |inheritable_attribute|
+          instance_var = "@#{inheritable_attribute}"
+          subclass.instance_variable_set(instance_var, instance_variable_get(instance_var))
+        end
       end
       
       def callbacks
@@ -41,7 +57,7 @@ module Graboid
       alias_method :root, :selector
 
       def set name, opts={}, &block
-        opts.merge!(:selector   => ".#{name}")  unless opts[:selector].present?
+        opts.merge!(:selector   => ".#{name}")  if opts[:selector].nil?
         opts.merge!(:processor  => block)       if block_given?
         
         attribute_map[name] = opts
@@ -60,14 +76,14 @@ module Graboid
     
     module InstanceMethods
       def initialize opts={}, &block
-        raise ArgumentError unless opts[:source].present?
+        raise ArgumentError if opts[:source].nil?
         self.source = opts[:source]
       end
       
       def all opts={}, reload=false
         return self.collection if reload and !self.collection.empty?
         reset_context
-        self.max_pages = opts[:max_pages] if opts[:max_pages].present?
+        self.max_pages = opts[:max_pages] unless opts[:max_pages].nil?
         all_fragments.collect{ |frag| extract_instance(frag) }
       end
       
@@ -202,11 +218,11 @@ module Graboid
         [:paginate, :extract].each do |suffix|
           method_name = "#{prefix}_#{suffix}"
           define_method "run_#{method_name}_callbacks" do
-            self.instance_eval &callbacks[method_name.to_sym] if callbacks[method_name.to_sym].present?
+            self.instance_eval &callbacks[method_name.to_sym] unless callbacks[method_name.to_sym].nil?
           end
         end
       end
       
     end
   end
-end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: graboid
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 15
   prerelease: false
   segments: 
   - 0
-  - 3
-  - 5
-  version: 0.3.5
+  - 4
+  - 0
+  version: 0.4.0
 platform: ruby
 authors: 
 - Christopher Burnett
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-09 00:00:00 -07:00
+date: 2011-02-22 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -48,20 +48,6 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: activesupport
-  prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :runtime
-  version_requirements: *id003
 description: web scraping made easier
 email: signalstatic@gmail.com
 executables: []
@@ -73,7 +59,6 @@ extra_rdoc_files:
 - README.mdown
 files: 
 - .document
-- .gitignore
 - LICENSE
 - README.mdown
 - Rakefile
@@ -94,15 +79,13 @@ files:
 - spec/graboid_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-- examples/reddit_post.rb
-- examples/tumblr_post.rb
 has_rdoc: true
 homepage: http://github.com/twoism/graboid
 licenses: []
 
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -131,13 +114,11 @@ signing_key:
 specification_version: 3
 summary: web scraping made easy
 test_files: 
+- examples/active_rain_post.rb
+- examples/live_journal_post.rb
+- examples/ning_post.rb
 - spec/fixtures/server.rb
 - spec/graboid/entity_spec.rb
 - spec/graboid/scraper_spec.rb
 - spec/graboid_spec.rb
 - spec/spec_helper.rb
-- examples/active_rain_post.rb
-- examples/live_journal_post.rb
-- examples/ning_post.rb
-- examples/reddit_post.rb
-- examples/tumblr_post.rb

data/.gitignore

@@ -1,21 +0,0 @@
-## MAC OS
-.DS_Store
-
-## TEXTMATE
-*.tmproj
-tmtags
-
-## EMACS
-*~
-\#*
-.\#*
-
-## VIM
-*.swp
-
-## PROJECT::GENERAL
-coverage
-rdoc
-pkg
-
-## PROJECT::SPECIFIC

data/examples/reddit_post.rb

@@ -1,35 +0,0 @@
-dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
-require File.join(dir, 'graboid')
-
-class RedditEntry
-  include Graboid::Scraper
-
-  selector '.entry'
-
-  set :title
-  set :domain, :selector => '.domain a'
-  
-  set :link,   :selector => '.title' do |entry| 
-    entry.css('a').first['href'] 
-  end
-
-  page_with do |doc|
-    self.doc.css('p.nextprev a').select{|a| a.text =~ /next/i  }.first['href']
-  end
-
-  before_paginate do
-    puts "opening page: #{self.source}"
-    puts "collection size: #{self.collection.length}"
-    puts "#{"*"*100}"
-  end
-
-end
-
-@posts = RedditEntry.new( :source => 'http://reddit.com' ).all( :max_pages => 3 )
-
-@posts.each do |p| 
-  puts "title: #{p.title}"
-  puts "domain: #{p.domain}"
-  puts "link: #{p.link}"
-  puts "#{"*"*100}"
-end

data/examples/tumblr_post.rb

@@ -1,33 +0,0 @@
-dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
-require File.join(dir, 'graboid')
-
-class TumblrEntry
-  include Graboid::Scraper
-  TUMBLR_CHUNK_SIZE = 20
-
-  selector 'post'
-
-  set :title, :selector => 'regular-title'
-
-  page_with do |doc|
-    next_tumblr_page
-  end
-  
-  def next_tumblr_page
-    return nil if self.doc.css('post').empty?
-    "#{self.original_source}?start=#{self.current_page*TUMBLR_CHUNK_SIZE}"
-  end  
-
-  before_paginate do
-    puts "opening page: #{self.source}"
-    puts "collection size: #{self.collection.length}"
-    puts "#{"*"*100}"
-  end
-
-end
-
-@posts = TumblrEntry.new( :source => 'http://chrisburnett.tumblr.com/api/read' ).all
-
-@posts.each do |p| 
-  puts "title: #{p.title}"
-end