eagleclaw 0.1.1

data/README.md

@@ -0,0 +1,4 @@
+# EagleClaw
+
+EagleClaw is a Ruby library for building screen scrapers. It’s very much a
+work-in-progress.

data/bin/eagleclaw

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'eagleclaw/runner'
+
+EagleClaw::Runner.new.run!

data/lib/eagleclaw/browser.rb

@@ -0,0 +1,24 @@
+require 'mechanize'
+
+module EagleClaw
+  ##
+  # Add browsing capabilities to a class.
+  module Browser
+    ##
+    # This browser's `Mechanize` agent.
+    #
+    # @return [Mechanize]
+    #
+    # @see http://mechanize.rubyforge.org/mechanize/
+    def agent
+      @agent ||= Mechanize.new
+    end
+    
+    ##
+    # If a non-existent method is called, pass it along to the `Mechanize`
+    # agent.
+    def method_missing(name, *args, &block)
+      agent.send(name, *args, &block)
+    end
+  end # module Browser
+end # module EagleClaw

data/lib/eagleclaw/callbacks.rb

@@ -0,0 +1,64 @@
+module EagleClaw
+  module Callbacks
+    ##
+    # Register a callback for a given context.
+    #
+    # @param [Object] context
+    #   an object (such as a symbol or list of symbols) which refers to a
+    #   certain callback context.
+    # @param [optional, Symbol] meth the name of the callback method.
+    # 
+    # @overload register(context, :method_name)
+    #   Register a method as a callback.
+    # @overload register(context, &block)
+    #   Register a block as a callback.
+    # 
+    # @example Registering a method as a callback
+    #   class MyCLS
+    #     include Callbacks
+    #     
+    #     register(:preprocessing, :setup_db)
+    #     
+    #     def setup_db
+    #       @database = DB.connect("username", "password")
+    #     end
+    #   end
+    #
+    # @example Registering a block as a callback
+    #   class MyCLS
+    #     include Callbacks
+    #     
+    #     register :preprocessing do
+    #       @database = DB.connect("username", "password")
+    #     end
+    #   end
+    #
+    def register(context, meth = nil, &block)
+      callback = block_given? ? block : meth
+      ((@callbacks ||= {})[context] ||= []) << callback
+    end
+    
+    ##
+    # Run the callbacks for a given context.
+    #
+    # @param context
+    # @param [optional, Object] recipient the object to run methods/procs on.
+    # @return [nil]
+    def run_callbacks(context, recipient = self)
+      (@callbacks[context] || []).each { |callback| run_proc(callback, recipient) }
+      nil
+    end
+    
+    private
+    
+    ##
+    # Run a given method or `Proc` on an instance.
+    def run_proc(meth, recipient = self)
+      if meth.is_a? Symbol
+        recipient.send(meth)
+      else
+        recipient.instance_eval(&meth)
+      end
+    end
+  end
+end

data/lib/eagleclaw/constmatch.rb

@@ -0,0 +1,30 @@
+module EagleClaw
+  ##
+  # Mixin to allow an object to act as a case-insensitive hash against its own
+  # constants. This should be used with singleton metaclasses, like so:
+  # 
+  #     class MyClass
+  #       class << self
+  #         include EagleClaw::ConstantMatcher
+  #       end
+  #     end
+  #
+  module ConstantMatcher
+    ##
+    # Retrieve the constant for the given symbol. Matching will be performed
+    # case-insensitively.
+    # 
+    # @param [Symbol] constant
+    #   A symbol referring to the constant to get.
+    #
+    # @example Get the JSON EagleClaw formatter
+    #   EagleClaw::Formatters[:json] => EagleClaw::Formatters::JSON
+    #
+    def [](constant)
+      constants.each do |const|
+        return const_get(const) if const.downcase == constant.to_s.downcase
+      end
+      raise NameError, "Constant #{constant.inspect} not found"
+    end
+  end
+end

data/lib/eagleclaw/formatters/csv.rb

@@ -0,0 +1,19 @@
+require 'csv'
+
+module EagleClaw::Formatters
+  module CSV
+    include ::EagleClaw::Formatter
+    
+    def self.format(data, problems)
+      keys = data.first.keys
+      s = StringIO.new
+      ::CSV::Writer.generate(s) do |out|
+        out << keys
+        data.each do |datum|
+          out << keys.map(&datum.method(:[]))
+        end
+      end
+      s.string
+    end
+  end
+end

data/lib/eagleclaw/formatters/json.rb

@@ -0,0 +1,13 @@
+require 'rubygems'
+gem 'json', '>= 1.4.3'
+require 'json'
+
+module EagleClaw::Formatters
+  module JSON
+    include ::EagleClaw::Formatter
+    
+    def self.format(data, problems)
+      {"data" => data, "problems" => problems}.to_json
+    end
+  end
+end

data/lib/eagleclaw/formatters.rb

@@ -0,0 +1,18 @@
+require 'eagleclaw/constmatch'
+
+module EagleClaw
+  ##
+  # This is just a placeholder at the moment; you should include it in your
+  # formatter class in case it gets some useful methods in the future.
+  module Formatter
+  end
+  
+  module Formatters
+    class << self
+      include EagleClaw::ConstantMatcher
+    end
+    
+    autoload :JSON, 'eagleclaw/formatters/json'
+    autoload :CSV,  'eagleclaw/formatters/csv'
+  end
+end

data/lib/eagleclaw/runner.rb

@@ -0,0 +1,50 @@
+require 'optparse'
+require 'ostruct'
+
+module EagleClaw
+  class Runner
+    attr_accessor :options
+    
+    def initialize
+      @options = OpenStruct.new(:require => [], :format => :json)
+    end
+    
+    def parser
+      @parser ||= OptionParser.new do |opts|
+        opts.banner = "Usage: #{$0} [options] ScraperName"
+        
+        # ---- MAIN ----
+        
+        opts.on('-r', '--require LIBRARY',
+                "Require LIBRARY before loading scrapers") do |lib|
+          options.require << lib
+        end
+        
+        opts.on('-o', '--output FORMAT',
+                "Output data in format FORMAT (default: JSON)") do |format|
+          options.format = format.downcase.to_sym
+        end
+        
+        # ---- TAIL ----
+        
+        opts.on_tail('-h', '--help', "Show this help message") { puts opts and exit }
+      end
+    end
+    
+    def parse!(args = ARGV)
+      parser.parse!(args)
+    end
+    
+    def run!(args = ARGV)
+      parse!(args)
+      @options.require.each { |lib| require(lib) }
+      formatter = EagleClaw::Formatters[@options.format]
+      
+      scraper = EagleClaw::Scrapers[args.shift.downcase.to_sym].new
+      scraper.run
+      data, problems = scraper.data, scraper.problems
+      output = formatter.format(data, problems)
+      puts output
+    end
+  end
+end

data/lib/eagleclaw/scrapers.rb

@@ -0,0 +1,9 @@
+require 'eagleclaw/constmatch'
+
+module EagleClaw
+  module Scrapers
+    class << self
+      include EagleClaw::ConstantMatcher
+    end
+  end
+end

data/lib/eagleclaw/string.rb

@@ -0,0 +1,15 @@
+require 'digest/sha1'
+
+class String
+  def starts_with?(string)
+    !! (self =~ Regexp.new('^' + Regexp.escape(string)))
+  end
+  
+  def ends_with?(string)
+    !! (self =~ Regexp.new(Regexp.escape(string) + '$'))
+  end
+  
+  def sha1
+    (Digest::SHA1.new << self).hexdigest
+  end
+end

data/lib/eagleclaw/xml.rb

@@ -0,0 +1,38 @@
+require 'nokogiri'
+
+class Nokogiri::XML::Element
+  
+  ##
+  # Keep consuming elements until the block returns `true`.
+  #
+  # @param [Symbol] method
+  #   The method to call on the current element to get the next one. The default
+  #   is to use `:next_element`. Use `:next` to include text elements in the
+  #   iteration.
+  # 
+  # @yield [Nokogiri::XML::Element] element
+  #   The current element. If the block returns `true` (or any non-false value)
+  #   then this is what the method will return.
+  def next_until(method = :next_element)
+    current = self
+    until yield(current)
+      current = current.send(method)
+    end
+    current
+  end
+  
+  ##
+  # Keep consuming elements until the block returns `false`.
+  #
+  # The behaviour of this method is identical to {#next_until}, only it will
+  # keep iterating until the block yields a *false* value instead of a true one.
+  #
+  # @see #next_until
+  def next_while(method = :next_element)
+    current = self
+    while yield(current)
+      current = current.send(method)
+    end
+    current
+  end
+end

data/lib/eagleclaw.rb

@@ -0,0 +1,139 @@
+require 'eagleclaw/browser'
+require 'eagleclaw/callbacks'
+require 'eagleclaw/formatters'
+require 'eagleclaw/scrapers'
+require 'eagleclaw/string'
+require 'eagleclaw/xml'
+
+module EagleClaw
+  class Scraper
+    class << self
+      include ::EagleClaw::Callbacks
+      
+      attr_accessor :properties
+      
+      ##
+      # Define a pre-processor to run in a certain context.
+      #
+      # @param [Symbol] context either `:each` or `:all`.
+      # @param [optional, Symbol] meth name of method to call.
+      # @return [nil]
+      #
+      # @overload before(:each, :method_name)
+      #   Run the given method before each component of the run.
+      # @overload before(:all, :method_name)
+      #   Run the given method before the run itself.
+      # @overload before(:each, &block)
+      #   Run the given block (using `instance_eval`) before each component of
+      #   the run.
+      # @overload before(:all, &block)
+      #   Run the given block (using `instance_eval`) before the run itself.
+      #
+      # @example Fetch a page before the run
+      #   before(:all) do
+      #     agent.get("http://google.com/")
+      #   end
+      # 
+      # @example Reset the page before each component of the run
+      #   before(:each) do
+      #     agent.get("http://google.com/")
+      #   end
+      #
+      def before(context, meth = nil, &block)
+        register([:before, context], meth, &block)
+      end
+      
+      ##
+      # Define a post-processor to run in a certain context.
+      # 
+      # @param [Symbol] context either `:each` or `:all`.
+      # @param [optional, Symbol] meth name of method to call.
+      # @return [nil]
+      # 
+      # @overload after(:each, :method_name)
+      #   Run the given method after each component of the run.
+      # @overload after(:all, :method_name)
+      #   Run the given method after the run itself.
+      # @overload after(:each, &block)
+      #   Run the given block (using `instance_eval`) after each component of
+      #   the run.
+      # @overload after(:all, &block)
+      #   Run the given block (using `instance_eval`) after the entire run.
+      #
+      # @see before
+      #
+      def after(context, meth = nil, &block)
+        register([:after, context], meth, &block)
+      end
+      
+      def prop(prop_name, meth = nil, &block)
+        (@properties ||= []) << prop_name.to_sym
+        register([:property, prop_name.to_sym], meth, &block)
+      end
+    end # class << self (Scraper)
+    
+    include Browser
+    
+    ##
+    # A `Hash` which holds data collected during a run.
+    #
+    # @see #initialize
+    # @see #reset
+    attr_accessor :data
+    
+    ##
+    # An `Array` which collects 
+    attr_accessor :problems
+    
+    ##
+    # Create a new {Scraper} instance.
+    #
+    # By default, just sets {#data @data} and {#problems @problems} to empty
+    # `Array`s.
+    #
+    def initialize
+      @data = []
+      @problems = []
+    end
+    
+    ##
+    # Reset this scraper instance's state.
+    # 
+    # The default version of this method just clears {#data @data} and
+    # {#problems @problems}.
+    # 
+    # @return [nil]
+    # @abstract Subclass and extend to reset the scraper state.
+    #
+    def reset
+      data.clear
+      problems.clear
+    end
+    
+    ##
+    # Run the scraper.
+    #
+    # Operating procedure:
+    #
+    # 1.  Run {Scraper.before before(:all)} blocks.
+    # 2.  For each property (defined with {Scraper.prop prop(:prop_name)}):
+    #     1.  Run {Scraper.before before(:each)} blocks.
+    #     2.  Run the property itself.
+    #     3.  Runs {Scraper.after after(:each)} blocks.
+    # 3.  Runs {Scraper.after after(:all)} blocks.
+    # 4.  Return {#data data}.
+    #
+    # @see #reset
+    # @see #data
+    def run
+      self.class.run_callbacks([:before, :all], self)
+      self.class.properties.each do |property|
+        self.class.run_callbacks([:before, :each], self)
+        self.class.run_callbacks([:property, property], self)
+        self.class.run_callbacks([:after, :each], self)
+      end
+      self.class.run_callbacks([:after, :all], self)
+      data
+    end
+  end # class Scraper
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification 
+name: eagleclaw
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 1
+  version: 0.1.1
+platform: ruby
+authors: 
+- Zachary Voase
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-05-10 00:00:00 +00:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mechanize
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 4
+        - 1
+        version: 1.4.1
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 3
+        - 0
+        version: 1.3.0
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: yard
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 5
+        - 4
+        version: 0.5.4
+  type: :development
+  version_requirements: *id004
+description: eagleclaw is a small library to help build screen-scrapers
+email: z@zacharyvoase.com
+executables: 
+- eagleclaw
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.md
+- lib/eagleclaw/browser.rb
+- lib/eagleclaw/callbacks.rb
+- lib/eagleclaw/constmatch.rb
+- lib/eagleclaw/formatters/csv.rb
+- lib/eagleclaw/formatters/json.rb
+- lib/eagleclaw/formatters.rb
+- lib/eagleclaw/runner.rb
+- lib/eagleclaw/scrapers.rb
+- lib/eagleclaw/string.rb
+- lib/eagleclaw/xml.rb
+- lib/eagleclaw.rb
+has_rdoc: false
+homepage: http://github.com/zacharyvoase/eagleclaw
+licenses: 
+- Public Domain
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 8
+      - 6
+      version: 1.8.6
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: A small screen-scraping library
+test_files: []
+