imw 0.2.18 → 0.3.0

data/examples/nibbler_test.rb

@@ -0,0 +1,24 @@
+require 'rubygems'
+require 'nibbler'
+require 'open-uri'
+
+class Game < Nibbler
+  element 'td/i/a' => :name
+  element 'td/a'   => :publisher
+  element 'td[2]'  => :year
+end
+
+class Table < Nibbler
+
+  elements "//h2/span[@id='Licensed_games']/following::table[1]/tr"   => :licensed_games,   :with => Game
+  elements "//h2/span[@id='Unlicensed_games']/following::table[1]/tr" => :unlicensed_games, :with => Game
+
+end
+
+foo = Table.parse open('http://en.wikipedia.org/wiki/List_of_Nintendo_Entertainment_System_games')
+
+puts foo.unlicensed_games[1].inspect
+
+
+
+

data/examples/script.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'open-uri'
+require 'nokogiri'
+require 'fastercsv'
+
+url = "http://www.gamespot.com/games.html?platform=19&category=&type=games&mode=all&sort=title&sortdir=desc&page="
+pages = (0..27)
+
+FasterCSV.open("nes_gamespot.csv","w", :write_headers => true, :headers => %w[ title category release ]) do |csv|
+  pages.each do |page|
+    doc = Nokogiri::HTML(open(url + page.to_s))
+    doc.xpath("//tr").each do |node|
+      game = node.content.split("\n").map {|i| i.strip }.reject { |i| i.length == 0 }
+      csv << game unless game.include?("Release Date")
+    end
+  end
+end

data/lib/imw.rb

@@ -1,60 +1,16 @@
 require 'rubygems'
-require 'bundler/setup'
-require 'imw/boot'
 require 'imw/utils'
+require 'imw/error'
+require 'imw/uri'
 
-# The Infinite Monkeywrench (IMW) is a Ruby library for ripping,
-# extracting, parsing, munging, and packaging datasets.  It allows you
-# to handle different data formats transparently as well as organize
-# transformations of data as a network of dependencies (a la Make or
-# Rake).
-#
-# IMW has a few central concepts: resources, metadata, datasets,
-# workflows, and repositories.
-#
-# Resources represent individual data resources like local files,
-# websites, databases, &c.  An IMW::Resource is typically instantiated
-# via IMW.open, with IMW doing the work of figuring out what to return
-# based on the URI passed in.
-#
-# A Resource can have a schema which describes the fields in its data.
-# IMW::Metadata consists of classes which describe fields.
-#
-# Datasets represent collections of related data resources ..  An
-# IMW::Dataset comes with a pre-defined (but customizable) workflow
-# that takes data resources through several steps: rip, parse, munge,
-# and package.  The workflow leverages Rake and so the various tasks
-# that are necessary to process the data till it is nice and pretty
-# can all be linked with dependencies.
-#
-# Repositories are collections of datasets and it is on these
-# collections that the +imw+ command line tool operates.
 module IMW
+
+  autoload :Recordizer,      'imw/recordizer'
   autoload :Resource,        'imw/resource'
   autoload :Schemes,         'imw/schemes'
-  autoload :Archives,        'imw/archives'
-  autoload :CompressedFiles, 'imw/compressed_files'
-  autoload :Formats,         'imw/formats'  
-  autoload :Tools,           'imw/tools'
+  autoload :Formats,         'imw/formats'
   autoload :Parsers,         'imw/parsers'
-  autoload :Dataset,         'imw/dataset'  
-  autoload :Repository,      'imw/repository'
-  autoload :Metadata,        'imw/metadata'
 
-  # Open a resource at the given +uri+.  The resource will
-  # automatically be extended by modules which make sense given the
-  # +uri+.
-  # 
-  # See the documentation for IMW::Resource and the various modules
-  # within IMW::Resources for more information and options.
-  #
-  # Passing in an IMW::Resource will simply return it.
-  #
-  # @param  [String, Addressable::URI, IMW::Resource] obj the URI to open
-  # @param [Hash] options
-  # @option options [Array<String,Module>] as same as <tt>:use_modules</tt> in IMW::Resource.extend_instance!
-  # @option options [Array<String,Module>] without same as <tt>:skip_modules</tt> in IMW::Resource.extend_instance!  
-  # @return [IMW::Resource] the resulting resource, property extended for the given URI
   def self.open obj, options={}, &block
     if obj.is_a?(IMW::Resource)
       resource = obj
@@ -63,103 +19,35 @@ module IMW
       options[:skip_modules] ||= (options[:without] || [])
       resource = IMW::Resource.new(obj, options)
     end
-    if block_given?
-      yield resource
-      resource.close
-    else
-      resource
-    end
   end
 
-  # Open (and create if necessary) a directory at the given URI.
-  #
-  # Will automatically create directories recursively.  Options will
-  # be passed to IMW.open and interpreted appropriately.  If a block
-  # is passed, the directory will be created before the block is
-  # yielded to.
-  #
-  # @param [String, IMW::Resource] uri
-  # @param [Hash] options
-  # @return [IMW::Resource]
-  def self.dir! uri, options={}, &block
-    if block_given?
-      new_dir = open(uri, options.merge(:as => (options[:as] || []) + [Schemes::Local::LocalDirectory])) do |d|
-        new_dir.create
-        yield
+  class Resource
+
+    attr_reader :uri
+
+    def initialize(uri, mode='r')
+      raise FileModeError.new("'#{mode}' is not a valid access mode") unless valid_modes.include? mode
+      @uri = Uri.new(uri)
+    end
+
+    def self.open(uri, mode='r', &blk)
+      resource = Resource.new(uri, mode)
+      if block_given?
+        yield resource
+      else
+        return resource
       end
-    else
-      new_dir = open(uri, options.merge(:as => (options[:as] || []) + [Schemes::Local::LocalDirectory]))
-      new_dir.create
     end
-    new_dir
-  end
-  
-  # Works the same way as IMW.open except opens the resource for
-  # writing.
-  #
-  # @param  [String, Addressable::URI] uri the URI to open
-  # @return [IMW::Resource] the resultng resource, properly extended for the given URI and opened for writing.
-  def self.open! uri, options={}, &block
-    open(uri, options.merge(:mode => 'w'), &block)
-  end
 
-  # The default repository in which to place datasets.  See the
-  # documentation for IMW::Repository for more information on how
-  # datasets and repositories fit together.
-  #
-  # @return [IMW::Repository] the default IMW repository
-  def self.repository
-    @@repository ||= IMW::Repository.new
-  end
+    def self.exists? resource
+      true
+    end
 
-  # Create a dataset and put it in the default IMW repository.
-  #
-  # Evaluates the given block in the context of the new dataset.  This
-  # allows you to define tasks, add paths, and use defined metadata in
-  # an elegant way.
-  #
-  #   IMW.dataset :my_dataset do
-  #   
-  #     # Define some paths we're going to use
-  #     add_path :original, :rawd, 'original.csv'
-  #     add_path :filtered, :fixd, 'filtered.csv'
-  #     add_path :package,  :pkgd, 'filtered.tar.bz2'
-  #
-  #     # Copy a CSV filefrom a website to this machine.
-  #     rip do
-  #       open('http://mysite.com/data_archives/2010/03/03.csv').cp(path_to(:original))
-  #     end
-  #
-  #     # Filter the original CSV data by the
-  #     # <tt>meets_some_condition?</tt> method we define elsewhere...
-  #     munge do
-  #       open!(path_to(:filtered)) do |filtered|
-  #         open(path_to(:original)).each do |row|
-  #           filtered << row if meets_some_condition?(row)
-  #       end
-  #     end
-  #
-  #     # Compress the filtered data to an archive.
-  #     package do
-  #       open(path_to(:filtered)).compress.mv(path_to(:package))
-  #     end
-  #   end
-  #
-  # See the <tt>/examples</tt> directory of the IMW distribution for
-  # more examples.
-  #
-  # @param [Symbol, String] handle the handle to identify this dataset with
-  # @param [Hash]   options a hash of options (see IMW::Dataset)
-  # @return [IMW::Dataset] the new dataset
-  def self.dataset handle, options={}, &block
-    d = IMW::Dataset.new(handle, options.merge(:repository => IMW.repository))
-    d.instance_eval(&block) if block_given?
-    d
-  end
+    private
+    def valid_modes
+      %w[ r w a ]
+    end
 
-end
+  end
 
-# Works just like IMW.dataset but defined at a top-level scope.
-def dataset handle, options={}, &block
-  IMW.dataset(handle, options, &block)
 end

data/lib/imw/error.rb

@@ -0,0 +1,9 @@
+module IMW
+
+  Error = Class.new(StandardError)
+
+  FileModeError = Class.new(Error)
+
+  InvalidFormatError = Class.new(Error)
+
+end

data/lib/imw/recordizer.rb

@@ -0,0 +1,8 @@
+module IMW
+  module Recordizer
+
+    autoload :StringSliceRecordizer,  'imw/recordizer/string_slice_recordizer'
+    autoload :HTMLSelectorRecordizer, 'imw/recordizer/html_selector_recordizer'
+
+  end
+end

data/lib/imw/recordizer/html_selector_recordizer.rb

@@ -0,0 +1,86 @@
+module IMW
+  module Recordizer
+    class HTMLSelectorRecordizer
+
+      def self.element(*args, &block)
+        selector, name, delegate = parse_rule_declaration(*args, &block)
+        rules[name] = [selector, delegate]
+        attr_accessor name
+        name
+      end
+
+      def self.elements(*args, &block)
+        name = element(*args, &block)
+        rules[name] << true
+      end
+
+      def initialize
+        self.class.rules.each { |name, (s, k, plural)| send("#{name}=", []) if plural }
+      end
+
+      def self.recordize(doc)
+        self.new.recordize(doc)
+      end
+
+      def recordize(doc)
+        self.class.rules.each do |target, (selector, delegate, plural)|
+          if plural
+            send(target).concat doc.search(selector).map { |i| parse_result(i, delegate) }
+          else
+            send("#{target}=", parse_result(doc.at(selector), delegate))
+          end
+        end
+        self.to_hash
+      end
+
+      def to_hash
+        converter = lambda { |obj| obj.respond_to?(:to_hash) ? obj.to_hash : obj }
+        self.class.rules.keys.inject({}) do |hash, name|
+          value = send(name)
+          hash[name.to_sym] = Array === value ? value.map(&converter) : converter[value]
+          hash
+        end
+      end
+
+      protected
+
+      def parse_result(node, delegate)
+        if delegate
+          delegate.respond_to?(:call) ? delegate.call(node) : delegate.recordize(node)
+        elsif node.respond_to? :inner_text
+          node.inner_text
+        else
+          node
+        end unless node.nil?
+      end
+
+      private
+
+      def self.rules
+        @rules ||= {}
+      end
+
+      def self.inherited(subclass)
+        subclass.rules.update self.rules
+      end
+
+      # Rule declaration forms:
+      #
+      #   { 'selector' => :property, :with => delegate }
+      #     #=> ['selector', :property, delegate]
+      #
+      #   :title
+      #     #=> ['title', :title, nil]
+      def self.parse_rule_declaration(*args, &block)
+        options, name = Hash === args.last ? args.pop : {}, args.first
+        delegate = options.delete(:with)
+        selector, property = name ? [name.to_s, name.to_sym] : options.to_a.flatten
+        raise ArgumentError, "invalid rule declaration: #{args.inspect}" unless property
+        # eval block in context of a new scraper subclass
+        delegate = Class.new(delegate || Nibbler, &block) if block_given?
+        return selector, property, delegate
+      end
+
+    end
+  end
+end

data/lib/imw/recordizer/string_slice_recordizer.rb

@@ -0,0 +1,39 @@
+module IMW
+  module Recordizer
+    class StringSliceRecordizer
+
+      attr_reader :schema
+
+      def initialize ranges
+        @schema = ranges
+      end
+
+      def recordize line
+        format = schema
+        case format
+        when Array then slice_by_array(line, format)
+        when Hash  then slice_by_hash(line, format)
+        end
+      end
+
+      def slice_range string, range
+        string.slice(range).strip
+      end
+
+      def slice_by_array string, format
+        format.map { |range| slice_range(string, range) }
+      end
+
+      def slice_by_hash string, format
+        format.inject({}) do |hsh, (key, val)|
+          case val
+          when Range then hsh[key] = slice_range(string, val)
+          when Hash  then hsh[key] = slice_by_hash(string, val)
+          end
+          hsh
+        end
+      end
+
+    end
+  end
+end

data/lib/imw/resource.rb

@@ -1,80 +1,10 @@
 require 'imw/utils/has_uri'
 
 module IMW
-
-  # A resource can be anything addressable via a URI.  Examples
-  # include local files, remote files, webpages, &c.
-  #
-  # The IMW::Resource class takes a URI as input and then dynamically
-  # extends itself with appropriate modules from IMW.  As an example,
-  # calling
-  #
-  #   my_archive = IMW::Resource.new('/path/to/my/archive.tar.bz2')
-  #
-  # would return an IMW::Resource extended by
-  # IMW::Archives::Tarbz2 (among other modules) which
-  # therefore has methods for extracting, listing, and appending to
-  # the archive.
-  #
-  # Modules are so extended based on handlers defined in the
-  # <tt>imw/resources</tt> directory and accessible via
-  # IMW::Resource.handlers.  You can define your own handlers by
-  # defining the constant IMW::Resource::USER_DEFINED_HANDLERS in your
-  # configuration file.
-  #
-  # The modules extending a particular IMW::Resource instance can be
-  # listed as follows
-  #
-  #   my_archive.modules #=> [IMW::Local::Base, IMW::Local::File, IMW::Local::Compressible, IMW::Archives::Tarbz2]
-  #
-  # By default, resources are opened for reading.  Passing in the
-  # appropriate <tt>:mode</tt> option changes this:
-  #
-  #   IMW::Resource.new('/path/to/my_new_file', :mode => 'w')
-  #
-  # If the <tt>:skip_modules</tt> option is passed in then the
-  # resource will not extend itself with any modules and will
-  # essentially only retain the bare functionality of a URI.  This can
-  # be useful when subclassing IMW::Resource or dealing with a very
-  # strange kind of resource.
-  #
-  # Read the documentation for modules in IMW::Resources to learn more
-  # about the various behaviors an IMW::Resource can acquire.
-  #
-  # You can also instantiate an IMW::Resource using IMW.open, which
-  # accepts all the same arguments as IMW::Resource.new.
   class Resource
 
-    # The mode in which to access this resource.
-    attr_accessor :mode
-
-    # A copy of the options passed to this resource on initialization.
-    attr_accessor :resource_options
+    attr_accessor :mode, :resource_options
 
-    # Create a new resource representing +uri+.
-    #
-    # IMW will automatically extend the resulting IMW::Resource
-    # instance with modules appropriate for the given URI:
-    #
-    #   r = IMW::Resource.new("http://www.infochimps.com")
-    #   r.modules
-    #   => [IMW::Schemes::Remote::Base, IMW::Schemes::Remote::RemoteFile, IMW::Schemes::HTTP, IMW::Formats::Html]
-    #
-    # You can prevent this altogether by passing in
-    # <tt>:no_modules</tt>:
-    #
-    #   r = IMW::Resource.new("http://www.infochimps.com", :no_modules => true)
-    #   r.modules
-    #   => []
-    #
-    # And you can exert more fine-grained control with the
-    # <tt>:use_modules</tt> and <tt>:skip_modules</tt> options, see
-    # IMW::Resource.extend_instance! for details.
-    #
-    # @param [String, Addressable::URI] uri
-    # @param [Hash] options
-    # @option options [String] mode the mode to open the resource in (will be ignored when inapplicable)
-    # @return [IMW::Resource]
     def initialize uri, options={}
       self.uri              = uri
       self.resource_options = options
@@ -85,20 +15,14 @@ module IMW
     # Provides resources with a wrapped Addressable::URI object.
     include IMW::Utils::HasURI
 
-    # Provides resources with a summary, metadata, & schema.
-    include IMW::Metadata::HasSummary
-    
     # Gives IMW::Resource instances with the ability to dynamically
     # extend themselves with modules chosen from a set of handlers
     # stored by the IMW::Resource class.
     include IMW::Utils::DynamicallyExtendable
-    [IMW::Schemes::HANDLERS, IMW::CompressedFiles::HANDLERS, IMW::Archives::HANDLERS, IMW::Formats::HANDLERS].each do |handlers|
+    [IMW::Schemes::HANDLERS, IMW::Formats::HANDLERS].each do |handlers|
       register_handlers *handlers
     end
-    
-    # Raise an error unless this resource exists.
-    #
-    # @param [String] message an optional message to include
+
     def should_exist!(message=nil)
       raise IMW::Error.new([message, "No path defined for #{self.inspect} extended by #{modules.join(' ')}"].compact.join(', '))          unless respond_to?(:path)
       raise IMW::Error.new([message, "No exist? method defined for #{self.inspect} extended by #{modules.join(' ')}"].compact.join(', ')) unless respond_to?(:exist?)
@@ -106,52 +30,12 @@ module IMW
       self
     end
 
-    # Close this resource.
-    #
-    # Modules should hook into super() as they need to redefine this
-    # method.
     def close
     end
 
-    # Open a copy of this resource.
-    #
-    # This is useful when wanting to reset file handles.  Though -- be
-    # warned -- it does not close any file handles itself...
-    #
-    # @return [IMW::Resource] the new (old) resource
     def reopen
       IMW.open(uri.to_s)
     end
 
-    # If +method+ begins with the strings +is+, +on+, or +via+ and
-    # ends with a question mark then we interpret it as a question
-    # this resource doesn't know how to answer -- so we have it answer
-    # +false+.
-    #
-    # As an example, consider the following loop:
-    #
-    #   IMW.open('/tmp').all_contents.each do |obj|
-    #     if obj.is_archive?
-    #       # ... do something
-    #     end
-    #   end
-    #
-    # When +obj+ is initialized and it _isn't_ an archive, then it
-    # doesn't know about the <tt>is_archive?</tt> method -- but it
-    # should therefore answer false anyway.
-    #
-    # This lets a basic text file answer questions about whether it's
-    # an archive (or on S3, or accessed via some user-defined scheme,
-    # &c.) without needing to know anything about archives (or S3 or
-    # the user-defined scheme).
-    def method_missing method, *args
-      if args.empty? && method.to_s =~ /(is|on|via)_.*\?$/
-        # querying for a boolean response so answer false
-        return false
-      else
-        raise IMW::NoMethodError, "undefined method `#{method}' for #{self}, extended by #{modules.join(', ')}"
-      end
-    end
-
   end
 end