imw 0.2.18 → 0.3.0

data/lib/imw/metadata/schema.rb

@@ -1,17 +0,0 @@
-module IMW
-
-  class Metadata
-    
-    # Represents a schema for data.
-    #
-    # FIXME add methods that help couple nicely with Avro schemata.
-    class Schema < Hash
-
-      def initialize obj=nil
-        super()
-        merge!(obj) if obj.is_a?(Hash) || obj.is_a?(Schema)
-      end
-
-    end
-  end
-end

data/lib/imw/parsers.rb

@@ -1,8 +0,0 @@
-module IMW
-  module Parsers
-    autoload :LineParser,   'imw/parsers/line_parser'
-    autoload :RegexpParser, 'imw/parsers/regexp_parser'
-    autoload :HtmlParser,   'imw/parsers/html_parser'
-    autoload :Flat,         'imw/parsers/flat'
-  end
-end

data/lib/imw/parsers/flat.rb

@@ -1,44 +0,0 @@
-module IMW
-  module Parsers
-
-    class Flat
-
-      attr_accessor :io
-      attr_accessor :state
-      attr_accessor :accumulated
-      attr_accessor :current
-
-      def initialize io
-        self.io          = io
-        self.state       = nil
-        self.accumulated = []
-        self.current     = nil
-      end
-
-      def read_next!
-        self.current = io.readline.chomp
-      end
-
-      def parse!
-        while (! complete?)
-          read_next!
-          react_to_input!
-        end
-      end
-
-      def accumulate!
-        self.accumulated << current
-      end
-
-      def complete?
-        io.eof?
-      end
-
-      def react_to_input!
-        raise IMW::NotImplementedError.new("Override the `react_to_input!' method of the #{self.class} class")
-      end
-      
-    end
-  end
-end
-

data/lib/imw/parsers/html_parser.rb

@@ -1,387 +0,0 @@
-#
-# h2. lib/imw/parsers/html_parser.rb -- html parser
-#
-# == About
-#
-# h4. HTML Extractor
-#
-# * map repeating HTML elements to intermediate ruby data structure
-# * optimize all the common cases for expressive brevity
-# * output structure will come from HTML structure; map to desired output objects in transform stage.
-# * spec shouldn't be allowed to get too much more complicated than this; again, transform stage exists
-#
-# If this doesn't yield satisfaction you may enjoy
-# * http://blog.labnotes.org/2006/07/11/scraping-with-style-scrapi-toolkit-for-ruby/
-# * http://scrubyt.org/
-# Note of course that these have quite different goals.  For example, we don't
-# have any interest in "interactive" crawling, eg form submission, or at least
-# that goes elsewhere.
-#
-#
-# == Sample HTML (http://twitter.com):
-#
-#   <ul class="about vcard entry-author">
-#     <li         ><span class="label">Name</span>     <span class="fn" >MarsPhoenix       </span> </li>
-#     <li         ><span class="label">Location</span> <span class="adr">Mars, Solar System</span> </li>
-#     <li id="bio"><span class="label">Bio</span>      <span class="bio">I dig Mars!       </span> </li>
-#     <li         ><span class="label">Web</span>
-#        <a href="http://tinyurl.com/5wwaru" class="url" rel="me nofollow">http://tinyurl.co...</a></li>
-#   </ul>
-#
-# == Parser Spec:
-#   :hcard        => m_one('//ul.vcard.about',
-#     {
-#       :name     => 'li/span.fn',
-#       :location => 'li/span.adr',
-#       :url      => m_attr('li/a.url[@href]', 'href'),
-#       :bio      => 'li#bio/span.bio',
-#     }
-#   )
-#
-# == Example return:
-#   { :hcard => { :name => 'Mars Phoenix', :location => 'Mars, Solar System', :bio => 'I dig Mars!', :url => 'http://tinyurl.com/5wwaru' } }
-#
-# == Sample HTML (http://delicious.com):
-#   <ul id="bookmarklist" class="bookmarks NOTHUMB">
-#     <li class="post" id="item-...">
-#       <div class="bookmark NOTHUMB">
-#         <div class="dateGroup">         <span title="23 APR 08">23 APR 08</span>     </div>
-#         <div class="data">
-#           <h4>                          <a rel="nofollow" class="taggedlink" href="http://www.cs.biu.ac.il/~koppel/BlogCorpus.htm">Blog Authorship Corpus (Blogger.com 1994)</a>
-#                                         <a class="inlinesave" href="...">SAVE</a> </h4>
-#           <h5 class="savers-label">     PEOPLE</h5>
-#           <div class="savers savers2">  <a class="delNav" href="/url/7df6661946fca61863312644eb071953"><span class="delNavCount">26</span></a>  </div>
-#           <div class="description">     The Blog Authorship Corpus consists of the collected posts of 19,320 bloggers gathered from blogger.com in August 2004. The corpus incorporates a total of 681,288 posts and over 140 million words - or approximately 35 posts and 7250 words per person. </div>
-#         </div>
-#         <div class="meta"></div>
-#         <h5 class="tag-chain-label">TAGS</h5>
-#         <div class="tagdisplay">
-#           <ul class="tag-chain">
-#             <li class="tag-chain-item off first"><a class="tag-chain-item-link" rel="tag" href="/infochimps/blog"     ><span class="tag-chain-item-span">blog</span>    </a></li>
-#             <li class="tag-chain-item off">      <a class="tag-chain-item-link" rel="tag" href="/infochimps/corpus"   ><span class="tag-chain-item-span">corpus</span>  </a></li>
-#             <li class="tag-chain-item off">      <a class="tag-chain-item-link" rel="tag" href="/infochimps/analysis" ><span class="tag-chain-item-span">analysis</span></a></li>
-#             <li class="tag-chain-item off">      <a class="tag-chain-item-link" rel="tag" href="/infochimps/nlp"      ><span class="tag-chain-item-span">nlp</span>     </a></li>
-#             <li class="tag-chain-item on  last"> <a class="tag-chain-item-link" rel="tag" href="/infochimps/dataset"  ><span class="tag-chain-item-span">dataset</span> </a></li>
-#           </ul>
-#         </div>
-#         <div class="clr"></div>
-#       </div>
-#     </li>
-#   </ul>
-#
-# == Parser Specification:
-#   :bookmarks            => [ 'ul#bookmarklist/li.post/.bookmark',
-#     {
-#       :date                     => hash(    '.dateGroup/span',
-#          [:year, :month, :day]  => regexp(  '', /(\d{2}) ([A-Z]{3}) (\d{2})/),
-#          ),
-#       :title                    =>          '.data/h4/a.taggedlink',
-#       :url                      => attr(    '.data/h4/a.taggedlink', 'href'),
-#       :del_link_url             => href(    '.data/.savers/a.delNav),
-#       :num_savers               => to_i(    '.data/.savers//span.delNavCount'),
-#       :description              =>          '.data/.description',
-#       :tags                     =>         ['.tagdisplay//tag-chain-item-span']
-#     }
-#   ]
-#
-# == Example output:
-#   { :bookmarks => [
-#     { :date             => { :year => '08', :month => 'APR', :day => '23' },
-#       :title            => 'Blog Authorship Corpus (Blogger.com 1994)',
-#       :url              => 'http://www.cs.biu.ac.il/~koppel/BlogCorpus.htm',
-#       :del_link_url     => '/url/7df6661946fca61863312644eb071953',
-#       :num_savers       => 26,
-#       :description      => 'The Blog ... ',
-#       :tags             => ['blog', 'corpus', 'analysis', 'nlp', 'dataset'],
-#      }
-#    ]}
-#
-# == Implementation:
-#
-# Internally, we take the spec and turn it into a recursive structure of Matcher
-# objects.  These consume Hpricot Elements and return the appropriately extracted
-# object.
-#
-# Note that the /default/ is for a bare selector to match ONE element, and to not
-# complain if there are many.
-#
-# Missing elements are silently ignored -- for example if
-#   :foo => 'li.missing'
-# there will simply be no :foo element in the hash (as opposed to having hsh[:foo]
-# set to nil -- hsh.include?(foo) will be false)
-#
-#
-# == List of Matchers:
-#     { :field => /spec/, ... }           # hash          hash, each field taken from spec.
-#     [ "hpricot_path" ]                  # 1-el array    array: for each element matching
-#                                                         hpricot_path, the inner_html
-#     [ "hpricot_path", /spec/ ]          # 2-el array    array: for each element matching
-#                                                         hpricot_path, pass to spec
-#     "hpricot_path"                      # string        same as one("hpricot_path")
-#     one("hpricot_path")                 # one           first match to hpricot_path
-#     one("hpricot_path", /spec/)         # one           applies spec to first match to hpricot_path
-#     (these all match on one path:)
-#     regexp("hpricot_path", /RE/)        # regexp        capture groups from matching RE against
-#                                                         inner_html of first match to hpricot_path
-#     attr("hpricot_path", 'attr_name')   # attr
-#     href("hpricot_path")                # href          shorthand for attr(foo, 'href')
-#     no_html                             #               strip tags from contents
-#     html_encoded                        #               html encode contents
-#     to_i, to_f, etc                     # convert
-#     lambda{|doc| ... }                  # proc          calls proc on current doc
-#
-# == Complicated HCard example:
-#     :hcards                     =>      [ '//ul.users/li.vcard',
-#       {
-#         :name                   =>      '.fn',
-#         :address                =>      one('.adr',
-#           :street               =>      '.street',
-#           :city                 =>      '.city',
-#           :zip                  =>      '.postal'
-#         )
-#         :tel                    =>      [ 'span.tel',
-#           {
-#             :type               =>      'span.type',
-#             [:cc, :area, :num]  =>      hp.regexp('span.value', /+(\d+).(\d{3})-(\d{3}-\d{4})/),
-#           }
-#         ]
-#         :tags                   =>      [ '.tag' ],
-#       }
-#     ]
-#
-# == Resulting Parser
-#     MatchHash({:hcards  =>      MatchArray('//ul.users/li.hcard',
-#       MatchHash({
-#         :name                   =>      MatchFirst('.fn'),
-#         :address                =>      MatchFirst('.adr',
-#           MatchHash({
-#             :street             =>      MatchFirst('.street'),
-#             :city               =>      MatchFirst('.locality),
-#             :state              =>      MatchFirst('.region),
-#             :zip                =>      MatchFirst('.postal'),
-#           }))
-#         :tel                    =>      MatchArray('span.tel',
-#           MatchHash({
-#             :type               =>      MatchFirst('span.type'),
-#             [:cc, :area, :num]  =>      RegexpMatcher('span.value', /+(\d+).(\d{3})-(\d{3}-\d{4})/),
-#           })
-#         )
-#         :tags                   =>      MatchArray('.tag'),
-#       })
-#     )
-#
-# == Example output
-#     [
-#       {:tel     => [ {:type => 'home', :cc => '49', :area => '305', :num => '555-1212'},
-#                      {:type => 'work', :cc => '49', :area => '305', :num => '555-6969'}, ],
-#        :name    => "Bob Dobbs, Jr.",
-#        :tags    => ["church"] },
-#       {:tel     => [ {:type => 'fax',  :cc => '49', :area => '305', :num => '867-5309'}, ],
-#        :name    => "Jenny",
-#        :address => { :street => "53 Evergreen Terr.", :city => "Springfield" },
-#        :tags    => ["bathroom", "wall"] },
-#     ]
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-#
-# puts "#{File.basename(__FILE__)}: Something clever" # at bottom
-
-require 'imw/parsers/html_parser/matchers'
-
-module IMW
-  module Parsers
-    class HtmlParser
-
-      include IMW::Parsers::HtmlMatchers
-
-      attr_accessor :parse_tree
-
-      #
-      # Parse Tree
-      #
-      def initialize arg_spec=nil
-        spec = arg_spec || self.class.parser_spec
-        self.parse_tree = IMW::Parsers::HtmlMatchers.build_parse_tree(spec)
-      end
-
-      #
-      # See IMW::HtmlParser for syntax
-      #
-      #
-      def self.parser_spec
-        raise "Override this to create your own parser spec"
-      end
-
-      #
-      # Walk
-      #
-      def parse doc
-        self.parse_tree.match(doc)
-      end
-
-      # one("hpricot_path")                 first match to hpricot_path
-      # one("hpricot_path", /spec/)         applies spec to first match to hpricot_path
-      #
-      def self.one selector, matcher
-        MatchFirstElement.new(selector, IMW::Parsers::HtmlMatchers.build_parse_tree(matcher))
-      end
-      # match the +attr+ attribute of the first element given by +selector+
-      def self.attr selector, attr, matcher=nil
-        MatchAttribute.new(selector, attr, IMW::Parsers::HtmlMatchers.build_parse_tree(matcher))
-      end
-      # shorthand for +attr(foo, 'href')+
-      def self.href selector, matcher=nil
-        self.attr(selector, 'href', matcher)
-      end
-      # shorthand for +attr(foo, 'src')+
-      def self.src selector, matcher=nil
-        self.attr(selector, 'src', matcher)
-      end
-
-      def self.proc selector, proc, matcher=nil
-        MatchProc.new(selector, proc, IMW::Parsers::HtmlMatchers.build_parse_tree(matcher))
-      end
-
-      # strip ","s (!! thus disrespecting locale !!!)
-      # and convert to int
-      def self.to_num selector, matcher=nil
-        proc selector, lambda{|num| num.to_s.gsub(/,/,'').to_i if num }, matcher
-      end
-      def self.to_json selector, matcher=nil
-        proc selector, lambda{|v| v.to_json if v }, matcher
-      end
-
-      def self.strip selector, matcher=nil
-        proc selector, lambda{|v| v.strip }, matcher
-      end
-
-      def self.re_group selector, re
-        MatchRegexp.new(selector, re)
-      end
-      def self.re selector, re
-        MatchRegexp.new(selector, re, nil, :capture => 1)
-      end
-      def self.re_all selector, re, matcher=nil
-        MatchRegexpRepeatedly.new(selector, re)
-      end
-
-      # def self.plain_text selector, matcher=nil
-      #   proc selector, lambda{|el| el.inner_text if el }, matcher
-      # end
-
-      # attr_accessor :mapping
-      #
-      # #
-      # # Feed me a hash and I'll semantify HTML
-      # #
-      # # The hash should magically adhere to the too-complicated,
-      # # ever evolving goatrope that works for the below
-      # #
-      # #
-      # def initialize mapping
-      #   self.mapping = mapping
-      # end
-      #
-      # #
-      # # take a document subtree,
-      # # and a mapping of hpricot paths to that subtree's data mapping
-      # # recursively extract that datamapping
-      # #
-      # def extract_tree  hdoc, content, sub_mapping
-      #   data = { }
-      #   sub_mapping.each do |selector, target|
-      #     data[selector] = []
-      #     sub_contents = content/selector
-      #     sub_contents.each do |sub_content|
-      #       sub_data = {}
-      #       extract_node hdoc, sub_content, sub_data, selector, target
-      #       data[selector] << sub_data
-      #     end
-      #   end
-      #   data
-      #   # end
-      #   #   if selector.is_a?(String)
-      #   #     conts = (content)
-      #   #   else
-      #   #     conts = [content]
-      #   #   end
-      #   #   conts[0..0].each do |content|
-      #   #     extract_node hdoc, content, data, selector, target
-      #   #   end
-      #   # end
-      #   data
-      # end
-      #
-      # #
-      # # insert the extracted element into the data mapping
-      # #
-      # def extract_node hdoc, content, data, selector, target
-      #   classification = classify_node(selector, target)
-      #   result = \
-      #   case classification
-      #   when :subtree
-      #     target.each do |sub_selector, sub_target|
-      #       extract_node hdoc, content, data, sub_selector, sub_target
-      #     end
-      #
-      #   when :sub_attribute
-      #     k, v = selector.to_a[0]
-      #     subcontent = (k[0..0] == '/') ? (hdoc.at(k)) : (content.at(k))
-      #     val  = subcontent.attributes[v.to_s] if subcontent
-      #     data[target] = val unless val.blank?
-      #
-      #   when :attribute then
-      #     val = content.attributes[selector.to_s]
-      #     data[target] = val unless val.blank?
-      #
-      #   when :flatten_list
-      #     subcontents = (selector[0..0] == '/') ? (hdoc/selector) : (content/selector)
-      #     data[target.first] = subcontents.map{|subcontent| subcontent.inner_html }
-      #
-      #   when :inner_html
-      #     subcontent = (selector[0..0] == '/') ? (hdoc.at(selector)) : (content.at(selector))
-      #     data[target] = subcontent.inner_html.strip if subcontent
-      #
-      #   else
-      #     raise "classify_node shouldn't ever return #{classification}"
-      #   end
-      #   # puts "%-19s %-19s %-31s %s" % [target.inspect[0..18], classification.inspect[0..18], selector.inspect[0..30], result.inspect[0..90]] if (classification == :sub_attribute)
-      #   # puts '' if classification == :subtree
-      # end
-      #
-      # def classify_node selector, target
-      #   case
-      #   when target.is_a?(Hash)                             then :subtree
-      #   when selector.is_a?(Hash) && (selector.length == 1) then
-      #     k, v = selector.to_a[0]
-      #     case v
-      #     when Symbol then :sub_attribute
-      #     end
-      #   when selector.is_a?(Symbol)                         then :attribute
-      #   when selector.is_a?(String) && target.is_a?(Array)  then :flatten_list
-      #   when selector.is_a?(String) && target.is_a?(Symbol) then :inner_html
-      #   else
-      #     raise "Can't classify mapping: " + [selector, target].join(" - ")
-      #   end
-      # end
-      #
-      # # use #mapping to parse file
-      # def parse link
-      #   begin       hdoc = Hpricot(link.contents)
-      #   rescue;     warn "can't hpricot #{link.to_s}" ; return false;  end
-      #   raw_taggings = extract_tree hdoc, hdoc, self.mapping
-      # end
-      #
-      # # use #mapping to parse file
-      # def parse_file filename
-      #   begin       hdoc = Hpricot(File.open(filename))
-      #   rescue;     warn "can't hpricot #{filename}" ; return false;  end
-      #   raw_taggings = extract_tree hdoc, hdoc, self.mapping
-      # end
-    end
-  end
-end
-