gtl-parsley-ruby 0.5.0

data/ext/parsley/ylwrap

@@ -0,0 +1,222 @@
+#! /bin/sh
+# ylwrap - wrapper for lex/yacc invocations.
+
+scriptversion=2009-04-28.21; # UTC
+
+# Copyright (C) 1996, 1997, 1998, 1999, 2001, 2002, 2003, 2004, 2005,
+# 2007, 2009 Free Software Foundation, Inc.
+#
+# Written by Tom Tromey <tromey@cygnus.com>.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# This file is maintained in Automake, please report
+# bugs to <bug-automake@gnu.org> or send patches to
+# <automake-patches@gnu.org>.
+
+case "$1" in
+  '')
+    echo "$0: No files given.  Try \`$0 --help' for more information." 1>&2
+    exit 1
+    ;;
+  --basedir)
+    basedir=$2
+    shift 2
+    ;;
+  -h|--h*)
+    cat <<\EOF
+Usage: ylwrap [--help|--version] INPUT [OUTPUT DESIRED]... -- PROGRAM [ARGS]...
+
+Wrapper for lex/yacc invocations, renaming files as desired.
+
+  INPUT is the input file
+  OUTPUT is one file PROG generates
+  DESIRED is the file we actually want instead of OUTPUT
+  PROGRAM is program to run
+  ARGS are passed to PROG
+
+Any number of OUTPUT,DESIRED pairs may be used.
+
+Report bugs to <bug-automake@gnu.org>.
+EOF
+    exit $?
+    ;;
+  -v|--v*)
+    echo "ylwrap $scriptversion"
+    exit $?
+    ;;
+esac
+
+
+# The input.
+input="$1"
+shift
+case "$input" in
+  [\\/]* | ?:[\\/]*)
+    # Absolute path; do nothing.
+    ;;
+  *)
+    # Relative path.  Make it absolute.
+    input="`pwd`/$input"
+    ;;
+esac
+
+pairlist=
+while test "$#" -ne 0; do
+  if test "$1" = "--"; then
+    shift
+    break
+  fi
+  pairlist="$pairlist $1"
+  shift
+done
+
+# The program to run.
+prog="$1"
+shift
+# Make any relative path in $prog absolute.
+case "$prog" in
+  [\\/]* | ?:[\\/]*) ;;
+  *[\\/]*) prog="`pwd`/$prog" ;;
+esac
+
+# FIXME: add hostname here for parallel makes that run commands on
+# other machines.  But that might take us over the 14-char limit.
+dirname=ylwrap$$
+trap "cd '`pwd`'; rm -rf $dirname > /dev/null 2>&1" 1 2 3 15
+mkdir $dirname || exit 1
+
+cd $dirname
+
+case $# in
+  0) "$prog" "$input" ;;
+  *) "$prog" "$@" "$input" ;;
+esac
+ret=$?
+
+if test $ret -eq 0; then
+  set X $pairlist
+  shift
+  first=yes
+  # Since DOS filename conventions don't allow two dots,
+  # the DOS version of Bison writes out y_tab.c instead of y.tab.c
+  # and y_tab.h instead of y.tab.h. Test to see if this is the case.
+  y_tab_nodot="no"
+  if test -f y_tab.c || test -f y_tab.h; then
+    y_tab_nodot="yes"
+  fi
+
+  # The directory holding the input.
+  input_dir=`echo "$input" | sed -e 's,\([\\/]\)[^\\/]*$,\1,'`
+  # Quote $INPUT_DIR so we can use it in a regexp.
+  # FIXME: really we should care about more than `.' and `\'.
+  input_rx=`echo "$input_dir" | sed 's,\\\\,\\\\\\\\,g;s,\\.,\\\\.,g'`
+
+  while test "$#" -ne 0; do
+    from="$1"
+    # Handle y_tab.c and y_tab.h output by DOS
+    if test $y_tab_nodot = "yes"; then
+      if test $from = "y.tab.c"; then
+    	from="y_tab.c"
+      else
+    	if test $from = "y.tab.h"; then
+    	  from="y_tab.h"
+    	fi
+      fi
+    fi
+    if test -f "$from"; then
+      # If $2 is an absolute path name, then just use that,
+      # otherwise prepend `../'.
+      case "$2" in
+    	[\\/]* | ?:[\\/]*) target="$2";;
+    	*) target="../$2";;
+      esac
+
+      # We do not want to overwrite a header file if it hasn't
+      # changed.  This avoid useless recompilations.  However the
+      # parser itself (the first file) should always be updated,
+      # because it is the destination of the .y.c rule in the
+      # Makefile.  Divert the output of all other files to a temporary
+      # file so we can compare them to existing versions.
+      if test $first = no; then
+	realtarget="$target"
+	target="tmp-`echo $target | sed s/.*[\\/]//g`"
+      fi
+      # Edit out `#line' or `#' directives.
+      #
+      # We don't want the resulting debug information to point at
+      # an absolute srcdir; it is better for it to just mention the
+      # .y file with no path.
+      #
+      # We want to use the real output file name, not yy.lex.c for
+      # instance.
+      #
+      # We want the include guards to be adjusted too.
+      FROM=`echo "$from" | sed \
+            -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'\
+            -e 's/[^ABCDEFGHIJKLMNOPQRSTUVWXYZ]/_/g'`
+      TARGET=`echo "$2" | sed \
+            -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'\
+            -e 's/[^ABCDEFGHIJKLMNOPQRSTUVWXYZ]/_/g'`
+
+      sed -e "/^#/!b" -e "s,$input_rx,," -e "s,$from,$2," \
+          -e "s,$FROM,$TARGET," "$from" >"$target" || ret=$?
+
+      # Check whether header files must be updated.
+      if test $first = no; then
+	if test -f "$realtarget" && cmp -s "$realtarget" "$target"; then
+	  echo "$2" is unchanged
+	  rm -f "$target"
+	else
+          echo updating "$2"
+          mv -f "$target" "$realtarget"
+        fi
+      fi
+    else
+      # A missing file is only an error for the first file.  This
+      # is a blatant hack to let us support using "yacc -d".  If -d
+      # is not specified, we don't want an error when the header
+      # file is "missing".
+      if test $first = yes; then
+        ret=1
+      fi
+    fi
+    shift
+    shift
+    first=no
+  done
+else
+  ret=$?
+fi
+
+# Remove the directory.
+cd ..
+rm -rf $dirname
+
+exit $ret
+
+# Local Variables:
+# mode: shell-script
+# sh-indentation: 2
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC"
+# time-stamp-end: "; # UTC"
+# End:

data/lib/parsley.rb

@@ -0,0 +1,84 @@
+require File.expand_path(File.dirname(__FILE__) + "/../ext/cparsley")
+require "rubygems"
+require "json"
+require "thread"
+
+class Parsley
+  
+  def self.user_agent=(agent)
+    @user_agent = agent
+    CParsley.set_user_agent(agent.to_s)
+  end
+  
+  def self.user_agent
+    @user_agent
+  end
+  
+  def initialize(parsley, incl = "")
+    if(parsley.is_a?(Hash))
+      parsley = recursive_stringify(parsley).to_json 
+    end
+    @@mutex ||= Mutex.new
+    @@mutex.synchronize do
+      @parsley = CParsley.new(parsley, incl)
+    end
+  end
+  
+  # Valid options:
+  #
+  # Requires one of:
+  # :file -- the input file path or url
+  # :string -- the input string
+  #
+  # And optionally (default is the first listed value):
+  # :input => [:html, :xml]
+  # :output => [:ruby, :json, :xml]
+  # :prune => [true, false]
+  # :sgwrap => [false, true]
+  # :collate => [true, false]
+  # :base => "http://some/base/href"
+  # :allow_net => [true, false]
+  # :allow_local => [true, false]
+  def parse(options = {})
+    options[:file] || options[:string] || (raise ParsleyError.new("must specify what to parse"))
+    
+    options[:sgwrap] = !!options[:sgwrap]
+    options[:is_file] = !!options[:file]
+    options[:has_base] = !!options[:base]
+    
+    options[:base] = options[:base].to_s
+    options[:file] = options[:file].to_s
+    options[:string] = options[:string].to_s
+    
+    options[:input]  ||= :html
+    options[:output] ||= :ruby
+    
+    options[:collate] = true unless options.has_key?(:collate)
+    options[:prune] = true unless options.has_key?(:prune)
+    options[:allow_net] = true unless options.has_key?(:allow_net)
+    options[:allow_local] = true unless options.has_key?(:allow_local)
+    
+    options[:collate] = !!options[:collate]
+    options[:prune] = !!options[:prune]
+    options[:allow_net] = !!options[:allow_net]
+    options[:allow_local] = !!options[:allow_local]
+    
+    @parsley.parse(options)
+  end
+  private
+  
+  def recursive_stringify(obj)
+    case obj
+    when Hash
+      obj.inject({}) do |memo, (k, v)|
+        memo[k.to_s] = recursive_stringify(v)
+        memo
+      end
+    when Array
+      obj.map{|e| recursive_stringify(e) }
+    else
+      obj.to_s
+    end
+  end
+  
+end

data/test/test_parsley.rb

@@ -0,0 +1,120 @@
+#encoding: UTF-8
+require "test/unit"
+require File.expand_path(File.dirname(__FILE__) + "/../lib/parsley")
+
+class TestParsley < Test::Unit::TestCase
+  def setup
+    @page = File.expand_path(File.dirname(__FILE__) + "/yelp.html")
+    @home = File.expand_path(File.dirname(__FILE__) + "/yelp-home.html")
+    @let  = File.expand_path(File.dirname(__FILE__) + "/yelp-home.let")
+  end
+  
+  def test_segfault_regression
+    simple_html = <<-HTML
+      <html>
+      <body>
+        <h1 class="iCIMS_Header_JobTitle">CEO</h1>
+        <h2 class="temperature">28ºF</h2>
+      </body>
+      </html>
+    HTML
+    
+    struct = {
+      'jobs' => [{
+        'title'         => ".iCIMS_Header_JobTitle",
+        'temperature'   => ".temperature",
+        'description?'  => "blah",
+        'location?'     => "blah",
+        'experience?'   => "blah",
+        'education?'    => "blah"
+      }]
+    }
+    parselet = Parsley.new(struct)
+    result = parselet.parse(:string => simple_html)
+    assert_equal "CEO", result['jobs'].first['title']
+    assert_equal "28ºF", result['jobs'].first['temperature']
+    assert result['jobs'].first['description'].nil?
+  end
+  # 
+  # def test_yelp
+  #   @parsley = Parsley.new(File.read(@let))
+  #   out = @parsley.parse(:file => @home)
+  #   assert_equal "/c/sf/shopping", out["categories"][0]["href"]
+  # end
+  # 
+  # def test_parsley_should_raise_if_value_syntax_error
+  #   assert_raises(ParsleyError) do
+  #     Parsley.new({"foo" => nil})
+  #   end
+  #   
+  #   assert_raises(ParsleyError) do
+  #     Parsley.new({"foo" => ""})
+  #   end
+  #   
+  #   assert_raises(ParsleyError) do
+  #     Parsley.new({"foo" => "<<<<<<<<<<<"})
+  #   end
+  # end
+  # 
+  # def test_yelp_xml
+  #   @parsley = Parsley.new(File.read(@let))
+  #   out = @parsley.parse(:file => @home, :output => :xml)
+  # end
+  # 
+  # def test_broken
+  #   @parsley = Parsley.new("hi" => "no-ns:match(h1)")
+  #   assert_raises(ParsleyError) {
+  #     @parsley.parse(:file => @page)
+  #   }
+  # end
+  # 
+  # def test_simple
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   assert_equal({"hi" => "Nick's Crispy Tacos"}, @parsley.parse(:file => @page))
+  # end
+  # 
+  # def test_simple_string
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   assert_equal({"hi" => "Nick's Crispy Tacos"}, @parsley.parse(:string => "<html><body><h1>Nick's Crispy Tacos</h1></body></html>"))
+  # end  
+  # 
+  # def test_xml
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   xml = "<?xml version=\"1.0\"?>\n<parsley:root xmlns:parsley=\"http://parselets.com/json\"><hi position=\"63\">Nick's Crispy Tacos</hi></parsley:root>\n"
+  #   assert_equal(xml, @parsley.parse(:file => @page, :output => :xml))
+  # end
+  # 
+  # def test_sgwrap
+  #   @parsley = Parsley.new("hi" => "p sg_wrap")
+  #   html = "<p><b>hi</b>world</p>"
+  #   assert_equal({"hi" => "world"}, @parsley.parse(:string => html, :sgwrap => true))
+  # end
+  # 
+  # def test_sgwrap_off
+  #   @parsley = Parsley.new("hi" => "p sg_wrap")
+  #   html = "<p><b>hi</b>world</p>"
+  #   assert_raises(ParsleyError) do
+  #     @parsley.parse(:string => html, :sgwrap => false)
+  #   end
+  # end
+  # 
+  # 
+  # def test_json
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   assert_equal('{ "hi": "Nick\'s Crispy Tacos" }', @parsley.parse(:file => @page, :output => :json))
+  # end
+  # 
+  # def test_rescuable_file_error
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   @nonexistant_file = File.dirname(__FILE__) + "/../fixtures/yelp.html"
+  #   assert_equal({"hi" => "Nick's Crispy Tacos"}, @parsley.parse(:file => @nonexistant_file)) rescue nil
+  # end
+  # 
+  # def test_array_string
+  #   @parsley = Parsley.new({"foo" => ["li"]})
+  #   out = @parsley.parse(:file => @page)
+  #   assert_kind_of Hash, out
+  #   assert_kind_of Array, out["foo"], out.inspect
+  #   assert out["foo"].length > 1
+  # end
+end

data/test/yelp-benchmark.rb

@@ -0,0 +1,53 @@
+require "rubygems"
+require "nokogiri"
+require "hpricot"
+require "parsley"
+require "benchmark"
+require "pp"
+
+YELP_HTML = File.dirname(__FILE__) + "/yelp.html"
+
+def noko
+  parse Nokogiri.Hpricot(File.open(YELP_HTML))
+end
+
+def hpri
+  parse Hpricot(File.open(YELP_HTML))  
+end
+
+def parse(doc)
+  out = {}
+  out["name"] = (doc / "h1").first.inner_text
+  out["phone"] = (doc / "#bizPhone").first.inner_text
+  out["address"] = (doc / "address").first.inner_text
+  out["reviews"] = (doc / ".nonfavoriteReview").map do |node|
+    review = {}
+    review["date"] = (node / ".ieSucks .smaller").first.inner_text
+    review["user_name"] = (node / ".reviewer_info a").first.inner_text
+    review["comment"] = (node / ".review_comment").first.inner_text
+    review
+  end
+end
+
+def pars
+  parslet = Parsley.new({
+    "name" => "h1",
+    "phone" => "#bizPhone",
+    "address" => "address",
+    "reviews(.nonfavoriteReview)" => [
+      {
+        "date" => ".ieSucks .smaller",
+        "user_name" => ".reviewer_info a",
+        "comment" => ".review_comment"
+      }
+    ]
+  })
+  pp parslet.parse(:file => YELP_HTML)
+end
+
+Benchmark.bm do |x|
+  x.report("nokogiri: ")  { 3.times { noko } }
+  x.report("hpricot: ")   { 3.times { hpri } }
+  x.report("parsley: ") { 3.times { pars } }
+end
+