parsley-ruby 0.0.0 → 0.4.1

data/.gitignore

@@ -0,0 +1,26 @@
+.libs/
+*.o
+*.lo
+dexterc
+dexter
+parsleyc
+parsley
+.deps/
+Makefile
+y.tab.c
+autom4te.cache/
+autoscan.log
+config.log
+configure.scan
+parser.c
+scanner.c
+libparsley.la
+parser.h
+test.log
+parsley*.gem
+ext/cparsley.bundle
+ext/cparsley.so
+ext/Makefile
+ext/conftest.dSYM/
+work
+ext/mkmf.log

data/README

@@ -0,0 +1,32 @@
+ABOUT
+
+Ruby bindings for Parsley.
+
+INSTALLATION
+
+= Get Parsley and Dependancies = 
+
+Download Parsley from http://github.com/fizx/parsley/tree/master following the installation directions located at http://github.com/fizx/parsley/blob/master/INSTALL
+
+= Install parsley-ruby =
+
+From source:
+  sudo rake install
+
+From GitHub: DEPRECATED!
+
+From GemCutter
+
+  Run the following if you haven't already:
+  gem sources -a http://gemcutter.org
+  Install the gem:
+  sudo gem install parsley-ruby
+
+PARSLETS.COM INTEGRATION
+
+We also recommend installing the free online_parselets rubygem in order to use other people's parselets and to share your own:
+  Run the following if you haven't already:
+  gem sources -a http://gems.github.com
+  Install the gem:
+  sudo gem install iterationlabs-online_parslets
+

data/Rakefile

@@ -0,0 +1,56 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "parsley-ruby"
+    gem.summary = "Ruby binding for parsley"
+    gem.description = "XML/HTML Parser"
+    gem.email = "kyle@kylemaxwell.com"
+    gem.homepage = "http://github.com/fizx/parsley-ruby"
+    gem.authors = ["Kyle Maxwell"]
+    gem.add_dependency("json", ["> 0.0.0"])
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "robots #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.4.1

data/ext/cparsley.c

@@ -0,0 +1,140 @@
+#include "ruby.h"
+#include <stdio.h>
+#include <libxslt/xslt.h>
+#include <libexslt/exslt.h>
+#include <libxslt/xsltInternals.h>
+#include <libxslt/transform.h>
+#include <libxml/parser.h>
+#include <libxml/HTMLparser.h>
+#include <libxml/HTMLtree.h>
+#include <libxml/xmlwriter.h>
+#include <parsley.h>
+#include <json/json.h>
+#include <xml2json.h>
+
+VALUE _new(VALUE, VALUE, VALUE);
+VALUE _parse(VALUE, VALUE);
+VALUE _rb_set_user_agent(VALUE self, VALUE agent);
+VALUE c_parsley_err;
+VALUE c_parsley;
+
+void Init_cparsley()
+{
+	c_parsley = rb_define_class("CParsley", rb_cObject);
+	c_parsley_err = rb_define_class("ParsleyError", rb_eRuntimeError);
+	rb_define_singleton_method(c_parsley, "new", _new, 2);
+	rb_define_singleton_method(c_parsley, "set_user_agent", _rb_set_user_agent, 1);
+	rb_define_method(c_parsley, "parse", _parse, 1);
+}
+
+VALUE 
+_new(VALUE self, VALUE parsley, VALUE incl){
+	parsleyPtr ptr = parsley_compile(STR2CSTR(parsley), STR2CSTR(incl));
+	if(ptr->error != NULL) {
+	  rb_raise(c_parsley_err, ptr->error);
+    parsley_free(ptr);
+    return Qnil;
+	}
+	
+ 	return Data_Wrap_Struct(c_parsley, 0, parsley_free, ptr);
+}
+
+VALUE 
+_rb_set_user_agent(VALUE self, VALUE agent) {
+  parsley_set_user_agent(STR2CSTR(agent));
+  return Qtrue;
+}
+
+
+static VALUE 
+rubify_recurse(xmlNodePtr xml) {
+  if(xml == NULL) return NULL;
+  xmlNodePtr child;
+  VALUE obj = Qnil;
+
+  switch(xml->type) {
+    case XML_ELEMENT_NODE:
+      child = xml->children;
+      if(xml->ns == NULL) {
+        child = xml;
+        obj = rb_hash_new();
+        while(child != NULL) {
+          rb_hash_aset(obj, rb_str_new2(child->name), rubify_recurse(child->children));
+          child = child->next;
+        }
+      } else if(!strcmp(xml->ns->prefix, "parsley")) {
+        if(!strcmp(xml->name, "groups")) {
+          obj = rb_ary_new();
+          while(child != NULL) {
+            rb_ary_push(obj, rubify_recurse(child->children));
+            child = child->next;
+          }          
+        } else if(!strcmp(xml->name, "group")) {
+          // Implicitly handled by parsley:groups handler
+        }
+      }
+      break;
+    case XML_TEXT_NODE:
+      obj = rb_str_new2(xml->content);
+      break;
+  }
+  // inspect(obj);
+  return obj;
+}
+
+static VALUE 
+_parse_doc(parsedParsleyPtr ptr, VALUE type) {
+	if(ptr->error != NULL || ptr->xml == NULL) {
+    if(ptr->error == NULL) ptr->error = strdup("Unknown parsley error");
+		rb_raise(c_parsley_err, ptr->error);
+    parsed_parsley_free(ptr);
+		return Qnil;
+	}
+	
+	VALUE output;
+	if(type == ID2SYM(rb_intern("json"))) {
+		struct json_object *json = xml2json(ptr->xml->children->children);
+		char* str = json_object_to_json_string(json);
+		output = rb_str_new2(str);
+		json_object_put(json);
+	} else if(type == ID2SYM(rb_intern("xml"))) {
+		xmlChar* str;
+		int size;
+		xmlDocDumpMemory(ptr->xml, &str, &size);
+		output = rb_str_new(str, size);
+	} else {
+ 		output = rubify_recurse(ptr->xml->children->children);
+		if((void*)output == NULL) output = Qnil; 
+	}
+	
+  parsed_parsley_free(ptr);
+  
+	return output;
+}
+
+#define OPT(A) rb_hash_aref(options, ID2SYM(rb_intern(A)))
+#define OPT_BOOL(A) (OPT(A) != Qnil && OPT(A) != Qfalse)
+#define OPT_MATCH(A, B) (rb_hash_aref(options, ID2SYM(rb_intern(A))) == ID2SYM(rb_intern(B)))
+
+VALUE _parse(VALUE self, VALUE options){
+	parsleyPtr parsley;
+	Data_Get_Struct(self, parsleyPtr, parsley);
+  int flags = 0;
+  char *base = NULL;
+  if(OPT_MATCH("input", "html"))    flags |= PARSLEY_OPTIONS_HTML;
+  if(OPT_BOOL("prune"))             flags |= PARSLEY_OPTIONS_PRUNE;
+  if(OPT_BOOL("collate"))           flags |= PARSLEY_OPTIONS_COLLATE;
+  if(OPT_BOOL("allow_net"))         flags |= PARSLEY_OPTIONS_ALLOW_NET;
+  if(OPT_BOOL("allow_local"))       flags |= PARSLEY_OPTIONS_ALLOW_LOCAL;
+  if(OPT_BOOL("sgwrap"))            flags |= PARSLEY_OPTIONS_SGWRAP;
+  if(OPT_BOOL("has_base"))          base = STR2CSTR(OPT("base"));
+  
+  // printf("prune: %d\nallow_net: %d\nallow_local: %d\nhas_base: %d\nflags: %d\n", OPT_BOOL("prune"), OPT_BOOL("allow_net"), OPT_BOOL("allow_local"), OPT_BOOL("has_base"), flags);
+  
+  if(OPT_BOOL("is_file")) {
+    return _parse_doc(parsley_parse_file(parsley, STR2CSTR(OPT("file")), flags), OPT("output"));
+  } else {
+    char * str = STR2CSTR(OPT("string"));
+    return _parse_doc(parsley_parse_string(parsley, str, strlen(str), base, flags), OPT("output"));
+  }
+}

data/ext/extconf.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+ENV["ARCHFLAGS"] = "-arch #{`uname -p` =~ /powerpc/ ? 'ppc' : 'i386'}"
+
+require 'mkmf'
+
+ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+LIBDIR = Config::CONFIG['libdir']
+INCLUDEDIR = Config::CONFIG['includedir']
+
+$CFLAGS << " #{ENV["CFLAGS"]}"
+if Config::CONFIG['target_os'] == 'mingw32'
+  $CFLAGS << " -DXP_WIN -DXP_WIN32"
+else
+  $CFLAGS << " -g -DXP_UNIX"
+end
+
+$CFLAGS << " -O3 -Wall -Wextra -Wcast-qual -Wwrite-strings -Wconversion -Wmissing-noreturn -Winline"
+
+if Config::CONFIG['target_os'] == 'mingw32'
+  find_library('xml2', 'xmlParseDoc',
+               File.join(ROOT, 'cross', 'libxml2-2.7.2.win32', 'bin'))
+  find_library('xslt', 'xsltParseStylesheetDoc',
+               File.join(ROOT, 'cross', 'libxslt-1.1.24.win32', 'bin'))
+else
+  find_library('xml2', 'xmlParseDoc', LIBDIR)
+  find_library('xslt', 'xsltParseStylesheetDoc', LIBDIR)
+end
+
+
+if Config::CONFIG['target_os'] == 'mingw32'
+  header = File.join(ROOT, 'cross', 'libxml2-2.7.2.win32', 'include')
+  unless find_header('libxml/xmlversion.h', header)
+    abort "need libxml"
+  end
+
+  header = File.join(ROOT, 'cross', 'libxslt-1.1.24.win32', 'include')
+  unless find_header('libxslt/libxslt.h', header)
+    abort "need libxslt"
+  end
+
+  header = File.join(ROOT, 'cross', 'iconv-1.9.2.win32', 'include')
+  unless find_header('iconv.h', header)
+    abort "need iconv"
+  end
+else
+  unless find_header('libxml/xmlversion.h',
+                     File.join(INCLUDEDIR, "libxml2"), '/usr/include/libxml2'
+                    )
+    abort "need libxml"
+  end
+  unless find_header('libxslt/xslt.h', INCLUDEDIR, '/usr/include')
+    abort "need libxslt"
+  end
+
+  version = try_constant('LIBXML_VERSION', 'libxml/xmlversion.h')
+end
+
+myincl = %w[/usr/local/include /opt/local/include /usr/include]
+mylib = %w[/usr/local/lib /opt/local/lib /usr/lib]
+
+find_header('ruby.h', INCLUDEDIR, *myincl) or abort "need ruby.h"
+
+find_header('json/json.h', INCLUDEDIR, *myincl) or abort "need json/json.h"
+find_library('json', 'json_object_new_string', LIBDIR, *mylib) or abort "need libjson"
+
+find_header('parsley.h', INCLUDEDIR, *myincl) or abort "need parsley.h"
+find_library('parsley', 'parsley_compile', LIBDIR, *mylib) or abort "need libparsley"
+
+create_makefile('cparsley')

data/lib/parsley.rb

@@ -0,0 +1,84 @@
+require File.dirname(__FILE__) + "/../ext/cparsley"
+require "rubygems"
+require "json"
+require "thread"
+
+class Parsley
+  
+  def self.user_agent=(agent)
+    @user_agent = agent
+    CParsley.set_user_agent(agent.to_s)
+  end
+  
+  def self.user_agent
+    @user_agent
+  end
+  
+  def initialize(parsley, incl = "")
+    if(parsley.is_a?(Hash))
+      parsley = recursive_stringify(parsley).to_json 
+    end
+    @@mutex ||= Mutex.new
+    @@mutex.synchronize do
+      @parsley = CParsley.new(parsley, incl)
+    end
+  end
+  
+  # Valid options:
+  #
+  # Requires one of:
+  # :file -- the input file path or url
+  # :string -- the input string
+  #
+  # And optionally (default is the first listed value):
+  # :input => [:html, :xml]
+  # :output => [:ruby, :json, :xml]
+  # :prune => [true, false]
+  # :sgwrap => [false, true]
+  # :collate => [true, false]
+  # :base => "http://some/base/href"
+  # :allow_net => [true, false]
+  # :allow_local => [true, false]
+  def parse(options = {})
+    options[:file] || options[:string] || (raise ParsleyError.new("must specify what to parse"))
+    
+    options[:sgwrap] = !!options[:sgwrap]
+    options[:is_file] = !!options[:file]
+    options[:has_base] = !!options[:base]
+    
+    options[:base] = options[:base].to_s
+    options[:file] = options[:file].to_s
+    options[:string] = options[:string].to_s
+    
+    options[:input]  ||= :html
+    options[:output] ||= :ruby
+    
+    options[:collate] = true unless options.has_key?(:collate)
+    options[:prune] = true unless options.has_key?(:prune)
+    options[:allow_net] = true unless options.has_key?(:allow_net)
+    options[:allow_local] = true unless options.has_key?(:allow_local)
+    
+    options[:collate] = !!options[:collate]
+    options[:prune] = !!options[:prune]
+    options[:allow_net] = !!options[:allow_net]
+    options[:allow_local] = !!options[:allow_local]
+    
+    @parsley.parse(options)
+  end
+  private
+  
+  def recursive_stringify(obj)
+    case obj
+    when Hash
+      obj.inject({}) do |memo, (k, v)|
+        memo[k.to_s] = recursive_stringify(v)
+        memo
+      end
+    when Array
+      obj.map{|e| recursive_stringify(e) }
+    else
+      obj.to_s
+    end
+  end
+  
+end

data/parsley-ruby.gemspec

@@ -0,0 +1,56 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{parsley-ruby}
+  s.version = "0.4.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Kyle Maxwell"]
+  s.date = %q{2009-11-01}
+  s.description = %q{XML/HTML Parser}
+  s.email = %q{kyle@kylemaxwell.com}
+  s.extra_rdoc_files = [
+    "README"
+  ]
+  s.files = [
+    ".gitignore",
+     "README",
+     "Rakefile",
+     "VERSION",
+     "ext/cparsley.c",
+     "ext/extconf.rb",
+     "lib/parsley.rb",
+     "parsley-ruby.gemspec",
+     "test/test_parsley.rb",
+     "test/yelp-benchmark.rb",
+     "test/yelp-home.html",
+     "test/yelp-home.let",
+     "test/yelp.html"
+  ]
+  s.homepage = %q{http://github.com/fizx/parsley-ruby}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Ruby binding for parsley}
+  s.test_files = [
+    "test/test_parsley.rb",
+     "test/yelp-benchmark.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<json>, ["> 0.0.0"])
+    else
+      s.add_dependency(%q<json>, ["> 0.0.0"])
+    end
+  else
+    s.add_dependency(%q<json>, ["> 0.0.0"])
+  end
+end
+

data/test/test_parsley.rb

@@ -0,0 +1,116 @@
+require "test/unit"
+require File.dirname(__FILE__) + "/../lib/parsley"
+
+class TestParsley < Test::Unit::TestCase
+  def setup
+    @page = File.expand_path(File.dirname(__FILE__) + "/yelp.html")
+    @home = File.expand_path(File.dirname(__FILE__) + "/yelp-home.html")
+    @let  = File.expand_path(File.dirname(__FILE__) + "/yelp-home.let")
+  end
+  
+  def test_segfault_regression
+    simple_html = <<-HTML
+      <html>
+      <body>
+        <h1 class="iCIMS_Header_JobTitle">CEO</h1>
+      </body>
+      </html>
+    HTML
+    
+    struct = {
+      'jobs' => [{
+        'title'         => ".iCIMS_Header_JobTitle",
+        'description?'  => "blah",
+        'location?'     => "blah",
+        'experience?'   => "blah",
+        'education?'    => "blah"
+      }]
+    }
+    parselet = Parsley.new(struct)
+    result = parselet.parse(:string => simple_html)
+    assert_equal "CEO", result['jobs'].first['title']
+    assert result['jobs'].first['description'].nil?
+  end
+  # 
+  # def test_yelp
+  #   @parsley = Parsley.new(File.read(@let))
+  #   out = @parsley.parse(:file => @home)
+  #   assert_equal "/c/sf/shopping", out["categories"][0]["href"]
+  # end
+  # 
+  # def test_parsley_should_raise_if_value_syntax_error
+  #   assert_raises(ParsleyError) do
+  #     Parsley.new({"foo" => nil})
+  #   end
+  #   
+  #   assert_raises(ParsleyError) do
+  #     Parsley.new({"foo" => ""})
+  #   end
+  #   
+  #   assert_raises(ParsleyError) do
+  #     Parsley.new({"foo" => "<<<<<<<<<<<"})
+  #   end
+  # end
+  # 
+  # def test_yelp_xml
+  #   @parsley = Parsley.new(File.read(@let))
+  #   out = @parsley.parse(:file => @home, :output => :xml)
+  # end
+  # 
+  # def test_broken
+  #   @parsley = Parsley.new("hi" => "no-ns:match(h1)")
+  #   assert_raises(ParsleyError) {
+  #     @parsley.parse(:file => @page)
+  #   }
+  # end
+  # 
+  # def test_simple
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   assert_equal({"hi" => "Nick's Crispy Tacos"}, @parsley.parse(:file => @page))
+  # end
+  # 
+  # def test_simple_string
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   assert_equal({"hi" => "Nick's Crispy Tacos"}, @parsley.parse(:string => "<html><body><h1>Nick's Crispy Tacos</h1></body></html>"))
+  # end  
+  # 
+  # def test_xml
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   xml = "<?xml version=\"1.0\"?>\n<parsley:root xmlns:parsley=\"http://parselets.com/json\"><hi position=\"63\">Nick's Crispy Tacos</hi></parsley:root>\n"
+  #   assert_equal(xml, @parsley.parse(:file => @page, :output => :xml))
+  # end
+  # 
+  # def test_sgwrap
+  #   @parsley = Parsley.new("hi" => "p sg_wrap")
+  #   html = "<p><b>hi</b>world</p>"
+  #   assert_equal({"hi" => "world"}, @parsley.parse(:string => html, :sgwrap => true))
+  # end
+  # 
+  # def test_sgwrap_off
+  #   @parsley = Parsley.new("hi" => "p sg_wrap")
+  #   html = "<p><b>hi</b>world</p>"
+  #   assert_raises(ParsleyError) do
+  #     @parsley.parse(:string => html, :sgwrap => false)
+  #   end
+  # end
+  # 
+  # 
+  # def test_json
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   assert_equal('{ "hi": "Nick\'s Crispy Tacos" }', @parsley.parse(:file => @page, :output => :json))
+  # end
+  # 
+  # def test_rescuable_file_error
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   @nonexistant_file = File.dirname(__FILE__) + "/../fixtures/yelp.html"
+  #   assert_equal({"hi" => "Nick's Crispy Tacos"}, @parsley.parse(:file => @nonexistant_file)) rescue nil
+  # end
+  # 
+  # def test_array_string
+  #   @parsley = Parsley.new({"foo" => ["li"]})
+  #   out = @parsley.parse(:file => @page)
+  #   assert_kind_of Hash, out
+  #   assert_kind_of Array, out["foo"], out.inspect
+  #   assert out["foo"].length > 1
+  # end
+end

data/test/yelp-benchmark.rb

@@ -0,0 +1,53 @@
+require "rubygems"
+require "nokogiri"
+require "hpricot"
+require "parsley"
+require "benchmark"
+require "pp"
+
+YELP_HTML = File.dirname(__FILE__) + "/yelp.html"
+
+def noko
+  parse Nokogiri.Hpricot(File.open(YELP_HTML))
+end
+
+def hpri
+  parse Hpricot(File.open(YELP_HTML))  
+end
+
+def parse(doc)
+  out = {}
+  out["name"] = (doc / "h1").first.inner_text
+  out["phone"] = (doc / "#bizPhone").first.inner_text
+  out["address"] = (doc / "address").first.inner_text
+  out["reviews"] = (doc / ".nonfavoriteReview").map do |node|
+    review = {}
+    review["date"] = (node / ".ieSucks .smaller").first.inner_text
+    review["user_name"] = (node / ".reviewer_info a").first.inner_text
+    review["comment"] = (node / ".review_comment").first.inner_text
+    review
+  end
+end
+
+def pars
+  parslet = Parsley.new({
+    "name" => "h1",
+    "phone" => "#bizPhone",
+    "address" => "address",
+    "reviews(.nonfavoriteReview)" => [
+      {
+        "date" => ".ieSucks .smaller",
+        "user_name" => ".reviewer_info a",
+        "comment" => ".review_comment"
+      }
+    ]
+  })
+  pp parslet.parse(:file => YELP_HTML)
+end
+
+Benchmark.bm do |x|
+  x.report("nokogiri: ")  { 3.times { noko } }
+  x.report("hpricot: ")   { 3.times { hpri } }
+  x.report("parsley: ") { 3.times { pars } }
+end
+