le1t0-parsley-ruby 0.4.5.001

data/.gitignore

@@ -0,0 +1,28 @@
+.libs/
+*.o
+*.lo
+dexterc
+dexter
+parsleyc
+parsley
+.deps/
+Makefile
+y.tab.c
+autom4te.cache/
+autoscan.log
+config.log
+configure.scan
+parser.c
+scanner.c
+libparsley.la
+parser.h
+test.log
+parsley*.gem
+ext/cparsley.bundle
+ext/cparsley.so
+ext/Makefile
+ext/conftest.dSYM/
+work
+ext/mkmf.log
+pkg
+le1t0-parsley-ruby.gemspec

data/CHANGELOG

@@ -0,0 +1,8 @@
+0.4.5.001
+
+- Changed ext/extconf.rb such that it is much simpler and compiles. Added external dependency for parsley instead of compiling
+inline.
+
+0.4.3
+- Added CHANGELOG
+- Only assigning ARCHFLAGS if not already specified.

data/README

@@ -0,0 +1,32 @@
+ABOUT
+
+Ruby bindings for Parsley.
+
+INSTALLATION
+
+= Get Parsley and Dependancies = 
+
+Download Parsley from http://github.com/fizx/parsley/tree/master following the installation directions located at http://github.com/fizx/parsley/blob/master/INSTALL
+
+= Install parsley-ruby =
+
+From source:
+  sudo rake install
+
+From GitHub: DEPRECATED!
+
+From GemCutter
+
+  Run the following if you haven't already:
+  gem sources -a http://gemcutter.org
+  Install the gem:
+  sudo gem install parsley-ruby
+
+PARSLETS.COM INTEGRATION
+
+We also recommend installing the free online_parselets rubygem in order to use other people's parselets and to share your own:
+  Run the following if you haven't already:
+  gem sources -a http://gems.github.com
+  Install the gem:
+  sudo gem install iterationlabs-online_parslets
+

data/Rakefile

@@ -0,0 +1,57 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "le1t0-parsley-ruby"
+    gem.summary = "Ruby binding for parsley"
+    gem.description = "XML/HTML Parser"
+    gem.email = "dev@ewout.to"
+    gem.homepage = "http://github.com/le1t0/parsley-ruby"
+    gem.authors = ["Le1t0"]
+    gem.add_dependency("json", ["> 0.0.0"])
+    gem.require_paths = ["lib", "ext"]
+    gem.extensions = "ext/extconf.rb"  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "robots #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.4.5.001

data/ext/cparsley.c

@@ -0,0 +1,140 @@
+#include "ruby.h"
+#include <stdio.h>
+#include <libxslt/xslt.h>
+#include <libexslt/exslt.h>
+#include <libxslt/xsltInternals.h>
+#include <libxslt/transform.h>
+#include <libxml/parser.h>
+#include <libxml/HTMLparser.h>
+#include <libxml/HTMLtree.h>
+#include <libxml/xmlwriter.h>
+#include <parsley.h>
+#include <json/json.h>
+#include <xml2json.h>
+
+VALUE _new(VALUE, VALUE, VALUE);
+VALUE _parse(VALUE, VALUE);
+VALUE _rb_set_user_agent(VALUE self, VALUE agent);
+VALUE c_parsley_err;
+VALUE c_parsley;
+
+void Init_cparsley()
+{
+	c_parsley = rb_define_class("CParsley", rb_cObject);
+	c_parsley_err = rb_define_class("ParsleyError", rb_eRuntimeError);
+	rb_define_singleton_method(c_parsley, "new", _new, 2);
+	rb_define_singleton_method(c_parsley, "set_user_agent", _rb_set_user_agent, 1);
+	rb_define_method(c_parsley, "parse", _parse, 1);
+}
+
+VALUE 
+_new(VALUE self, VALUE parsley, VALUE incl){
+	parsleyPtr ptr = parsley_compile(STR2CSTR(parsley), STR2CSTR(incl));
+	if(ptr->error != NULL) {
+	  rb_raise(c_parsley_err, ptr->error);
+    parsley_free(ptr);
+    return Qnil;
+	}
+	
+ 	return Data_Wrap_Struct(c_parsley, 0, parsley_free, ptr);
+}
+
+VALUE 
+_rb_set_user_agent(VALUE self, VALUE agent) {
+  parsley_set_user_agent(STR2CSTR(agent));
+  return Qtrue;
+}
+
+
+static VALUE 
+rubify_recurse(xmlNodePtr xml) {
+  if(xml == NULL) return NULL;
+  xmlNodePtr child;
+  VALUE obj = Qnil;
+
+  switch(xml->type) {
+    case XML_ELEMENT_NODE:
+      child = xml->children;
+      if(xml->ns == NULL) {
+        child = xml;
+        obj = rb_hash_new();
+        while(child != NULL) {
+          rb_hash_aset(obj, rb_str_new2(child->name), rubify_recurse(child->children));
+          child = child->next;
+        }
+      } else if(!strcmp(xml->ns->prefix, "parsley")) {
+        if(!strcmp(xml->name, "groups")) {
+          obj = rb_ary_new();
+          while(child != NULL) {
+            rb_ary_push(obj, rubify_recurse(child->children));
+            child = child->next;
+          }          
+        } else if(!strcmp(xml->name, "group")) {
+          // Implicitly handled by parsley:groups handler
+        }
+      }
+      break;
+    case XML_TEXT_NODE:
+      obj = rb_str_new2(xml->content);
+      break;
+  }
+  // inspect(obj);
+  return obj;
+}
+
+static VALUE 
+_parse_doc(parsedParsleyPtr ptr, VALUE type) {
+	if(ptr->error != NULL || ptr->xml == NULL) {
+    if(ptr->error == NULL) ptr->error = strdup("Unknown parsley error");
+		rb_raise(c_parsley_err, ptr->error);
+    parsed_parsley_free(ptr);
+		return Qnil;
+	}
+	
+	VALUE output;
+	if(type == ID2SYM(rb_intern("json"))) {
+		struct json_object *json = xml2json(ptr->xml->children->children);
+		char* str = json_object_to_json_string(json);
+		output = rb_str_new2(str);
+		json_object_put(json);
+	} else if(type == ID2SYM(rb_intern("xml"))) {
+		xmlChar* str;
+		int size;
+		xmlDocDumpMemory(ptr->xml, &str, &size);
+		output = rb_str_new(str, size);
+	} else {
+ 		output = rubify_recurse(ptr->xml->children->children);
+		if((void*)output == NULL) output = Qnil; 
+	}
+	
+  parsed_parsley_free(ptr);
+  
+	return output;
+}
+
+#define OPT(A) rb_hash_aref(options, ID2SYM(rb_intern(A)))
+#define OPT_BOOL(A) (OPT(A) != Qnil && OPT(A) != Qfalse)
+#define OPT_MATCH(A, B) (rb_hash_aref(options, ID2SYM(rb_intern(A))) == ID2SYM(rb_intern(B)))
+
+VALUE _parse(VALUE self, VALUE options){
+	parsleyPtr parsley;
+	Data_Get_Struct(self, parsleyPtr, parsley);
+  int flags = 0;
+  char *base = NULL;
+  if(OPT_MATCH("input", "html"))    flags |= PARSLEY_OPTIONS_HTML;
+  if(OPT_BOOL("prune"))             flags |= PARSLEY_OPTIONS_PRUNE;
+  if(OPT_BOOL("collate"))           flags |= PARSLEY_OPTIONS_COLLATE;
+  if(OPT_BOOL("allow_net"))         flags |= PARSLEY_OPTIONS_ALLOW_NET;
+  if(OPT_BOOL("allow_local"))       flags |= PARSLEY_OPTIONS_ALLOW_LOCAL;
+  if(OPT_BOOL("sgwrap"))            flags |= PARSLEY_OPTIONS_SGWRAP;
+  if(OPT_BOOL("has_base"))          base = STR2CSTR(OPT("base"));
+  
+  // printf("prune: %d\nallow_net: %d\nallow_local: %d\nhas_base: %d\nflags: %d\n", OPT_BOOL("prune"), OPT_BOOL("allow_net"), OPT_BOOL("allow_local"), OPT_BOOL("has_base"), flags);
+  
+  if(OPT_BOOL("is_file")) {
+    return _parse_doc(parsley_parse_file(parsley, STR2CSTR(OPT("file")), flags), OPT("output"));
+  } else {
+    char * str = STR2CSTR(OPT("string"));
+    return _parse_doc(parsley_parse_string(parsley, str, strlen(str), base, flags), OPT("output"));
+  }
+}

data/ext/extconf.rb

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require 'mkmf'
+
+$CFLAGS << " -I/usr/local/include -I/usr/include/libxml2"
+$LDFLAGS << " -L/usr/local/lib -lparsley"
+
+create_makefile('cparsley')

data/lib/parsley.rb

@@ -0,0 +1,84 @@
+require File.dirname(__FILE__) + "/../ext/cparsley"
+require "rubygems"
+require "json"
+require "thread"
+
+class Parsley
+  
+  def self.user_agent=(agent)
+    @user_agent = agent
+    CParsley.set_user_agent(agent.to_s)
+  end
+  
+  def self.user_agent
+    @user_agent
+  end
+  
+  def initialize(parsley, incl = "")
+    if(parsley.is_a?(Hash))
+      parsley = recursive_stringify(parsley).to_json 
+    end
+    @@mutex ||= Mutex.new
+    @@mutex.synchronize do
+      @parsley = CParsley.new(parsley, incl)
+    end
+  end
+  
+  # Valid options:
+  #
+  # Requires one of:
+  # :file -- the input file path or url
+  # :string -- the input string
+  #
+  # And optionally (default is the first listed value):
+  # :input => [:html, :xml]
+  # :output => [:ruby, :json, :xml]
+  # :prune => [true, false]
+  # :sgwrap => [false, true]
+  # :collate => [true, false]
+  # :base => "http://some/base/href"
+  # :allow_net => [true, false]
+  # :allow_local => [true, false]
+  def parse(options = {})
+    options[:file] || options[:string] || (raise ParsleyError.new("must specify what to parse"))
+    
+    options[:sgwrap] = !!options[:sgwrap]
+    options[:is_file] = !!options[:file]
+    options[:has_base] = !!options[:base]
+    
+    options[:base] = options[:base].to_s
+    options[:file] = options[:file].to_s
+    options[:string] = options[:string].to_s
+    
+    options[:input]  ||= :html
+    options[:output] ||= :ruby
+    
+    options[:collate] = true unless options.has_key?(:collate)
+    options[:prune] = true unless options.has_key?(:prune)
+    options[:allow_net] = true unless options.has_key?(:allow_net)
+    options[:allow_local] = true unless options.has_key?(:allow_local)
+    
+    options[:collate] = !!options[:collate]
+    options[:prune] = !!options[:prune]
+    options[:allow_net] = !!options[:allow_net]
+    options[:allow_local] = !!options[:allow_local]
+    
+    @parsley.parse(options)
+  end
+  private
+  
+  def recursive_stringify(obj)
+    case obj
+    when Hash
+      obj.inject({}) do |memo, (k, v)|
+        memo[k.to_s] = recursive_stringify(v)
+        memo
+      end
+    when Array
+      obj.map{|e| recursive_stringify(e) }
+    else
+      obj.to_s
+    end
+  end
+  
+end

data/test/test_parsley.rb

@@ -0,0 +1,116 @@
+require "test/unit"
+require File.dirname(__FILE__) + "/../lib/parsley"
+
+class TestParsley < Test::Unit::TestCase
+  def setup
+    @page = File.expand_path(File.dirname(__FILE__) + "/yelp.html")
+    @home = File.expand_path(File.dirname(__FILE__) + "/yelp-home.html")
+    @let  = File.expand_path(File.dirname(__FILE__) + "/yelp-home.let")
+  end
+  
+  def test_segfault_regression
+    simple_html = <<-HTML
+      <html>
+      <body>
+        <h1 class="iCIMS_Header_JobTitle">CEO</h1>
+      </body>
+      </html>
+    HTML
+    
+    struct = {
+      'jobs' => [{
+        'title'         => ".iCIMS_Header_JobTitle",
+        'description?'  => "blah",
+        'location?'     => "blah",
+        'experience?'   => "blah",
+        'education?'    => "blah"
+      }]
+    }
+    parselet = Parsley.new(struct)
+    result = parselet.parse(:string => simple_html)
+    assert_equal "CEO", result['jobs'].first['title']
+    assert result['jobs'].first['description'].nil?
+  end
+  # 
+  # def test_yelp
+  #   @parsley = Parsley.new(File.read(@let))
+  #   out = @parsley.parse(:file => @home)
+  #   assert_equal "/c/sf/shopping", out["categories"][0]["href"]
+  # end
+  # 
+  # def test_parsley_should_raise_if_value_syntax_error
+  #   assert_raises(ParsleyError) do
+  #     Parsley.new({"foo" => nil})
+  #   end
+  #   
+  #   assert_raises(ParsleyError) do
+  #     Parsley.new({"foo" => ""})
+  #   end
+  #   
+  #   assert_raises(ParsleyError) do
+  #     Parsley.new({"foo" => "<<<<<<<<<<<"})
+  #   end
+  # end
+  # 
+  # def test_yelp_xml
+  #   @parsley = Parsley.new(File.read(@let))
+  #   out = @parsley.parse(:file => @home, :output => :xml)
+  # end
+  # 
+  # def test_broken
+  #   @parsley = Parsley.new("hi" => "no-ns:match(h1)")
+  #   assert_raises(ParsleyError) {
+  #     @parsley.parse(:file => @page)
+  #   }
+  # end
+  # 
+  # def test_simple
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   assert_equal({"hi" => "Nick's Crispy Tacos"}, @parsley.parse(:file => @page))
+  # end
+  # 
+  # def test_simple_string
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   assert_equal({"hi" => "Nick's Crispy Tacos"}, @parsley.parse(:string => "<html><body><h1>Nick's Crispy Tacos</h1></body></html>"))
+  # end  
+  # 
+  # def test_xml
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   xml = "<?xml version=\"1.0\"?>\n<parsley:root xmlns:parsley=\"http://parselets.com/json\"><hi position=\"63\">Nick's Crispy Tacos</hi></parsley:root>\n"
+  #   assert_equal(xml, @parsley.parse(:file => @page, :output => :xml))
+  # end
+  # 
+  # def test_sgwrap
+  #   @parsley = Parsley.new("hi" => "p sg_wrap")
+  #   html = "<p><b>hi</b>world</p>"
+  #   assert_equal({"hi" => "world"}, @parsley.parse(:string => html, :sgwrap => true))
+  # end
+  # 
+  # def test_sgwrap_off
+  #   @parsley = Parsley.new("hi" => "p sg_wrap")
+  #   html = "<p><b>hi</b>world</p>"
+  #   assert_raises(ParsleyError) do
+  #     @parsley.parse(:string => html, :sgwrap => false)
+  #   end
+  # end
+  # 
+  # 
+  # def test_json
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   assert_equal('{ "hi": "Nick\'s Crispy Tacos" }', @parsley.parse(:file => @page, :output => :json))
+  # end
+  # 
+  # def test_rescuable_file_error
+  #   @parsley = Parsley.new("hi" => "h1")
+  #   @nonexistant_file = File.dirname(__FILE__) + "/../fixtures/yelp.html"
+  #   assert_equal({"hi" => "Nick's Crispy Tacos"}, @parsley.parse(:file => @nonexistant_file)) rescue nil
+  # end
+  # 
+  # def test_array_string
+  #   @parsley = Parsley.new({"foo" => ["li"]})
+  #   out = @parsley.parse(:file => @page)
+  #   assert_kind_of Hash, out
+  #   assert_kind_of Array, out["foo"], out.inspect
+  #   assert out["foo"].length > 1
+  # end
+end