fastxml 0.1.91

data/ext/fastxml_node.h

@@ -0,0 +1,21 @@
+/*
+ * $Id$
+ */
+
+#ifndef fastxml_node_h
+#define fastxml_node_h
+RUBY_EXTERN VALUE fastxml_node_initialize(VALUE self);
+RUBY_EXTERN VALUE fastxml_node_search(VALUE self,VALUE raw_xpath, VALUE blk);
+RUBY_EXTERN VALUE fastxml_node_name(VALUE self);
+RUBY_EXTERN VALUE fastxml_node_value(VALUE self);
+RUBY_EXTERN VALUE fastxml_node_value_set(VALUE self, VALUE new_val);
+RUBY_EXTERN VALUE fastxml_node_innerxml(VALUE self);
+RUBY_EXTERN VALUE fastxml_node_to_s(VALUE self);
+RUBY_EXTERN VALUE fastxml_node_xpath(VALUE self);
+RUBY_EXTERN VALUE fastxml_node_attr(VALUE self);
+RUBY_EXTERN VALUE fastxml_node_children(VALUE self);
+RUBY_EXTERN VALUE fastxml_node_next(VALUE self);
+RUBY_EXTERN VALUE fastxml_node_prev(VALUE self);
+RUBY_EXTERN VALUE fastxml_node_parent(VALUE self);
+RUBY_EXTERN VALUE fastxml_node_inspect(VALUE self);
+#endif

data/ext/fastxml_nodelist.c

@@ -0,0 +1,146 @@
+/*
+ *  $Id: fastxml_node.c 29 2007-08-16 05:16:47Z segfault $
+ */
+
+#include "fastxml.h"
+#include "fastxml_node.h"
+#include "fastxml_doc.h"
+#include "fastxml_nodelist.h"
+
+/* {{{ fastxml_nodelist
+ */
+VALUE fastxml_nodelist_inspect(VALUE self)
+{
+    VALUE dv;
+    VALUE *argv;
+    fxml_data_t *data;
+
+    dv = rb_iv_get( self, "@lxml_doc" );    
+    Data_Get_Struct( dv, fxml_data_t, data );
+
+    argv = ALLOCA_N( VALUE, 4 );
+    argv[0] = rb_str_new2( "#<%s:0x%x %d>" );
+    argv[1] = CLASS_OF( self );
+    argv[2] = rb_obj_id( self );
+    argv[3] = fastxml_nodelist_length( self );
+    return rb_f_sprintf( 4, argv );
+}
+
+VALUE fastxml_nodelist_initialize(VALUE self)
+{
+    return self;
+}
+
+VALUE fastxml_nodelist_length(VALUE self)
+{
+    VALUE dv;
+    xmlNodePtr cur;
+    fxml_data_t *data;
+
+    dv = rb_iv_get( self, "@lxml_doc" );    
+    Data_Get_Struct( dv, fxml_data_t, data );
+	
+	if (data->list_len == -1)
+	{
+		data->list_len = 0;
+		cur = data->list;
+		while (cur != NULL)
+		{
+			data->list_len++;
+			cur = cur->next;
+		}
+	}
+	
+	return rb_int2inum( data->list_len );
+}
+
+VALUE fastxml_nodelist_obj_to_ary(fxml_data_t *root)
+{
+	VALUE ret;
+    xmlNodePtr cur = root->list;	
+	
+ 	ret = rb_ary_new();
+    while (cur != NULL) {        
+        rb_ary_push( ret, fastxml_raw_node_to_obj( cur ) );
+		cur = cur->next;
+	}	
+	
+	return ret;
+}
+
+VALUE fastxml_nodeset_obj_to_ary(fxml_data_t *root)
+{
+	VALUE ret;
+	xmlNodePtr cur = root->xpath_obj->nodesetval->nodeTab;	
+	int i;
+	
+ 	ret = rb_ary_new();
+    for (i = 0; i < root->list_len; i++) {
+        rb_ary_push( ret, fastxml_raw_node_to_obj( cur ) );
+		cur++;
+    }
+	
+	return ret;
+}
+
+VALUE fastxml_nodelist_gen_list(VALUE self, fxml_data_t *data)
+{
+	VALUE lst = rb_iv_get( self, "@list" );
+
+	if (lst == Qnil) {
+	 	if (data->xpath_obj != NULL) {
+			lst = fastxml_nodeset_obj_to_ary( data );
+			rb_iv_set( self, "@list", lst );
+		} else {
+			lst = fastxml_nodelist_obj_to_ary( data );
+			rb_iv_set( self, "@list", lst );
+		}
+	}
+
+	return lst;
+}
+
+VALUE fastxml_nodelist_to_ary(VALUE self)
+{
+	VALUE dv;
+	fxml_data_t *data;
+	
+	dv = rb_iv_get( self, "@lxml_doc" );
+	Data_Get_Struct( dv, fxml_data_t, data );	
+	return fastxml_nodelist_gen_list( self, data );		
+}
+
+VALUE fastxml_nodelist_each(VALUE self)
+{
+	VALUE lst, dv;
+	fxml_data_t *data;
+	int i;
+	
+	dv = rb_iv_get( self, "@lxml_doc" );
+	Data_Get_Struct( dv, fxml_data_t, data );	
+	lst = fastxml_nodelist_gen_list( self, data );
+	
+	for (i=0; i<RARRAY(lst)->len; i++) {
+		rb_yield( RARRAY(lst)->ptr[i] );
+    }
+	
+	return self;
+}
+
+VALUE fastxml_nodelist_entry(VALUE self, long idx)
+{
+	VALUE lst, dv;
+	fxml_data_t *data;
+	
+	dv = rb_iv_get( self, "@lxml_doc" );
+	Data_Get_Struct( dv, fxml_data_t, data );	
+	lst = fastxml_nodelist_gen_list( self, data );
+	if (idx > 0) // this comes in offset by 1
+		idx = idx-1;
+	// TODO: find out why this is provided offset by 1 and not 0-based
+
+	return rb_ary_entry( lst, idx );
+}
+
+/* }}} fastxml_nodelist
+ */

data/ext/fastxml_nodelist.h

@@ -0,0 +1,13 @@
+/*
+ * $Id$
+ */
+
+#ifndef fastxml_nodelist_h
+#define fastxml_nodelist_h
+RUBY_EXTERN VALUE fastxml_nodelist_initialize(VALUE self);
+RUBY_EXTERN VALUE fastxml_nodelist_inspect(VALUE self);
+RUBY_EXTERN VALUE fastxml_nodelist_length(VALUE self);
+RUBY_EXTERN VALUE fastxml_nodelist_entry(VALUE self, long idx);
+RUBY_EXTERN VALUE fastxml_nodelist_each(VALUE self);
+RUBY_EXTERN VALUE fastxml_nodelist_to_ary(VALUE self);
+#endif /*fastxml_nodelist_h*/

data/ext/mkmf.log

@@ -0,0 +1,119 @@
+find_header: checking for #include <libxml/tree.h>
+... -------------------- yes
+
+"/usr/bin/cpp-4.0 -I. -I/opt/local/lib/ruby/1.8/i686-darwin9.2.2 -I. -I/opt/local/include -O2  -fno-common -pipe -fno-common  conftest.c -o conftest.i"
+conftest.c:1:25: error: libxml/tree.h: No such file or directory
+checked program was:
+/* begin */
+1: #include <libxml/tree.h>
+/* end */
+
+"/usr/bin/cpp-4.0 -I. -I/opt/local/lib/ruby/1.8/i686-darwin9.2.2 -I. -I/opt/local/include -O2  -fno-common -pipe -fno-common -I/usr/include/libxml2 conftest.c -o conftest.i"
+checked program was:
+/* begin */
+1: #include <libxml/tree.h>
+/* end */
+
+--------------------
+
+find_header: checking for #include <libxslt/xslt.h>
+... -------------------- yes
+
+"/usr/bin/cpp-4.0 -I. -I/opt/local/lib/ruby/1.8/i686-darwin9.2.2 -I. -I/usr/include/libxml2 -I/opt/local/include -O2  -fno-common -pipe -fno-common  conftest.c -o conftest.i"
+checked program was:
+/* begin */
+1: #include <libxslt/xslt.h>
+/* end */
+
+--------------------
+
+find_library: checking for xmlInitParser() in -lxml2... -------------------- yes
+
+"/usr/bin/gcc-4.0 -o conftest -I. -I/opt/local/lib/ruby/1.8/i686-darwin9.2.2 -I. -I/usr/include/libxml2 -I/opt/local/include -O2  -fno-common -pipe -fno-common conftest.c  -L"." -L"/opt/local/lib" -L. -L/opt/local/lib     -lruby-static -lxml2  -lpthread -ldl -lobjc  "
+conftest.c: In function ‘t’:
+conftest.c:3: error: ‘xmlInitParser’ undeclared (first use in this function)
+conftest.c:3: error: (Each undeclared identifier is reported only once
+conftest.c:3: error: for each function it appears in.)
+checked program was:
+/* begin */
+1: /*top*/
+2: int main() { return 0; }
+3: int t() { void ((*volatile p)()); p = (void ((*)()))xmlInitParser; return 0; }
+/* end */
+
+"/usr/bin/gcc-4.0 -o conftest -I. -I/opt/local/lib/ruby/1.8/i686-darwin9.2.2 -I. -I/usr/include/libxml2 -I/opt/local/include -O2  -fno-common -pipe -fno-common conftest.c  -L"." -L"/opt/local/lib" -L. -L/opt/local/lib     -lruby-static -lxml2  -lpthread -ldl -lobjc  "
+checked program was:
+/* begin */
+1: /*top*/
+2: int main() { return 0; }
+3: int t() { xmlInitParser(); return 0; }
+/* end */
+
+--------------------
+
+have_library: checking for xmlInitParser() in -lxml2... -------------------- yes
+
+"/usr/bin/gcc-4.0 -o conftest -I. -I/opt/local/lib/ruby/1.8/i686-darwin9.2.2 -I. -I/usr/include/libxml2 -I/opt/local/include -O2  -fno-common -pipe -fno-common conftest.c  -L"." -L"/opt/local/lib" -L. -L/opt/local/lib    -lxml2  -lruby-static -lxml2 -lxml2  -lpthread -ldl -lobjc  "
+checked program was:
+/* begin */
+1: #include <libxml/parser.h>
+2: 
+3: /*top*/
+4: int main() { return 0; }
+5: int t() { void ((*volatile p)()); p = (void ((*)()))xmlInitParser; return 0; }
+/* end */
+
+--------------------
+
+find_library: checking for xmlInitParser() in -lxslt... -------------------- yes
+
+"/usr/bin/gcc-4.0 -o conftest -I. -I/opt/local/lib/ruby/1.8/i686-darwin9.2.2 -I. -I/usr/include/libxml2 -I/opt/local/include -O2  -fno-common -pipe -fno-common conftest.c  -L"." -L"/opt/local/lib" -L. -L/opt/local/lib    -lxml2 -lxml2  -lruby-static -lxslt -lxml2 -lxml2  -lpthread -ldl -lobjc  "
+conftest.c: In function ‘t’:
+conftest.c:3: error: ‘xmlInitParser’ undeclared (first use in this function)
+conftest.c:3: error: (Each undeclared identifier is reported only once
+conftest.c:3: error: for each function it appears in.)
+checked program was:
+/* begin */
+1: /*top*/
+2: int main() { return 0; }
+3: int t() { void ((*volatile p)()); p = (void ((*)()))xmlInitParser; return 0; }
+/* end */
+
+"/usr/bin/gcc-4.0 -o conftest -I. -I/opt/local/lib/ruby/1.8/i686-darwin9.2.2 -I. -I/usr/include/libxml2 -I/opt/local/include -O2  -fno-common -pipe -fno-common conftest.c  -L"." -L"/opt/local/lib" -L. -L/opt/local/lib    -lxml2 -lxml2  -lruby-static -lxslt -lxml2 -lxml2  -lpthread -ldl -lobjc  "
+checked program was:
+/* begin */
+1: /*top*/
+2: int main() { return 0; }
+3: int t() { xmlInitParser(); return 0; }
+/* end */
+
+--------------------
+
+have_library: checking for xsltParseStylesheetFile() in -lxslt... -------------------- yes
+
+"/usr/bin/gcc-4.0 -o conftest -I. -I/opt/local/lib/ruby/1.8/i686-darwin9.2.2 -I. -I/usr/include/libxml2 -I/opt/local/include -O2  -fno-common -pipe -fno-common conftest.c  -L"." -L"/opt/local/lib" -L. -L/opt/local/lib    -lxslt -lxml2 -lxml2  -lruby-static -lxslt -lxslt -lxml2 -lxml2  -lpthread -ldl -lobjc  "
+conftest.c: In function ‘t’:
+conftest.c:5: error: ‘xsltParseStylesheetFile’ undeclared (first use in this function)
+conftest.c:5: error: (Each undeclared identifier is reported only once
+conftest.c:5: error: for each function it appears in.)
+checked program was:
+/* begin */
+1: #include <libxslt/xslt.h>
+2: 
+3: /*top*/
+4: int main() { return 0; }
+5: int t() { void ((*volatile p)()); p = (void ((*)()))xsltParseStylesheetFile; return 0; }
+/* end */
+
+"/usr/bin/gcc-4.0 -o conftest -I. -I/opt/local/lib/ruby/1.8/i686-darwin9.2.2 -I. -I/usr/include/libxml2 -I/opt/local/include -O2  -fno-common -pipe -fno-common conftest.c  -L"." -L"/opt/local/lib" -L. -L/opt/local/lib    -lxslt -lxml2 -lxml2  -lruby-static -lxslt -lxslt -lxml2 -lxml2  -lpthread -ldl -lobjc  "
+checked program was:
+/* begin */
+1: #include <libxslt/xslt.h>
+2: 
+3: /*top*/
+4: int main() { return 0; }
+5: int t() { xsltParseStylesheetFile(); return 0; }
+/* end */
+
+--------------------
+

data/lib/fastxml_lib.rb

@@ -0,0 +1,93 @@
+# $Id$
+module FastXml
+  VERSION = "0.1.91" 
+end
+
+module FastXml::Common
+  def children_of_type(type)
+    self.search( "//#{type}" )
+  end
+  
+  def each_child(&blk)
+    self.children.each { |chld| yield chld }
+  end
+  
+  def /(xpath)
+    self.search( "/#{xpath.to_s}" )
+  end
+  
+  def at(xpath)
+    nodes = self.search( xpath )
+    return nil unless nodes && nodes.length > 0
+    nodes[0]
+  end
+  
+  alias :to_s :display
+end
+
+class FastXml::Doc
+  include FastXml::Common
+  
+  def doc?
+    true
+  end
+  
+  def doctype?
+    nil
+  end
+  
+  def forgiving?
+    (@forgiving ||= false)
+  end
+  
+  def  validate?
+    (@validate_dtd ||= false)
+  end
+  
+  def xpath
+    "/"
+  end
+end
+
+class FastXml::Node
+  include FastXml::Common
+  def doc?
+    false
+  end
+end
+
+class FastXml::NodeList
+  def [](idx)
+    self.entry(idx)
+  end
+  
+  def first
+    self.entry(0)
+  end
+  
+  def last
+    self.entry(-1)
+  end
+  
+  def empty?
+    return (length == 0)
+  end
+  
+  def at(tgt)
+    return self.entry( tgt.to_i ) if tgt =~ /^\d+$/
+    ret = []
+    each { |nd| ret << (nd/tgt).to_ary }
+    ret.flatten!
+  end
+end
+
+
+def FastXml(data=nil, opts = {}, &blk)
+  FastXml::Doc.new( data, opts, &blk )
+end
+
+def FastHtml(data=nil, opts = {}, &blk)
+  opts ||= {}
+  opts[:html] = true
+  FastXml::Doc.new( data, opts, &blk )
+end

data/specs/basic_html_spec.rb

@@ -0,0 +1,70 @@
+# $Id$
+$: << '../ext'
+$: << './ext'
+
+require 'fastxml'
+
+describe FastXml::Doc, " doing html parsing" do
+  before(:all) do
+    @data_raw = open( "./test_data/hasno_feed.html" )
+    @data_ary = @data_raw.readlines
+    @data_str = @data_ary.join('')
+  end
+
+  before do
+    @data_raw.rewind if @data_raw
+  end
+
+  after(:all) do
+    @data_raw.close if @data_raw
+  end
+
+
+  it 'should parse string input' do
+    @data_str.should_not be_nil
+    doc = FastXml::Doc.new( @data_str, {:html=>true} )
+    doc.should_not be_nil
+    doc.to_s.should_not be_nil
+  end
+
+  it 'should parse array input' do
+    @data_ary.should_not be_nil
+    doc = FastXml::Doc.new( @data_ary, {:html=>true} )
+    doc.should_not be_nil
+    doc.to_s.should_not be_nil
+  end
+
+  it 'should be able to parse hasno and search' do
+    doc = FastHtml( @data_str )
+    descs = (doc/"p[class=description]")
+    descs.should_not be_nil
+    descs.each do |d|
+      d.should_not be_nil
+      d.length.should_be >= 1
+    end
+  end
+  
+  it 'should handle the twitter public timeline' do
+    raw_data = open( "./test_data/twitter_public.html" ).readlines.join('')
+    doc = FastHtml( raw_data )
+    doc.should_not be_nil
+    doc.to_s.should_not be_nil
+    doc.to_s.length.should >= 30000
+    doc.root.should_not be_nil
+    (doc/"").should_not be_nil
+    doc.root.children.should_not be_nil
+  end
+
+
+
+  it 'should be able to handle the cnn site' do
+    raw_data = open( "./test_data/cnn_main.html" ).readlines.join('')
+    doc = FastHtml( raw_data )
+    doc.should_not be_nil
+    doc.to_s.should_not be_nil
+    doc.to_s.length.should >= 10000
+    (doc/"").should_not be_nil
+    doc.root.children.should_not be_nil
+  end
+
+end