ox 1.2.15 → 1.3.0

data/lib/ox.rb

@@ -95,5 +95,6 @@ require 'ox/doctype'
 require 'ox/element'
 require 'ox/document'
 require 'ox/bag'
+require 'ox/sax'
 
 require 'ox/ox' # C extension

data/lib/ox/sax.rb

@@ -0,0 +1,72 @@
+module Ox
+  # A SAX style parse handler. The Ox::Sax handler class should be subclasses
+  # and then used with the Ox.sax_parse() method. The Sax methods will then be
+  # called as the file is parsed. This is best suited for very large files or
+  # IO streams.<p/>
+  # @example
+  # 
+  #  require 'ox'
+  #
+  #  class MySax < ::Ox::Sax
+  #    def initialize()
+  #      @element_name = []
+  #    end
+  #
+  #    def start_element(name, attrs)
+  #      @element_names << name
+  #    end
+  #  end
+  #
+  #  any = MySax.new()
+  #  File.open('any.xml', 'r') do |f|
+  #    Xml.sax_parse(any, f)
+  #  end
+  #
+  # To make the desired methods active while parsing the desired method should
+  # be made public in the subclasses. If the methods remain private they will
+  # not be called during parsing.
+  #
+  #    def instruct(target, attrs); end
+  #    def doctype(value); end
+  #    def comment(value); end
+  #    def cdata(value); end
+  #    def text(value); end
+  #    def start_element(name, attrs); end
+  #    def end_element(name); end
+  #
+  class Sax
+    # Create a new instance of the Sax handler class.
+    def initialize()
+    end
+
+    # To make the desired methods active while parsing the desired method
+    # should be made public in the subclasses. If the methods remain private
+    # they will not be called during parsing.
+    private
+
+    def instruct(target, attrs)
+    end
+
+    def doctype(value)
+    end
+
+    def comment(value)
+    end
+
+    def cdata(value)
+    end
+
+    def text(value)
+    end
+
+    def start_element(name, attrs)
+    end
+
+    def end_element(name)
+    end
+    
+    def error(message, line, column)
+    end
+    
+  end # Sax
+end # Ox

data/lib/ox/version.rb

@@ -1,5 +1,5 @@
 
 module Ox
   # Current version of the module. 
-  VERSION = '1.2.15'
+  VERSION = '1.3.0'
 end

data/test/func.rb

@@ -5,7 +5,7 @@ $: << '../lib'
 $: << '../ext'
 
 if __FILE__ == $0
-  if (i = ARGV.index('-I'))
+  while (i = ARGV.index('-I'))
     x,path = ARGV.slice!(i, 2)
     $: << path
   end
@@ -333,6 +333,29 @@ class Func < ::Test::Unit::TestCase
     loaded
   end
 
+  def test_nameerror
+    begin
+      "x".foo
+    rescue Exception => e
+      xml = Ox.dump(e, :effort => :tolerant)
+      o = Ox.load(xml, mode: :object)
+      xml2 = Ox.dump(o, :effort => :tolerant)
+      assert_equal(xml, xml2)
+    end
+  end
+  
+  def test_mutex
+    # Mutex can not be serialize but it should not raise an exception.
+    xml = Ox.dump(Mutex.new, :indent => 2, :effort => :tolerant)
+    assert_equal(%{<z/>
+}, xml)
+    xml = Ox.dump(Bag.new(:@x => Mutex.new), :indent => 2, :effort => :tolerant)
+    assert_equal(%{<o c="Bag">
+  <z a="@x"/>
+</o>
+}, xml)
+  end
+  
   def test_encoding
     if RUBY_VERSION.start_with?('1.8')
       assert(true)

data/test/perf_sax.rb

@@ -0,0 +1,233 @@
+#!/usr/bin/env ruby -wW1
+
+$: << '.'
+$: << '..'
+$: << '../lib'
+$: << '../ext'
+
+if __FILE__ == $0
+  while (i = ARGV.index('-I'))
+    x,path = ARGV.slice!(i, 2)
+    $: << path
+  end
+end
+
+require 'optparse'
+require 'ox'
+require 'sample'
+require 'files'
+begin
+  require 'nokogiri'
+rescue Exception => e
+end
+begin
+  require 'libxml'
+rescue Exception => e
+end
+
+$verbose = 0
+$ox_only = false
+$all_cbs = false
+$filename = nil # nil indicates new file names perf.xml will be created and used
+$filesize = 1000 # KBytes
+$iter = 100
+
+opts = OptionParser.new
+opts.on("-v", "increase verbosity")                            { $verbose += 1 }
+opts.on("-x", "ox only")                                       { $ox_only = true }
+opts.on("-a", "all callbacks")                                 { $all_cbs = true }
+opts.on("-f", "--file [String]", String, "filename")           { |f| $filename = f }
+opts.on("-i", "--iterations [Int]", Integer, "iterations")     { |i| $iter = i }
+opts.on("-s", "--size [Int]", Integer, "file size in KBytes")  { |s| $filesize = s }
+opts.on("-h", "--help", "Show this display")                   { puts opts; Process.exit!(0) }
+rest = opts.parse(ARGV)
+
+$xml_str = nil
+$ox_time = 0
+$no_time = 0
+$lx_time = 0
+
+# size is in Kbytes
+def create_file(filename, size)
+  head = %{<?xml version="1.0"?>
+<?ox version="1.0" mode="object" circular="no" xsd_date="no"?>
+<!DOCTYPE table PUBLIC "-//ox//DTD TABLE 1.0//EN" "http://www.ohler.com/DTDs/TestTable-1.0.dtd">
+<table>
+}
+  tail = %{</table>
+}
+  row = %{  <!-- row %08d element -->
+  <row id="%08d">
+    <cell id="A" type="Fixnum">1234</cell>
+    <cell id="B" type="String">A string.</cell>
+    <cell id="C" type="String">This is a longer string that stretches over a larger number of characters.</cell>
+    <cell id="D" type="Float">-12.345</cell>
+    <cell id="E" type="Date">2011-09-18 23:07:26 +0900</cell>
+    <cell id="F" type="Image"><![CDATA[xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00]]></cell>
+  </row>
+}
+  cnt = (size * 1000 - head.size - tail.size) / row.size
+  File.open(filename, "w") do |f|
+    f.write(head)
+    cnt.times do |i|
+      f.write(row % [i,i])
+    end
+    f.write(tail)
+  end
+end
+
+class OxSax < ::Ox::Sax
+  def start_element(name, attrs); end
+  def error(message, line, column); puts message; end
+end
+
+class OxAllSax < OxSax
+  def end_element(name);  end
+  def instruct(target, attrs); end
+  def doctype(value); end
+  def comment(value); end
+  def cdata(value); end
+  def text(value); end
+end
+
+unless defined?(::Nokogiri).nil?
+  class NoSax < Nokogiri::XML::SAX::Document
+    def start_element(name, attrs = []); end
+    def error(message); puts message; end
+    def warning(message); puts message; end
+  end
+  class NoAllSax < NoSax
+    def characters(text); end
+    def cdata_block(string); end
+    def comment(string); end
+    def end_document(); end
+    def end_element(name); end
+    def start_document(); end
+    def xmldecl(version, encoding, standalone); end
+  end
+end
+
+unless defined?(::LibXML).nil?
+  class LxSax
+    include LibXML::XML::SaxParser::Callbacks
+    def on_start_element(element, attributes); end
+  end
+  class LxAllSax < LxSax
+    def on_cdata_block(cdata); end
+    def on_characters(chars); end
+    def on_comment(msg); end
+    def on_end_document(); end
+    def on_end_element(element); end
+    def on_end_element_ns(name, prefix, uri); end
+    def on_error(msg); end
+    def on_external_subset(name, external_id, system_id); end
+    def on_has_external_subset(); end
+    def on_has_internal_subset(); end
+    def on_internal_subset(name, external_id, system_id); end
+    def on_is_standalone(); end
+    def on_processing_instruction(target, data); end
+    def on_reference(name); end
+    def on_start_document(); end
+    def on_start_element_ns(name, attributes, prefix, uri, namespaces); end
+  end
+end
+
+def perf_stringio()
+  start = Time.now
+  handler = $all_cbs ? OxAllSax.new() : OxSax.new()
+  $iter.times do
+    input = StringIO.new($xml_str)
+    Ox.sax_parse(handler, input)
+    input.close
+  end
+  $ox_time = Time.now - start
+  puts "StringIO SAX parsing #{$iter} times with Ox took #{$ox_time} seconds."
+
+  return if $ox_only
+
+  unless defined?(::Nokogiri).nil?
+    handler = Nokogiri::XML::SAX::Parser.new($all_cbs ? NoAllSax.new() : NoSax.new())
+    start = Time.now
+    $iter.times do
+      input = StringIO.new($xml_str)
+      handler.parse(input)
+      input.close
+    end
+    $no_time = Time.now - start
+    puts "StringIO SAX parsing #{$iter} times with Nokogiri took #{$no_time} seconds."
+  end
+
+  unless defined?(::LibXML).nil?
+    start = Time.now
+    $iter.times do
+      input = StringIO.new($xml_str)
+      parser = LibXML::XML::SaxParser.io(input)
+      parser.callbacks = $all_cbs ? LxAllSax.new() : LxSax.new()
+      parser.parse
+      input.close
+    end
+    $lx_time = Time.now - start
+    puts "StringIO SAX parsing #{$iter} times with LibXML took #{$lx_time} seconds."
+  end
+
+  puts "\n"
+  puts ">>> Ox is %0.1f faster than Nokogiri SAX parsing using StringIO." % [$no_time/$ox_time] unless defined?(::Nokogiri).nil?
+  puts ">>> Ox is %0.1f faster than LibXML SAX parsing using StringIO." % [$lx_time/$ox_time] unless defined?(::LibXML).nil?
+  puts "\n"
+end
+
+def perf_fileio()
+  puts "\n"
+  puts "A #{$filesize} KByte XML file was parsed #{$iter} for this test."
+  puts "\n"
+  start = Time.now
+  handler = $all_cbs ? OxAllSax.new() : OxSax.new()
+  $iter.times do
+    input = IO.open(IO.sysopen($filename))
+    Ox.sax_parse(handler, input)
+    input.close
+  end
+  $ox_time = Time.now - start
+  puts "File IO SAX parsing #{$iter} times with Ox took #{$ox_time} seconds."
+
+  return if $ox_only
+
+  unless defined?(::Nokogiri).nil?
+    handler = Nokogiri::XML::SAX::Parser.new($all_cbs ? NoAllSax.new() : NoSax.new())
+    start = Time.now
+    $iter.times do
+      input = IO.open(IO.sysopen($filename))
+      handler.parse(input)
+      input.close
+    end
+    $no_time = Time.now - start
+    puts "File IO SAX parsing #{$iter} times with Nokogiri took #{$no_time} seconds."
+  end
+
+  unless defined?(::LibXML).nil?
+    start = Time.now
+    $iter.times do
+      input = IO.open(IO.sysopen($filename))
+      parser = LibXML::XML::SaxParser.io(input)
+      parser.callbacks = $all_cbs ? LxAllSax.new() : LxSax.new()
+      parser.parse
+      input.close
+    end
+    $lx_time = Time.now - start
+    puts "File IO SAX parsing #{$iter} times with LibXML took #{$lx_time} seconds."
+  end
+
+  puts "\n"
+  puts ">>> Ox is %0.1f faster than Nokogiri SAX parsing using file IO." % [$no_time/$ox_time] unless defined?(::Nokogiri).nil?
+  puts ">>> Ox is %0.1f faster than LibXML SAX parsing using file IO." % [$lx_time/$ox_time] unless defined?(::LibXML).nil?
+  puts "\n"
+end
+
+if $filename.nil?
+  create_file('perf.xml', $filesize)
+  $filename = 'perf.xml'
+end
+$xml_str = File.read($filename)
+
+# perf_stringio()
+perf_fileio()

data/test/sax_test.rb

@@ -0,0 +1,381 @@
+#!/usr/bin/env ruby -wW1
+# encoding: UTF-8
+
+$: << '../lib'
+$: << '../ext'
+
+if __FILE__ == $0
+  while (i = ARGV.index('-I'))
+    x,path = ARGV.slice!(i, 2)
+    $: << path
+  end
+end
+
+require 'stringio'
+require 'test/unit'
+require 'optparse'
+require 'ox'
+
+opts = OptionParser.new
+opts.on("-h", "--help", "Show this display")                { puts opts; Process.exit!(0) }
+files = opts.parse(ARGV)
+
+class StartSax < ::Ox::Sax
+  attr_accessor :calls
+
+  def initialize()
+    @calls = []
+  end
+
+  def start_element(name, attrs)
+    @calls << [:start_element, name, attrs]
+  end
+end
+
+class AllSax < StartSax
+  def initialize()
+    super
+  end
+
+  def instruct(target, attrs)
+    @calls << [:instruct, target, attrs]
+  end
+
+  def doctype(value)
+    @calls << [:doctype, value]
+  end
+
+  def comment(value)
+    @calls << [:comment, value]
+  end
+
+  def cdata(value)
+    @calls << [:cdata, value]
+  end
+
+  def text(value)
+    @calls << [:text, value]
+  end
+
+  def end_element(name)
+    @calls << [:end_element, name]
+  end
+  
+  def error(message, line, column)
+    @calls << [:error, message, line, column]
+  end
+end
+
+class Func < ::Test::Unit::TestCase
+
+  def test_sax_io_pipe
+    handler = AllSax.new()
+    input,w = IO.pipe
+    w << %{<top/>}
+    w.close
+    Ox.sax_parse(handler, input)
+    assert_equal(handler.calls,
+                 [[:start_element, :top, nil],
+                  [:end_element, :top]])
+  end
+
+  def test_sax_io_file
+    handler = AllSax.new()
+    input = IO.open(IO.sysopen('basic.xml'))
+    Ox.sax_parse(handler, input)
+    assert_equal(handler.calls,
+                 [[:start_element, :top, nil],
+                  [:end_element, :top]])
+  end
+
+  def parse_compare(xml, expected, handler_class=AllSax)
+    handler = handler_class.new()
+    input = StringIO.new(xml)
+    Ox.sax_parse(handler, input)
+    assert_equal(expected, handler.calls)
+  end
+  
+  def test_sax_instruct_simple
+    parse_compare(%{<?xml?>}, [[:instruct, 'xml', nil]])
+  end
+
+  def test_sax_instruct_blank
+    parse_compare(%{<?xml?>}, [], StartSax)
+  end
+
+  def test_sax_instruct_attrs
+    parse_compare(%{<?xml version="1.0" encoding="UTF-8"?>},
+                  [[:instruct, 'xml', {:version => '1.0', :encoding => 'UTF-8'}]])
+  end
+
+  def test_sax_instruct_loose
+    parse_compare(%{<? xml
+version = "1.0"
+encoding = "UTF-8" ?>},
+                  [[:instruct, 'xml', {:version => '1.0', :encoding => 'UTF-8'}]])
+  end
+
+  def test_sax_element_simple
+    parse_compare(%{<top/>},
+                  [[:start_element, :top, nil],
+                   [:end_element, :top]])
+  end
+
+  def test_sax_element_attrs
+    parse_compare(%{<top x="57" y="42"/>}, 
+                  [[:start_element, :top, {:x => '57', :y => '42'}],
+                   [:end_element, :top]])
+  end
+
+  def test_sax_two_top
+    parse_compare(%{<top/><top/>},
+                  [[:start_element, :top, nil],
+                   [:end_element, :top],
+                   [:error, "invalid format, multiple top level elements", 1, 9],
+                   [:start_element, :top, nil],
+                   [:end_element, :top]])
+
+
+  end
+
+  def test_sax_nested1
+    parse_compare(%{<?xml version="1.0"?>
+<top>
+  <child>
+    <grandchild/>
+  </child>
+</top>
+},
+                  [[:instruct, 'xml', {:version => '1.0'}],
+                   [:start_element, :top, nil],
+                   [:start_element, :child, nil],
+                   [:start_element, :grandchild, nil],
+                   [:end_element, :grandchild],
+                   [:end_element, :child],
+                   [:end_element, :top],
+                  ])
+  end
+
+  def test_sax_nested1_tight
+    parse_compare(%{<?xml version="1.0"?><top><child><grandchild/></child></top>},
+                  [[:instruct, 'xml', {:version => '1.0'}],
+                   [:start_element, :top, nil],
+                   [:start_element, :child, nil],
+                   [:start_element, :grandchild, nil],
+                   [:end_element, :grandchild],
+                   [:end_element, :child],
+                   [:end_element, :top],
+                  ])
+  end
+
+  def test_sax_element_name_mismatch
+    parse_compare(%{<?xml version="1.0"?>
+<top>
+  <child>
+    <grandchild/>
+  </parent>
+</top>},
+                  [[:instruct, 'xml', {:version => '1.0'}],
+                   [:start_element, :top, nil],
+                   [:start_element, :child, nil],
+                   [:start_element, :grandchild, nil],
+                   [:end_element, :grandchild],
+                   [:error, "invalid format, element start and end names do not match", 5, 12]
+                  ])
+  end
+
+  def test_sax_nested
+    parse_compare(%{<?xml version="1.0"?>
+<top>
+  <child>
+    <grandchild/>
+  </child>
+  <child>
+    <grandchild/>
+    <grandchild/>
+  </child>
+</top>
+},
+                  [[:instruct, 'xml', {:version => '1.0'}],
+                   [:start_element, :top, nil],
+                   [:start_element, :child, nil],
+                   [:start_element, :grandchild, nil],
+                   [:end_element, :grandchild],
+                   [:end_element, :child],
+                   [:start_element, :child, nil],
+                   [:start_element, :grandchild, nil],
+                   [:end_element, :grandchild],
+                   [:start_element, :grandchild, nil],
+                   [:end_element, :grandchild],
+                   [:end_element, :child],
+                   [:end_element, :top],
+                  ])
+  end
+  def test_sax_element_no_term
+    parse_compare(%{
+<top>
+  <child/>
+},
+                  [[:start_element, :top, nil],
+                   [:start_element, :child, nil],
+                   [:end_element, :child],
+                   [:error, "invalid format, element not terminated", 4, 1]
+                  ])
+  end
+
+  def test_sax_text
+    parse_compare(%{<top>This is some text.</top>},
+                  [[:start_element, :top, nil],
+                   [:text, "This is some text."],
+                   [:end_element, :top]
+                  ])
+  end
+
+  def test_sax_text_no_term
+    parse_compare(%{<top>This is some text.},
+                  [[:start_element, :top, nil],
+                   [:error, "invalid format, text terminated unexpectedly", 1, 24],
+                  ])
+  end
+  # TBD invalid chacters in text
+
+  def test_sax_doctype
+    parse_compare(%{<?xml version="1.0"?>
+<!DOCTYPE top PUBLIC "top.dtd">
+<top/>
+},
+                  [[:instruct, 'xml', {:version => '1.0'}],
+                   [:doctype, ' top PUBLIC "top.dtd"'],
+                   [:start_element, :top, nil],
+                   [:end_element, :top]])
+  end
+
+  def test_sax_doctype_bad_order
+    parse_compare(%{<?xml version="1.0"?>
+<top/>
+<!DOCTYPE top PUBLIC "top.dtd">
+},
+                  [[:instruct, 'xml', {:version => '1.0'}],
+                   [:start_element, :top, nil],
+                   [:end_element, :top],
+                   [:error, "invalid format, DOCTYPE can not come after an element", 3, 11],
+                   [:doctype, ' top PUBLIC "top.dtd"']])
+  end
+  
+  def test_sax_instruct_bad_order
+    parse_compare(%{
+<!DOCTYPE top PUBLIC "top.dtd">
+<?xml version="1.0"?>
+<top/>
+},
+                  [[:doctype, " top PUBLIC \"top.dtd\""],
+                   [:error, "invalid format, instruction must come before elements", 3, 3],
+                   [:instruct, "xml", {:version => "1.0"}],
+                   [:start_element, :top, nil],
+                   [:end_element, :top]])
+  end
+  
+  def test_sax_comment
+    parse_compare(%{<?xml version="1.0"?>
+<!--First comment.-->
+<top>Before<!--Nested comment.-->After</top>
+},
+                  [[:instruct, 'xml', {:version => '1.0'}],
+                   [:comment, 'First comment.'],
+                   [:start_element, :top, nil],
+                   [:text, 'Before'],
+                   [:comment, 'Nested comment.'],
+                   [:text, 'After'],
+                   [:end_element, :top]])
+  end
+
+  def test_sax_comment_no_term
+    parse_compare(%{<?xml version="1.0"?>
+<!--First comment.--
+<top/>
+},
+                  [[:instruct, 'xml', {:version => '1.0'}],
+                   [:error, "invalid format, comment terminated unexpectedly", 3, 1], # continue on
+                   [:comment, 'First comment.'],
+                   [:start_element, :top, nil],
+                   [:end_element, :top]])
+  end
+
+  def test_sax_cdata
+    parse_compare(%{<?xml version="1.0"?>
+<top>
+  <![CDATA[This is CDATA.]]>
+</top>
+},
+                  [[:instruct, 'xml', {:version => '1.0'}],
+                   [:start_element, :top, nil],
+                   [:cdata, 'This is CDATA.'],
+                   [:end_element, :top]])
+  end
+
+  def test_sax_cdata_no_term
+    parse_compare(%{<?xml version="1.0"?>
+<top>
+  <![CDATA[This is CDATA.]]
+</top>
+},
+                  [[:instruct, 'xml', {:version => '1.0'}],
+                   [:start_element, :top, nil],
+                   [:error, "invalid format, cdata terminated unexpectedly", 5, 1]])
+  end
+  
+
+  def test_sax_mixed
+    parse_compare(%{<?xml version="1.0"?>
+<?ox version="1.0" mode="object" circular="no" xsd_date="no"?>
+<!DOCTYPE table PUBLIC "-//ox//DTD TABLE 1.0//EN" "http://www.ohler.com/DTDs/TestTable-1.0.dtd">
+<table>
+  <row id="00004">
+    <cell id="A" type="Fixnum">1234</cell>
+    <cell id="B" type="String">A string.</cell>
+    <cell id="C" type="String">This is a longer string that stretches over a larger number of characters.</cell>
+    <cell id="D" type="Float">-12.345</cell>
+    <cell id="E" type="Date">2011-09-18 23:07:26 +0900</cell>
+    <cell id="F" type="Image"><![CDATA[xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00]]></cell>
+  </row>
+</table>
+},
+                  [[:instruct, 'xml', {:version => '1.0'}],
+                   [:instruct, "ox", {:version=>"1.0", :mode=>"object", :circular=>"no", :xsd_date=>"no"}],
+                   [:doctype, " table PUBLIC \"-//ox//DTD TABLE 1.0//EN\" \"http://www.ohler.com/DTDs/TestTable-1.0.dtd\""],
+                   [:start_element, :table, nil],
+                   [:start_element, :row, {:id=>"00004"}],
+                   [:start_element, :cell, {:id=>"A", :type=>"Fixnum"}],
+                   [:text, "1234"],
+                   [:end_element, :cell],
+                   [:start_element, :cell, {:id=>"B", :type=>"String"}],
+                   [:text, "A string."],
+                   [:end_element, :cell],
+                   [:start_element, :cell, {:id=>"C", :type=>"String"}],
+                   [:text, "This is a longer string that stretches over a larger number of characters."],
+                   [:end_element, :cell],
+                   [:start_element, :cell, {:id=>"D", :type=>"Float"}],
+                   [:text, "-12.345"],
+                   [:end_element, :cell],
+                   [:start_element, :cell, {:id=>"E", :type=>"Date"}],
+                   [:text, "2011-09-18 23:07:26 +0900"],
+                   [:end_element, :cell],
+                   [:start_element, :cell, {:id=>"F", :type=>"Image"}],
+                   [:cdata, "xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00xx00"],
+                   [:end_element, :cell],
+                   [:end_element, :row],
+                   [:end_element, :table]])
+  end
+
+  def test_sax_encoding
+    parse_compare(%{<?xml version="1.0" encoding="UTF-8"?>
+<top>ピーター</top>
+},
+                  [[:instruct, 'xml', {:version => '1.0', :encoding => 'UTF-8'}],
+                   [:start_element, :top, nil],
+                   [:text, 'ピーター'],
+                   [:end_element, :top]])
+  end
+
+end
+