sax-machine 0.1.0 → 1.3.2

data/lib/sax-machine/handlers/sax_abstract_handler.rb

@@ -0,0 +1,199 @@
+require 'time'
+
+module SAXMachine
+  module SAXAbstractHandler
+    NO_BUFFER = :no_buffer
+
+    class StackNode < Struct.new(:object, :config, :buffer)
+      def initialize(object, config = nil, buffer = NO_BUFFER)
+        self.object = object
+        self.config = config
+        self.buffer = buffer
+      end
+    end
+
+    def sax_parse(xml_input)
+      raise NotImplementedError
+    end
+
+    def _initialize(object, on_error = nil, on_warning = nil)
+      @stack = [ StackNode.new(object) ]
+      @parsed_configs = {}
+      @on_error = on_error
+      @on_warning = on_warning
+    end
+
+    def _characters(data)
+      node = stack.last
+
+      if node.buffer == NO_BUFFER
+        node.buffer = data.dup
+      else
+        node.buffer << data
+      end
+    end
+
+    def _start_element(name, attrs = [])
+      name   = normalize_name(name)
+      node   = stack.last
+      object = node.object
+
+      sax_config = sax_config_for(object)
+
+      if sax_config
+        attrs = Hash[attrs]
+
+        if collection_config = sax_config.collection_config(name, attrs)
+          object     = collection_config.data_class.new
+          sax_config = sax_config_for(object)
+
+          stack.push(StackNode.new(object, collection_config))
+
+          set_attributes_on(object, attrs)
+        end
+
+        sax_config.element_configs_for_attribute(name, attrs).each do |ec|
+          unless parsed_config?(object, ec)
+            object.send(ec.setter, ec.value_from_attrs(attrs))
+            mark_as_parsed(object, ec)
+          end
+        end
+
+        if !collection_config && element_config = sax_config.element_config_for_tag(name, attrs)
+          new_object =
+            case element_config.data_class.to_s
+            when "Integer" then 0
+            when "Float"   then 0.0
+            when "Symbol"  then nil
+            when "Time"    then Time.at(0)
+            when ""        then object
+            else
+              element_config.data_class.new
+            end
+
+          stack.push(StackNode.new(new_object, element_config))
+
+          set_attributes_on(new_object, attrs)
+        end
+      end
+    end
+
+    def _end_element(name)
+      name = normalize_name(name)
+
+      start_tag = stack[-2]
+      close_tag = stack[-1]
+
+      return unless start_tag && close_tag
+
+      object  = start_tag.object
+      element = close_tag.object
+      config  = close_tag.config
+      value   = close_tag.buffer
+
+      return unless config.name == name
+
+      unless parsed_config?(object, config)
+        if (element_value_config = element_values_for(config))
+          element_value_config.each { |evc| element.send(evc.setter, value) }
+        end
+
+        if config.respond_to?(:accessor)
+          subconfig = sax_config_for(element)
+
+          if econf = subconfig.element_config_for_tag(name, [])
+            element.send(econf.setter, value) unless econf.value_configured?
+          end
+
+          object.send(config.accessor) << element
+        else
+          value = data_class_value(config.data_class, value) || element
+          object.send(config.setter, value) if value != NO_BUFFER
+          mark_as_parsed(object, config)
+        end
+
+        # try to set the ancestor
+        if (sax_config = sax_config_for(element))
+          sax_config.ancestors.each do |ancestor|
+            element.send(ancestor.setter, object)
+          end
+        end
+      end
+
+      stack.pop
+    end
+
+    def _error(string)
+      if @on_error
+        @on_error.call(string)
+      end
+    end
+
+    def _warning(string)
+      if @on_warning
+        @on_warning.call(string)
+      end
+    end
+
+    private
+
+    def mark_as_parsed(object, element_config)
+      unless element_config.collection?
+        @parsed_configs[[object.object_id, element_config.object_id]] = true
+      end
+    end
+
+    def parsed_config?(object, element_config)
+      @parsed_configs[[object.object_id, element_config.object_id]]
+    end
+
+    def sax_config_for(object)
+      if object.class.respond_to?(:sax_config)
+        object.class.sax_config
+      end
+    end
+
+    def element_values_for(config)
+      if config.data_class.respond_to?(:sax_config)
+        config.data_class.sax_config.element_values_for_element
+      end
+    end
+
+    def normalize_name(name)
+      name.to_s.tr("-", "_")
+    end
+
+    def set_attributes_on(object, attributes)
+      config = sax_config_for(object)
+
+      if config
+        config.attribute_configs_for_element(attributes).each do |ac|
+          value = data_class_value(ac.data_class, ac.value_from_attrs(attributes))
+          object.send(ac.setter, value)
+        end
+      end
+    end
+
+    def data_class_value(data_class, value)
+      case data_class.to_s
+      when "String"  then value != NO_BUFFER ? value.to_s : value
+      when "Integer" then value != NO_BUFFER ? value.to_i : value
+      when "Float"   then value != NO_BUFFER ? value.to_s.gsub(",",".").to_f : value
+      when "Symbol"  then
+        if value != NO_BUFFER
+          value.to_s.empty? ? nil : value.to_s.downcase.to_sym
+        else
+          value
+        end
+      # Assumes that time elements will be string-based and are not
+      # something else, e.g. seconds since epoch
+      when "Time"    then value != NO_BUFFER ? Time.parse(value.to_s) : value
+      when ""        then value
+      end
+    end
+
+    def stack
+      @stack
+    end
+  end
+end

data/lib/sax-machine/handlers/sax_nokogiri_handler.rb

@@ -0,0 +1,23 @@
+require 'sax-machine/handlers/sax_abstract_handler'
+require 'nokogiri'
+
+module SAXMachine
+  class SAXNokogiriHandler < Nokogiri::XML::SAX::Document
+    include SAXAbstractHandler
+
+    def sax_parse(xml_input)
+      parser = Nokogiri::XML::SAX::Parser.new(self)
+      parser.parse(xml_input) do |ctx|
+        ctx.replace_entities = true
+      end
+    end
+
+    alias_method :initialize, :_initialize
+    alias_method :characters, :_characters
+    alias_method :cdata_block, :_characters
+    alias_method :start_element, :_start_element
+    alias_method :end_element, :_end_element
+    alias_method :error, :_error
+    alias_method :warning, :_warning
+  end
+end

data/lib/sax-machine/handlers/sax_oga_handler.rb

@@ -0,0 +1,39 @@
+require 'sax-machine/handlers/sax_abstract_handler'
+require 'oga'
+
+module SAXMachine
+  class SAXOgaHandler
+    include SAXAbstractHandler
+
+    def initialize(*args)
+      _initialize(*args)
+    end
+
+    def sax_parse(xml_input)
+      Oga.sax_parse_xml(self, xml_input)
+    rescue LL::ParserError => e
+      on_error(e.message)
+    end
+
+    def on_element(namespace, name, attrs)
+      _start_element(node_name(namespace, name), attrs)
+    end
+
+    def after_element(namespace, name)
+      _end_element(node_name(namespace, name))
+    end
+
+    def on_error(*args)
+      _error(args.join(" "))
+    end
+
+    alias_method :on_text, :_characters
+    alias_method :on_cdata, :_characters
+
+    private
+
+    def node_name(namespace, name)
+      namespace ? "#{namespace}:#{name}" : name
+    end
+  end
+end

data/lib/sax-machine/handlers/sax_ox_handler.rb

@@ -0,0 +1,56 @@
+require 'sax-machine/handlers/sax_abstract_handler'
+require 'ox'
+
+module SAXMachine
+  class SAXOxHandler < Ox::Sax
+    include SAXAbstractHandler
+
+    def initialize(*args)
+      _initialize(*args)
+      _reset_element
+    end
+
+    def sax_parse(xml_input)
+      # Ox requires input to be streamable
+      xml_input = StringIO.new(xml_input) if xml_input.is_a?(String)
+
+      Ox.sax_parse(self, xml_input,
+        symbolize: false,
+        convert_special: true,
+        skip: :skip_return,
+      )
+    end
+
+    def attr(name, str)
+      @attrs[name] = str
+    end
+
+    def attrs_done
+      _start_element(@element, @attrs)
+      _reset_element
+    end
+
+    def start_element(name)
+      @element = name
+    end
+
+    def text(value)
+      _characters(value) if value && !value.empty?
+    end
+
+    alias_method :cdata, :text
+
+    def error(message, line, column)
+      _error("#{message} on line #{line} column #{column}")
+    end
+
+    alias_method :end_element, :_end_element
+
+    private
+
+    def _reset_element
+      @attrs = {}
+      @element = ""
+    end
+  end
+end

data/lib/sax-machine/sax_config.rb

@@ -1,17 +1,16 @@
-require "sax-machine/sax_attribute_config"
-require "sax-machine/sax_element_value_config"
-require "sax-machine/sax_element_config"
-require "sax-machine/sax_collection_config"
-require "sax-machine/sax_ancestor_config"
+require "sax-machine/config/sax_element_value"
+require "sax-machine/config/sax_attribute"
+require "sax-machine/config/sax_element"
+require "sax-machine/config/sax_collection"
+require "sax-machine/config/sax_ancestor"
 
 module SAXMachine
   class SAXConfig
-
     attr_accessor :top_level_elements, :top_level_attributes, :top_level_element_value, :collection_elements, :ancestors
 
     def initialize
       # Default value is an empty array
-      @top_level_elements  = Hash.new { |hash, key| hash[key] = [] }
+      @top_level_elements = Hash.new { |hash, key| hash[key] = [] }
       @top_level_attributes  = []
       @top_level_element_value = []
       @collection_elements = Hash.new { |hash, key| hash[key] = [] }
@@ -19,11 +18,12 @@ module SAXMachine
     end
 
     def columns
-      @top_level_elements.map {|name, ecs| ecs }.flatten
+      @top_level_elements.map { |_, ecs| ecs }.flatten
     end
 
     def initialize_copy(sax_config)
       super
+
       @top_level_elements = sax_config.top_level_elements.clone
       @top_level_attributes = sax_config.top_level_attributes.clone
       @top_level_element_value = sax_config.top_level_element_value.clone
@@ -64,11 +64,15 @@ module SAXMachine
     end
 
     def element_configs_for_attribute(name, attrs)
+      return [] unless @top_level_elements.has_key?(name.to_s)
+
       @top_level_elements[name.to_s].select { |ec| ec.has_value_and_attrs_match?(attrs) }
     end
 
     def element_config_for_tag(name, attrs)
+      return unless @top_level_elements.has_key?(name.to_s)
+
       @top_level_elements[name.to_s].detect { |ec| ec.attrs_match?(attrs) }
     end
   end
-end
+end

data/lib/sax-machine/sax_configure.rb

@@ -1,5 +1,4 @@
 module SAXMachine
-
   def self.configure(clazz)
     extended_clazz = Class.new(clazz)
     extended_clazz.send(:include, SAXMachine)
@@ -15,14 +14,12 @@ module SAXMachine
     clazz.extend LightWeightSaxMachine
     clazz.sax_config = extended_clazz.sax_config
 
-    (class << clazz;self;end).send(:define_method, :parse) do |xml_text|
-      extended_clazz.parse(xml_text)
+    (class << clazz;self;end).send(:define_method, :parse) do |xml_input|
+      extended_clazz.parse(xml_input)
     end
-
   end
 
   module LightWeightSaxMachine
-
     attr_writer :sax_config
 
     def sax_config
@@ -32,7 +29,5 @@ module SAXMachine
     def inherited(subclass)
       subclass.sax_config.send(:initialize_copy, self.sax_config)
     end
-
   end
-
-end
+end

data/lib/sax-machine/sax_document.rb

@@ -1,50 +1,96 @@
-require "nokogiri"
-
 module SAXMachine
-
   def self.included(base)
-    base.extend ClassMethods
+    base.send(:include, InstanceMethods)
+    base.extend(ClassMethods)
   end
 
-  def parse(xml_text, on_error = nil, on_warning = nil)
-    sax_handler = SAXHandler.new(self, on_error, on_warning)
-    parser = Nokogiri::XML::SAX::Parser.new(sax_handler)
-    parser.parse(xml_text)
+  def parse(xml_input, on_error = nil, on_warning = nil)
+    handler_klass = SAXMachine.const_get("SAX#{SAXMachine.handler.capitalize}Handler")
+
+    handler = handler_klass.new(self, on_error, on_warning)
+    handler.sax_parse(xml_input)
+
     self
   end
 
-  module ClassMethods
+  module InstanceMethods
+    def initialize(attributes = {})
+      attributes.each do |name, value|
+        send("#{name}=", value)
+      end
 
+      self.class.sax_config.top_level_elements.each do |_, configs|
+        configs.each do |config|
+          next if config.default.nil?
+          next unless send(config.as).nil?
+
+          send(config.setter, config.default)
+        end
+      end
+    end
+  end
+
+  module ClassMethods
     def inherited(subclass)
       subclass.sax_config.send(:initialize_copy, self.sax_config)
     end
 
-    def parse(xml_text, on_error = nil, on_warning = nil)
-      new.parse(xml_text, on_error, on_warning)
+    def parse(*args)
+      new.parse(*args)
     end
 
-    def element(name, options = {})
+    def element(name, options = {}, &block)
       real_name = (options[:as] ||= name).to_s
       sax_config.add_top_level_element(name, options)
-      create_attr real_name
+      create_attr(real_name, &block)
     end
 
-    def attribute(name, options = {})
+    def attribute(name, options = {}, &block)
       real_name = (options[:as] ||= name).to_s
-      sax_config.add_top_level_attribute(self.class.to_s, options.merge(:name => name))
-      create_attr real_name
+      sax_config.add_top_level_attribute(self.class.to_s, options.merge(name: name))
+      create_attr(real_name, &block)
     end
 
-    def value(name, options = {})
+    def value(name, options = {}, &block)
       real_name = (options[:as] ||= name).to_s
-      sax_config.add_top_level_element_value(self.class.to_s, options.merge(:name => name))
-      create_attr real_name
+      sax_config.add_top_level_element_value(self.class.to_s, options.merge(name: name))
+      create_attr(real_name, &block)
     end
 
-    def ancestor(name, options = {})
+    def ancestor(name, options = {}, &block)
       real_name = (options[:as] ||= name).to_s
       sax_config.add_ancestor(name, options)
-      create_attr(real_name)
+      create_attr(real_name, &block)
+    end
+
+    def elements(name, options = {}, &block)
+      real_name = (options[:as] ||= name).to_s
+
+      if options[:class]
+        sax_config.add_collection_element(name, options)
+      else
+        if block_given?
+          define_method("add_#{real_name}") do |value|
+            send(real_name).send(:<<, instance_exec(value, &block))
+          end
+        else
+          define_method("add_#{real_name}") do |value|
+            send(real_name).send(:<<, value)
+          end
+        end
+
+        sax_config.add_top_level_element(name, options.merge(collection: true))
+      end
+
+      if !method_defined?(real_name)
+        class_eval <<-SRC
+          def #{real_name}
+            @#{real_name} ||= []
+          end
+        SRC
+      end
+
+      attr_writer(options[:as]) unless method_defined?("#{options[:as]}=")
     end
 
     def columns
@@ -67,30 +113,6 @@ module SAXMachine
       columns.map { |e| e.column }
     end
 
-    def elements(name, options = {})
-      options[:as] ||= name
-      if options[:class]
-        sax_config.add_collection_element(name, options)
-      else
-        class_eval <<-SRC
-          def add_#{options[:as]}(value)
-            #{options[:as]} << value
-          end
-        SRC
-        sax_config.add_top_level_element(name, options.merge(:collection => true))
-      end
-
-      if !method_defined?(options[:as].to_s)
-        class_eval <<-SRC
-          def #{options[:as]}
-            @#{options[:as]} ||= []
-          end
-        SRC
-      end
-
-      attr_writer options[:as] unless method_defined?("#{options[:as]}=")
-    end
-
     def sax_config
       @sax_config ||= SAXConfig.new
     end
@@ -98,10 +120,18 @@ module SAXMachine
     # we only want to insert the getter and setter if they haven't defined it from elsewhere.
     # this is how we allow custom parsing behavior. So you could define the setter
     # and have it parse the string into a date or whatever.
-    def create_attr real_name
-      attr_reader real_name unless method_defined?(real_name)
-      attr_writer real_name unless method_defined?("#{real_name}=")
+    def create_attr(real_name, &block)
+      attr_reader(real_name) unless method_defined?(real_name)
+
+      if !method_defined?("#{real_name}=")
+        if block_given?
+          define_method("#{real_name}=") do |value|
+            instance_variable_set("@#{real_name}", instance_exec(value, &block))
+          end
+        else
+          attr_writer(real_name)
+        end
+      end
     end
   end
-
 end

data/lib/sax-machine/version.rb

@@ -0,0 +1,3 @@
+module SAXMachine
+  VERSION = "1.3.2"
+end

data/lib/sax-machine.rb

@@ -1,12 +1,31 @@
-require "rubygems"
-
-$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__))) unless $LOAD_PATH.include?(File.expand_path(File.dirname(__FILE__)))
-
+require "sax-machine/version"
 require "sax-machine/sax_document"
 require "sax-machine/sax_configure"
-require "sax-machine/sax_handler"
 require "sax-machine/sax_config"
 
 module SAXMachine
-  VERSION = "0.0.16"
-end
+  def self.handler
+    @@handler ||= nil
+  end
+
+  def self.handler=(handler)
+    if handler
+      require "sax-machine/handlers/sax_#{handler}_handler"
+      @@handler = handler
+    end
+  end
+end
+
+# Try handlers
+[:ox, :oga].each do |handler|
+  begin
+    SAXMachine.handler = handler
+    break
+  rescue LoadError
+  end
+end
+
+# Still no handler, use Nokogiri
+if SAXMachine.handler.nil?
+  SAXMachine.handler = :nokogiri
+end

data/sax-machine.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path("../lib/sax-machine/version", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name      = "sax-machine"
+  s.version   = SAXMachine::VERSION
+
+  s.authors   = ["Paul Dix", "Julien Kirch", "Ezekiel Templin", "Dmitry Krasnoukhov"]
+  s.email     = %q{paul@pauldix.net}
+  s.homepage  = %q{http://github.com/pauldix/sax-machine}
+  s.summary   = %q{Declarative SAX Parsing with Nokogiri, Ox or Oga}
+  s.license   = %q{MIT}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ["lib"]
+  s.platform      = Gem::Platform::RUBY
+
+  s.add_development_dependency "rspec", "~> 3.0"
+end

data/spec/fixtures/atom-content.html

@@ -0,0 +1,15 @@
+
+<div xmlns="http://www.w3.org/1999/xhtml"><p>In my previous <a href="http://www.pauldix.net/2008/08/serializing-dat.html">post about the speed of serializing data</a>, I concluded that Marshal was the quickest way to get things done. So I set about using Marshal to store some data in an ActiveRecord object. Things worked great at first, but on some test data I got this error: marshal data too short. Luckily, <a href="http://www.brynary.com/">Bryan Helmkamp</a> had helpfully pointed out that there were sometimes problems with storing marshaled data in the database. He said it was best to base64 encode the marshal dump before storing.</p>
+
+<p>I was curious why it was working on some things and not others. It turns out that some types of data being marshaled were causing the error to pop up. Here's the test data I used in my specs:</p>
+<pre>{ :foo =&gt; 3, :bar =&gt; 2 } # hash with symbols for keys and integer values<br />[3, 2.1, 4, 8]&nbsp; &nbsp;&nbsp; &nbsp;&nbsp; &nbsp;&nbsp; # array with integer and float values</pre>
+<p>Everything worked when I switched the array values to all integers so it seems that floats were causing the problem. However, in the interest of keeping everything working regardless of data types, I base64 encoded before going into the database and decoded on the way out.</p>
+
+<p>I also ran the benchmarks again to determine what impact this would have on speed. Here are the results for 100 iterations on a 10k element array and a 10k element hash with and without base64 encode/decode:</p>
+<pre>&nbsp; &nbsp;&nbsp; &nbsp;&nbsp; &nbsp;&nbsp; &nbsp;&nbsp; &nbsp; user&nbsp; &nbsp;&nbsp; &nbsp; system&nbsp; &nbsp;&nbsp; total&nbsp; &nbsp;&nbsp; &nbsp; real<br />array marshal&nbsp; 0.200000&nbsp; &nbsp;0.010000&nbsp; &nbsp;0.210000 (&nbsp; 0.214018) (without Base64)<br />array marshal&nbsp; 0.220000&nbsp; &nbsp;0.010000&nbsp; &nbsp;0.230000 (&nbsp; 0.250260)<br /><br />hash marshal&nbsp; &nbsp;1.830000&nbsp; &nbsp;0.040000&nbsp; &nbsp;1.870000 (&nbsp; 1.892874) (without Base64)<br />hash marshal&nbsp; &nbsp;2.040000&nbsp; &nbsp;0.100000&nbsp; &nbsp;2.140000 (&nbsp; 2.170405)</pre>
+<p>As you can see the difference in speed is pretty negligible. I assume that the error has to do with AR cleaning the stuff that gets inserted into the database, but I'm not really sure. In the end it's just easier to use Base64.encode64 when serializing data into a text field in ActiveRecord using Marshal.</p>
+
+<p>I've also read people posting about this error when using the database session store. I can only assume that it's because they were trying to store either way too much data in their session (too much for a regular text field) or they were storing float values or some other data type that would cause this to pop up. Hopefully this helps.</p></div>
+<div class="feedflare">
+<a href="http://feeds.feedburner.com/~f/PaulDixExplainsNothing?a=rWfWO"><img src="http://feeds.feedburner.com/~f/PaulDixExplainsNothing?i=rWfWO" border="0"></img></a> <a href="http://feeds.feedburner.com/~f/PaulDixExplainsNothing?a=RaCqo"><img src="http://feeds.feedburner.com/~f/PaulDixExplainsNothing?i=RaCqo" border="0"></img></a> <a href="http://feeds.feedburner.com/~f/PaulDixExplainsNothing?a=1CBLo"><img src="http://feeds.feedburner.com/~f/PaulDixExplainsNothing?i=1CBLo" border="0"></img></a>
+</div><img src="http://feeds.feedburner.com/~r/PaulDixExplainsNothing/~4/383536354" height="1" width="1"/>