saxerator 0.0.1

data/.gitignore

@@ -0,0 +1,8 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.DS_Store
+.idea/
+coverage/
+

data/.rvmrc

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional.
+environment_id="1.9.3@saxerator"
+
+#
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+#
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments" \
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+
+  if [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]]
+  then
+    . "${rvm_path:-$HOME/.rvm}/hooks/after_use"
+  fi
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  if ! rvm --create  "$environment_id"
+  then
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  fi
+fi
+
+if [[ $- == *i* ]] # check for interactive shells
+then
+    echo "Using: $(tput setaf 2)$GEM_HOME$(tput sgr0)" # show the user the ruby and gemset they are using in green
+else 
+	echo "Using: $GEM_HOME" # don't use colors in interactive shells
+fi

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in saxerator.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,13 @@
+# More info at https://github.com/guard/guard#readme
+
+guard :bundler do
+  watch('Gemfile')
+  watch(/^saxerator\.gemspec$/)
+end
+
+guard :rspec, :cli => '--color --format doc' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch(%r{^spec/fixtures/.+\.xml$}) { :spec }
+  watch('spec/spec_helper.rb')  { :spec }
+end

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (c) 2012 Bradley Schaefer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+Saxerator
+=========
+
+Saxerator is a SAX-based xml parser designed for parsing very large files into manageable chunks. Rather than
+dealing directly with SAX callback methods, Saxerator gives you Enumerable access to chunks of an xml document.
+This approach is ideal for large xml files that represent a collection of elements.
+
+Examples
+--------
+
+```ruby
+Saxerator.parser(File.new("rss.xml")).for_tag(:item).each do |item|
+    puts "#{item['title']}: #{item['author']}"
+end
+```
+
+Compatibility
+-------------
+This library is known to work with the following rubies:
+
+* MRI 1.9.3-p125
+* MRI 1.9.2-p318
+* JRuby 1.6.7 (with JRUBY_OPTS=--1.9)
+
+Saxerator may work with other versions with support for Fiber.
+
+Known incompatible rubies:
+
+* MRI 1.9.2-p290 (Fiber segfaults)
+
+FAQ
+---
+Why the name 'Saxerator'?
+
+  > It's a combination of SAX + Enumerator.
+
+Why use Saxerator over regular SAX parsing?
+
+  > Much of the SAX parsing code I've written over the years has fallen into a pattern that Saxerator encapsulates:
+  > marshall a chunk of an XML document into an object, operate on that object, then move on to the
+  > next chunk. Saxerator alleviates the pain of marshalling and allows you to focus solely on operating on the
+  > document chunk.
+
+Why not DOM parsing?
+
+  > DOM parsers load the entire document into memory. Saxerator only holds a single chunk in memory at a time. If your
+  > document is very large, this can be an important consideration.
+
+### Acknowledgements ###
+Saxerator was inspired by [Nori](https://github.com/rubiii/nori) and [Gregory Brown](http://majesticseacreature.com/)'s
+[Practicing Ruby](http://practicingruby.com/)

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/benchmark/benchmark.rb

@@ -0,0 +1,18 @@
+$:.push File.expand_path('../../lib', __FILE__)
+require 'saxerator'
+require 'benchmark'
+
+file = ARGV.shift
+if !File.exists?(file)
+  puts "Cannot find file #{file}"
+  exit 1
+end
+file = File.new(file)
+
+tag = ARGV.shift || :artist
+
+count = 0
+Benchmark.bm do |x|
+  x.report { Saxerator.parser(file).for_tag(tag).each { count = count + 1 } }
+end
+puts "#{count} #{tag} elements parsed"

data/benchmark/generate_sample_file.rb

@@ -0,0 +1,32 @@
+require 'ipsum'
+
+filename = ARGV.shift
+
+unless filename
+  puts "Syntax: ruby #{__FILE__} <filename>"
+  exit 1
+end
+
+num_records = ARGV.shift || 100000
+num_records = num_records.to_i
+
+element = <<-eos
+  <artist id="%i">
+    <name>Rock Star %i</name>
+    <active>true</active>
+    <description>%s</description>
+    <genre>Rock,Metal</genre>
+  </artist>
+eos
+
+puts "Writing #{num_records} sample records to #{filename}, this will take a while..."
+
+File.open(filename, 'w') do |f|
+  f.puts '<?xml version="1.0" encoding="UTF-8"?>'
+  f.puts '<artists>'
+  num_records.times do |count|
+    f.puts(element % [count, count, 5.sentences])
+  end
+  f.puts '</artists>'
+end
+puts "DONE!"

data/lib/saxerator.rb

@@ -0,0 +1,12 @@
+require "saxerator/version"
+require 'saxerator/configuration'
+require 'saxerator/xml_node'
+require 'saxerator/parser/nokogiri'
+
+module Saxerator
+  extend self
+
+  def parser(xml)
+    Saxerator::Configuration.new(xml)
+  end
+end

data/lib/saxerator/configuration.rb

@@ -0,0 +1,14 @@
+module Saxerator
+  class Configuration
+    attr_reader :source
+
+    def initialize(source)
+      @source = source
+    end
+
+    def for_tag(tag)
+      tag = tag.to_s
+      Saxerator::Parser::Nokogiri.new(self, source, tag)
+    end
+  end
+end

data/lib/saxerator/parser/nokogiri.rb

@@ -0,0 +1,63 @@
+require 'nokogiri'
+require 'fiber'
+
+module Saxerator
+  module Parser
+    class Nokogiri
+      include Enumerable
+
+      def initialize(config, source, tag)
+        @config = config
+        @source = source
+        @tag = tag
+      end
+
+      def each(&block)
+        begin
+          fiber = Fiber.new do
+            document = Document.new(@config, @tag)
+            parser = ::Nokogiri::XML::SAX::Parser.new document
+            parser.parse(@source)
+          end
+          while fiber.alive? do
+            result = fiber.resume
+            yield(result) unless result.nil?
+          end
+        rescue FiberError
+        end
+      end
+
+      class Document < ::Nokogiri::XML::SAX::Document
+        attr_accessor :stack
+
+        def initialize(config, tag)
+          @config = config
+          @tag = tag
+          @stack = []
+        end
+
+        def start_element(name, attrs = [])
+          if stack.size > 0 || name == @tag
+            stack.push Saxerator::XmlNode.new(@config, name, Hash[*attrs.flatten])
+          end
+        end
+
+        def end_element(name)
+          if stack.size > 1
+            last = stack.pop
+            stack.last.add_node last
+          elsif stack.size == 1
+            Fiber.yield(stack.pop.to_hash)
+          end
+        end
+
+        def characters(string)
+          stack.last.add_node(string) unless string.strip.length == 0 || stack.empty?
+        end
+
+        alias cdata_block characters
+
+      end
+    end
+  end
+end

data/lib/saxerator/string_with_attributes.rb

@@ -0,0 +1,9 @@
+module Saxerator
+  class StringWithAttributes < String
+    attr_accessor :attributes
+
+    def attribrutes
+      @attributes ||= {}
+    end
+  end
+end

data/lib/saxerator/version.rb

@@ -0,0 +1,3 @@
+module Saxerator
+  VERSION = "0.0.1"
+end

data/lib/saxerator/xml_node.rb

@@ -0,0 +1,38 @@
+module Saxerator
+  class XmlNode
+    attr_accessor :name, :attributes, :children, :type
+
+    def initialize(config, name, attributes)
+      @config = config
+      self.name = name
+      self.attributes = attributes
+      self.children = []
+      @text = false
+    end
+
+    def add_node(node)
+      @text = true if node.is_a? String
+      children << node
+    end
+
+    def to_hash
+      if @text
+        return children.join(' ')
+      else
+        out = {}
+        @children.each do |child|
+          name = child.name
+          if out[name]
+            if !out[name].is_a?(Array)
+              out[name] = [out[name]]
+            end
+            out[name] << child.to_hash
+          else
+            out[name] = child.to_hash
+          end
+        end
+        out
+      end
+    end
+  end
+end

data/saxerator.gemspec

@@ -0,0 +1,44 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
+require 'saxerator/version'
+require 'rubygems/package_task'
+
+Gem::Specification.new do |s|
+  s.name        = 'saxerator'
+  s.version     = Saxerator::VERSION
+  s.authors     = ['Bradley Schaefer']
+  s.email       = ['bradley.schaefer@gmail.com']
+  s.homepage    = 'https://github.com/soulcutter/saxerator'
+  s.summary     = 'A SAX-based XML parser for parsing large files into manageable chunks'
+  s.description = 'A SAX-based XML parser for parsing large files into manageable chunks'
+
+  s.required_ruby_version = '>= 1.9.2'
+
+  s.rubyforge_project = 'saxerator'
+
+  s.files = FileList[
+    'LICENSE',
+    'README.md',
+    'saxerator.gemspec',
+    'lib/**/*.rb',
+    'spec/**/*.*',
+    'benchmark/**/*.rb',
+    'Gemfile',
+    'Guardfile',
+    'Rakefile',
+    '.rvmrc',
+    '.gitignore'
+  ]
+  s.test_files    = FileList['spec/**/*.*']
+  s.executables   = []
+  s.require_paths = ['lib']
+
+  s.add_runtime_dependency 'nokogiri'
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'guard'
+  s.add_development_dependency 'guard-bundler'
+  s.add_development_dependency 'guard-rspec'
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'ipsum'
+end

data/spec/fixtures/flat_blurbs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<blurbs>
+  <blurb>one</blurb>
+  <blurb>two</blurb>
+  <blurb>three</blurb>
+</blurbs>

data/spec/fixtures/nested_elements.xml

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<feed xml:lang="en-US" xmlns:media="http://search.yahoo.com/mrss/" xmlns="http://www.w3.org/2005/Atom">
+  <id>/blog</id>
+  <link type="text/html" href="https://example.com/blog" rel="alternate"/>
+  <link type="application/atom+xml" href="https://example.com/blog.atom" rel="self"/>
+  <title>Example Blog</title>
+  <updated>2012-01-01T16:17:00-06:00</updated>
+  <entry>
+    <id>1</id>
+    <published>2012-01-01T16:17:00-06:00</published>
+    <updated>2012-01-01T16:17:00-06:00</updated>
+    <link type="text/html" href="https://example.com/blog/how-to-eat-an-airplane" rel="alternate"/>
+    <title>How to eat an airplane</title>
+    <content type="html">&lt;p&gt;Airplanes are very large, which can present difficulty in digestion.&lt;/p&gt;
+
+      &lt;p&gt;The key is to break it down into bite-sized portions and go slowly.&lt;/p&gt;
+    </content>
+    <media:thumbnail url="http://www.gravatar.com/avatar/a9eb6ba22e482b71b266daadf9c9a080?s=80"/>
+    <author>
+      <name>Soulcutter</name>
+    </author>
+    <contributor>
+      <name>Jane Doe</name>
+    </contributor>
+    <contributor>
+      <name>Leviticus Alabaster</name>
+    </contributor>
+  </entry>
+  <entry>
+    <id>2</id>
+    <published>2012-01-02T18:19:00-06:00</published>
+    <updated>2012-01-02T18:19:00-06:00</updated>
+    <link type="text/html" href="https://example.com/blog/sample-data-is-hard-work" rel="alternate"/>
+    <title>Sample data is hard work</title>
+    <content type="html">&lt;p&gt;People don't believe me when I tell them how tedious it is.&lt;/p&gt;</content>
+    <media:thumbnail url="http://www.gravatar.com/avatar/742798d92b705cfeba3937e122974982?s=80"/>
+    <author>
+      <name>Bradley Schaefer</name>
+    </author>
+  </entry>
+  <entry>
+    <id>3</id>
+    <published>2012-01-15T21:22:00-06:00</published>
+    <updated>2012-01-15T21:22:00-06:00</updated>
+    <link type="text/html" href="https://example.com/blog/three-should-be-enough" rel="alternate"/>
+    <title>Three should be enough</title>
+    <content type="html">&lt;p&gt;EOM&lt;/p&gt;</content>
+    <media:thumbnail url="http://www.gravatar.com/avatar/a9eb6ba22e482b71b266daadf9c9a080?s=80"/>
+    <author>
+      <name>Soulcutter</name>
+    </author>
+  </entry>
+</feed>

data/spec/lib/saxerator_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+describe Saxerator do
+  def fixture_file(name)
+    File.new(File.join(File.dirname(__FILE__), '..', 'fixtures', name))
+  end
+
+  context ".parser" do
+    subject { parser }
+    let(:parser) { Saxerator.parser(xml) }
+
+    context "with a string with blurbs" do
+      let(:xml) { "<blurbs><blurb>one</blurb><blurb>two</blurb><blurb>three</blurb></blurbs>" }
+
+      it "should parse simple strings" do
+        results = []
+        subject.for_tag(:blurb).each { |x| results << x }
+        results.should == ['one', 'two', 'three']
+      end
+
+      context "and one non-blurb" do
+        let(:xml) { "<blurbs><blurb>one</blurb><blurb>two</blurb><blurb>three</blurb><notablurb>four</notablurb></blurbs>" }
+        it "should only parse the requested tag" do
+          results = []
+          subject.for_tag(:blurb).each { |x| results << x }
+          results.should == ['one', 'two', 'three']
+          subject.for_tag(:notablurb).each { |x| results << x }
+          results.should == ['one', 'two', 'three', 'four']
+        end
+      end
+    end
+
+    context "with a file with blurbs" do
+      let(:xml) { fixture_file('flat_blurbs.xml') }
+
+      it "should parse simple strings" do
+        results = []
+        subject.for_tag(:blurb).each { |x| results << x }
+        results.should == ['one', 'two', 'three']
+      end
+    end
+
+    context "with a file with nested elements" do
+      let(:xml) { fixture_file('nested_elements.xml') }
+      subject { parser.for_tag(:entry).first }
+
+      specify { subject['title'].should == 'How to eat an airplane' }
+      specify { subject['author'].should == {'name' => 'Soulcutter'} }
+      specify { subject['contributor'].should == [{'name' => 'Jane Doe'}, {'name' => 'Leviticus Alabaster'}] }
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+require 'simplecov'
+SimpleCov.start do
+  add_filter "spec/"
+end
+
+require 'saxerator'

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: saxerator
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Bradley Schaefer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-04-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70348029624860 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70348029624860
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70348029624300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70348029624300
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: &70348029623840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70348029623840
+- !ruby/object:Gem::Dependency
+  name: guard-bundler
+  requirement: &70348029623320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70348029623320
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: &70348029622860 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70348029622860
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: &70348029622300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70348029622300
+- !ruby/object:Gem::Dependency
+  name: ipsum
+  requirement: &70348029621880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70348029621880
+description: A SAX-based XML parser for parsing large files into manageable chunks
+email:
+- bradley.schaefer@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- saxerator.gemspec
+- lib/saxerator/configuration.rb
+- lib/saxerator/parser/nokogiri.rb
+- lib/saxerator/string_with_attributes.rb
+- lib/saxerator/version.rb
+- lib/saxerator/xml_node.rb
+- lib/saxerator.rb
+- spec/fixtures/flat_blurbs.xml
+- spec/fixtures/nested_elements.xml
+- spec/lib/saxerator_spec.rb
+- spec/spec_helper.rb
+- benchmark/benchmark.rb
+- benchmark/generate_sample_file.rb
+- Gemfile
+- Guardfile
+- Rakefile
+- .rvmrc
+- .gitignore
+homepage: https://github.com/soulcutter/saxerator
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: saxerator
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: A SAX-based XML parser for parsing large files into manageable chunks
+test_files:
+- spec/fixtures/flat_blurbs.xml
+- spec/fixtures/nested_elements.xml
+- spec/lib/saxerator_spec.rb
+- spec/spec_helper.rb