saxony 0.1.0

data/CHANGES.txt

@@ -0,0 +1,5 @@
+SAXONY, CHANGES
+
+#### 0.1.0 (2010-01-31) ###########################
+
+* Initial release

data/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2010 Solutious Inc, Delano Mandelbaum
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,24 @@
+## Saxony - 0.1 ##
+
+**Parse gigantic XML files with pleasure and ease.**
+
+## Example ##
+    
+    sax = Saxony.new :SomeObject, 1000
+    sax.parse 'path/2/huge.xml' do
+      total_count     # => Total number of SomeObjects processed
+      doc             # => Nokogiri object for 1000 SomeObject
+      elapsed_time    # => time processing current batch
+    end
+      
+## Credits
+
+* Delano Mandelbaum (http://solutious.com)
+
+
+## Thanks 
+
+
+## License
+
+See LICENSE.txt

data/Rakefile

@@ -0,0 +1,55 @@
+
+require 'rake/clean'
+require 'rake/gempackagetask'
+require 'hanna/rdoctask'
+require 'rake/testtask'
+require 'shoulda/tasks'
+require 'rake/runtest'
+require 'fileutils'
+include FileUtils
+ 
+task :default => :test
+
+
+# PACKAGE =============================================================
+
+name = "saxony"
+load "#{name}.gemspec"
+
+version = @spec.version
+
+Rake::GemPackageTask.new(@spec) do |p|
+  p.need_tar = true if RUBY_PLATFORM !~ /mswin/
+end
+
+task :test do
+  puts "Success!"
+end
+
+task :install => [ :rdoc, :package ] do
+	sh %{sudo gem install pkg/#{name}-#{version}.gem}
+end
+
+task :uninstall => [ :clean ] do
+	sh %{sudo gem uninstall #{name}}
+end
+
+
+
+Rake::RDocTask.new do |t|
+	t.rdoc_dir = 'doc'
+	t.title    = @spec.summary
+	t.options << '--line-numbers' <<  '-A cattr_accessor=object'
+	t.options << '--charset' << 'utf-8'
+	t.rdoc_files.include('LICENSE.txt')
+	t.rdoc_files.include('README.md')
+	t.rdoc_files.include('CHANGES.txt')
+	#t.rdoc_files.include('Rudyfile')  # why is the formatting f'd?
+	#t.rdoc_files.include('bin/*')
+	t.rdoc_files.include('lib/**/*.rb')
+end
+
+CLEAN.include [ 'pkg', '*.gem', '.config', 'doc', 'coverage*' ]
+
+
+

data/Rudyfile

@@ -0,0 +1,227 @@
+  require 'stella'
+
+machines do
+  
+  region :'us-east-1' do
+    ami 'ami-212ccf48'               # Stella Debian 5.0, 32-bit (US)
+  end
+  region :'eu-west-1' do
+    ami 'ami-6ecde51a'               # Alestic Debian 5.0, 32-bit (EU)
+  end
+  
+  env :stage do
+                           
+    role :app do             
+      positions 2      
+      user :root           
+      size 'm1.small'              
+    end                    
+  
+    role :gen do
+      user :root
+      size 'm1.large'
+      ami 'ami-7133d018'
+    end
+    
+    role :demo do
+      user :root
+      size 'm1.small'
+    end
+    
+  end  
+  
+
+end
+
+
+
+commands do
+  allow :apt_get, "apt-get", :y, :q
+  allow :gem_install, "/usr/bin/gem", "install", :n, '/usr/bin', :y, :V, "--no-rdoc", "--no-ri"
+  allow :gem_sources, "/usr/bin/gem", "sources"
+  allow :gem_uninstall, "/usr/bin/gem", "uninstall", :V
+  allow :update_rubygems
+  allow :rake
+  allow :thin
+  allow :stella
+  allow :rm
+  allow :ulimit
+  allow :ruby19, "/usr/local/bin/ruby"
+  allow :gem19_install, "/usr/local/bin/gem", "install"
+  allow :rackup_path do
+    v = [Stella::VERSION::MAJOR, Stella::VERSION::MINOR, Stella::VERSION::TINY].join('.')
+    "/usr/lib/ruby/gems/1.8/gems/stella-#{v}/support/sample_webapp/config.ru"
+  end
+end
+
+routines do
+           
+  role :app do                     
+    
+    # rudy -r app -v start
+    start do
+      remote do
+        #ulimit :n, '30000'
+        #ulimit :n
+        rm :f, 'thin.log'
+        mkdir :p, 'stats'
+        thin :d, :l, 'thin.log', :p, 3114, :R, rackup_path, '--stats', './stats', '--max-conns', 8192, 'start'
+      end
+    end
+    
+    # rudy -r app -v stop
+    stop do 
+      remote do
+        thin :R, rackup_path, 'stop'
+        sleep 1
+        ps 'ux'
+      end
+    end
+
+  end
+  
+  
+  # rudy -v -r gen verify ip-10-251-27-245.ec2.internal:3114
+  verify do
+    remote do |arg|
+      file_upload 'examples/essentials/plan.rb'
+      file_upload 'examples/essentials/search_terms.txt'
+      file_upload 'examples/essentials/logo.png'
+      stella :v, 'verify', :p, 'plan.rb', "#{arg.first}"
+    end
+  end
+  
+  # rudy -v -r gen generate ip-10-251-27-245.ec2.internal:3114
+  generate do
+    remote do |arg|
+      file_upload 'examples/essentials/plan.rb'
+      file_upload 'examples/essentials/search_terms.txt'
+      file_upload 'examples/essentials/logo.png'
+      stella :v, 'generate', :p, 'plan.rb', :c, 1, :d, '1m', :W, "#{arg.first}"
+    end
+  end
+
+
+  setup do  
+    after :sysupdate, :installdeps, :install_ruby19
+  end                            
+                                 
+  shutdown do                                          
+  end                            
+                                 
+  reboot do                                             
+  end
+  
+  install_netperf do
+    #ftp://ftp.netperf.org/netperf/netperf-2.4.5.tar.bz2
+  end
+  
+  install_rubyforge do
+    remote :root do
+      gem19_install 'stella', :V
+      gem_install 'stella', :V
+    end
+  end
+  
+  install_github do
+    remote :root do
+      gem_sources :a, "http://gems.github.com"
+      gem_install 'solutious-stella'
+    end
+  end
+
+  package_gem do
+    local do
+      rm :r, :f, 'pkg'
+      rake 'package'
+    end
+  end
+  
+  remove_rudy do
+    remote :root do
+      gem_uninstall 'stella'
+      rm :r, :f, '.stella'
+    end
+  end
+  
+  install_gem do
+    before :package_gem
+    remote :root do
+      file_upload "pkg/stella-#{Stella::VERSION}.gem", "/tmp/"
+      gem_install "/tmp/stella-#{Stella::VERSION}.gem"
+    end
+
+  end
+  
+  install_zlib do
+    remote do
+      wget "http://www.zlib.net/zlib-1.2.3.tar.gz"
+      tar :x, :z, :f, "zlib-1.2.3.tar.gz"
+      cd "zlib-1.2.3"
+      configure '--prefix=/usr/local'
+      make
+      make "install"
+    end
+  end
+  
+  installdeps do
+    remote :root do
+      gem_install "test-spec", "rspec", "camping", "fcgi", "memcache-client"
+      gem_install "mongrel"
+      gem_install "ruby-openid", :v, "2.0.4" # thin requires 2.0.x
+      gem_install "rack", "thin", "sinatra"
+    end
+  end
+  
+  install_jruby do
+    remote do
+      wget 'http://jruby.kenai.com/downloads/1.4.0RC2/jruby-bin-1.4.0RC2.tar.gz'
+      tar :x, :z, :f, 'jruby-bin-1.4.0RC2.tar.gz'
+      mv 'jruby-1.4.0RC2', '/usr/jruby'
+    end
+  end
+  
+  install_ruby19 do
+    before :install_zlib
+    remote do
+      apt_get "install", "libssl-dev", "libreadline5-dev", "zlib1g-dev"
+      #wget 'ftp://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.1-p243.tar.bz2'
+      rm :r, :f, 'ruby-1.9.1-p243'
+      tar :x, :j, :v, :f, 'ruby-1.9.1-p243.tar.bz2'
+      cd 'ruby-1.9.1-p243'
+      configure '--prefix=/usr/local'
+      make
+      make 'install'
+    end
+  end
+  
+  sysupdate {
+    remote {
+      apt_get "update"             
+      apt_get "install", "libxml2-dev", "libxslt-dev"  
+      apt_get "install", "build-essential", "git-core"
+      apt_get "install", "ruby1.8-dev", "rdoc", "libzlib-ruby", "rubygems"
+      apt_get "install", "libfcgi-dev", "libfcgi-ruby1.8"
+      apt_get "install", "joe", "siege", "httperf"
+      gem_sources :a, "http://gems.github.com"
+      mkdir :p, "/var/lib/gems/1.8/bin" # Doesn't get created, but causes Rubygems to fail
+      gem_install "builder", "session"
+      gem_install 'hoe-seattlerb'
+      gem_install 'rubygems-update', "-v=1.3.4"
+      update_rubygems
+      gem_install 'hoe'
+    }
+  }
+                           
+  
+end
+
+
+defaults do
+  zone :'us-east-1a'
+  environment :stage
+  role :app
+  color true               
+  user :root               
+end
+

data/lib/saxony.rb

@@ -0,0 +1,122 @@
+require 'nokogiri'
+require 'stringio'
+
+class Array
+  def chunk(number_of_chunks)
+    chunks = (1..number_of_chunks).collect { [] }
+    while self.any?
+      chunks.each do |a_chunk|
+        a_chunk << self.shift if self.any?
+      end
+    end
+    chunks
+  end
+end
+
+class Saxony 
+  VERSION = "0.1.0".freeze unless defined?(Saxony::VERSION)
+  
+  class Document < Nokogiri::XML::SAX::Document
+    attr_reader :total_count, :granularity
+    def initialize(element, granularity, &processor)
+      @root_element = nil
+      @start_time = Time.now
+      @element, @processor = element, processor
+      @granularity, @total_count = granularity, 0
+      reset
+    end
+
+    def elapsed_time
+      Time.now - @start_time
+    end
+    def xml
+      @xml ||= "<#{@root_element}>#{@buffer.string}</#{@root_element}>"
+    end
+    def doc
+      @doc ||= Nokogiri::XML(xml)
+    end
+
+    def start_element(element, attributes)
+      if element == @element.to_s
+        @count += 1 and @total_count += 1
+        @collect = true 
+        @root_element = 'SAXONYDOC' if @root_element.nil?
+      else
+        @root_element = element if @root_element.nil?
+      end
+      @buffer << to_otag(element, attributes) if @collect
+    end
+    def characters(text)
+      @buffer << text if @collect
+    end
+    def cdata_block(text)
+      @buffer << to_cdata(text) if @collect
+    end
+    def end_element(element)
+      @buffer << to_ctag(element) if @collect
+      if element == @element.to_s
+        @collect = false
+        @buffer << $/
+        process_objects if @granularity > 0 && @count >= @granularity
+      end
+    end
+    def end_document
+      process_objects unless @buffer.pos <= 0
+    end
+
+  private
+    def process_objects
+      self.instance_eval &@processor
+      reset
+    end
+    def reset
+      @xml = nil
+      @buffer, @count, @doc, @start_time = StringIO.new, 0, nil, Time.now
+    end
+    def to_otag(name, attributes=[])
+      t = name
+      unless attributes.empty?
+        chunks = attributes.chunk(attributes.size/2)
+        t << chunks.collect { |a| %Q( #{a[0]}="#{a[1]}") }.join(' ')
+      end
+      "<#{t}>"
+    end
+    def to_ctag(name)
+      "</#{name}>"
+    end
+    def to_cdata(text)
+      "<![CDATA[#{text}]]>"
+    end
+  end
+  
+  attr_reader :granularity, :element
+  def initialize(element, granularity=1000)
+    @element, @granularity = element, granularity
+  end
+  
+  # * sources can be a list of file paths, IO objects, or XML strings
+  def parse *sources, &blk
+    sources.flatten!
+    sources.each do |src|
+      saxdoc = Saxony::Document.new @element, @granularity, &blk
+      parser = Nokogiri::XML::SAX::Parser.new(saxdoc)
+      xml = (String === src && File.exists?(src)) ? File.open(src) : src
+      parser.parse xml
+    end
+  end
+end
+
+#STDERR.print '.' if @samples % 5000 == 0
+
+if $0 == __FILE__
+  sax = Saxony.new :Listing, 1000
+  sax.parse ARGV do
+   # puts xml
+    #doc.xpath("//Listing").each do |obj|
+    #end
+    p [total_count, doc.xpath("//Listing").size, elapsed_time.to_f]
+#    p 
+
+  end
+end
+

data/saxony.gemspec

@@ -0,0 +1,33 @@
+@spec = Gem::Specification.new do |s|
+  s.name = "saxony"
+  s.rubyforge_project = 'bone'
+  s.version = "0.1.0"
+  s.summary = "Parse gigantic XML files with pleasure and ease."
+  s.description = s.summary
+  s.author = "Delano Mandelbaum"
+  s.email = "delano@solutious.com"
+  s.homepage = ""
+  
+  s.extra_rdoc_files = %w[README.md LICENSE.txt CHANGES.txt]
+  s.has_rdoc = true
+  s.rdoc_options = ["--line-numbers", "--title", s.summary, "--main", "README.md"]
+  s.require_paths = %w[lib]
+  
+  #s.executables = %w[bone]
+  
+  s.add_dependency 'nokogiri'
+  
+  # = MANIFEST =
+  # git ls-files
+  s.files = %w(
+  CHANGES.txt
+  LICENSE.txt
+  README.md
+  Rakefile
+  Rudyfile
+  lib/saxony.rb
+  saxony.gemspec
+  )
+
+  
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification 
+name: saxony
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Delano Mandelbaum
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-01-31 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Parse gigantic XML files with pleasure and ease.
+email: delano@solutious.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+- LICENSE.txt
+- CHANGES.txt
+files: 
+- CHANGES.txt
+- LICENSE.txt
+- README.md
+- Rakefile
+- Rudyfile
+- lib/saxony.rb
+- saxony.gemspec
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --title
+- Parse gigantic XML files with pleasure and ease.
+- --main
+- README.md
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: bone
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Parse gigantic XML files with pleasure and ease.
+test_files: []
+