easy_sax 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 728c5b578b32cb27cbb786cd5ee990b7ca03c2b8
+  data.tar.gz: 02ce51a239add7f7b57c1950e61bb851303baf35
+SHA512:
+  metadata.gz: 6b01b16e0e38a178410e74f2550bc0229ff2ea14638adceedbab648a6b98861219e41c7dcc502d4a8cca1df22159d746f0a02ddbfc22aaacf5a5010999afd573
+  data.tar.gz: 6d00ec699b5387f6665b3e2350a69c01c365d00d558e88d1c3a1284b5ff6f0c7dc182a4365848ada8032a19eacc4fe1ea16c8033391717d10b5206e0e1d6ece0

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.0
+before_install: gem install bundler -v 1.12.5

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in easy_sax.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Eric Northam
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,145 @@
+# EasySax
+
+EasySax allows you to easily parse large files without the messy syntax needed for working with most Sax parsers. It was inspired after attempting to use [SaxMachine](https://github.com/pauldix/sax-machine) to parse a 500mb XML file that resulted in a huge spike to 2gbs of memory inside a Rails app. EasySax is very lightweight and only stores the element currently being used in memory. It also allows you to access parent elements without storing the whole parent tree in memory. Testing with the same file mentioend above, the memory stayed constant and it processed the file much faster. EasySax is currently used in production at EasyBroker.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'easy_sax'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install easy_sax
+
+## Usage
+Given the following test XML
+```xml
+<agencies>
+  <agency id="1">
+    <name>Foo</name>
+    <phone>12345678</phone>
+    <properties>
+      <property id="2">
+        <title>Test 2</title>
+        <images>
+          <image url="http://test.com/1.jpg"/>
+          <image url="http://test.com/2.jpg"/>
+        </images>
+      </property>
+      <property id="3">
+        <title>Test 3</title>
+        <images>
+          <image url="http://test.com/4.jpg"/>
+          <image url="http://test.com/5.jpg"/>
+        </images>
+      </property>
+    </properties>
+  </agency>
+  <agency id="2">
+    <name>Bar</name>
+    <properties>
+      <property id="4">
+        <title>Test 4</title>
+        <images>
+          <image url="http://test.com/3.jpg"/>
+          <image url="http://test.com/4.jpg"/>
+        </images>
+      </property>
+    </properties>
+  </agency>
+</agencies>
+```
+You can parse all the property elements with
+
+```ruby
+parser = EasySax.parser(File.open('test.xml'))
+parser.parse_each(:property) do |property|
+  puts "Property id[#{property.attrs[:id]}] title[#{property[:title].text}]"
+end
+```
+
+Outputs
+
+```
+Property id[2] title[Test 2]
+Property id[3] title[Test 3]
+Property id[4] title[Test 4]
+```
+
+You can also use the `text_for` method if you prefer to get text elements. `property.text_for(:title)` is the same as `property[:title].text` except it returns nil if the title element doesn't exist.
+
+If you want to print the property image urls you need to let the parser know that it is an array
+
+```ruby
+parser = EasySax.parser(File.open('test.xml'))
+parser.parse_each(:property, arrays: ['images']) do |property|
+  image_urls = property[:images].map { |image| image.attrs[:url] }
+  puts "Property id[#{property.attrs[:id]}] images#{image_urls}"
+end
+```
+
+Outputs
+
+```
+Property id[2] images ["http://test.com/1.jpg", "http://test.com/2.jpg"]
+Property id[3] images ["http://test.com/4.jpg", "http://test.com/5.jpg"]
+Property id[4] images ["http://test.com/3.jpg", "http://test.com/4.jpg"]
+```
+
+Now for something really cool. If you want the root ancestor use the second param in the `parse_each` block
+
+```ruby
+parser = EasySax.parser(File.open('test.xml'))
+parser.parse_each(:property) do |property, ancestor|
+  puts "Property id[#{property.attrs[:id]}] agency id[#{ancestor[:agency].attrs[:id]}]"
+end
+```
+
+Outputs
+
+```
+Property id[2] agency id[1]
+Property id[3] agency id[1]
+Property id[4] agency id[2]
+```
+
+Now maybe you're lazy like me and don't care about the `agencies` element and want the `agency` to be the oldest ancestor.
+
+```ruby
+parser = EasySax.parser(File.open('test.xml'))
+parser.parse_each(:property, ignore: ['agencies']) do |property, ancestor|
+  puts "Property id[#{property.attrs[:id]}] agency id[#{ancestor.attrs[:id]}]"
+end
+```
+
+Outputs
+
+```
+Property id[2] agency id[1]
+Property id[3] agency id[1]
+Property id[4] agency id[2]
+```
+
+You can also use `ignore` to speed up the parser by allowing it to know that it doesn't need to keep track of the those elements.
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/easybroker/easy_sax.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "easy_sax"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/easy_sax.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'easy_sax/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "easy_sax"
+  spec.version       = EasySax::VERSION
+  spec.authors       = ["Eric Northam"]
+  spec.email         = ["eric@easybroker.com"]
+
+  spec.summary       = "A simple SAX parser that enables parsing of large files without the messy syntax of typical SAX parsers."
+  spec.description   = "A simple SAX parser that enables parsing of large files without the messy syntax of typical SAX parsers. Currently depends on Nokogiri."
+  spec.homepage      = "https://github.com/easybroker/easy_sax"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "nokogiri", ">= 1.6"
+  spec.add_dependency "activesupport", ">= 1.6"
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "pry"
+end

data/lib/easy_sax.rb

@@ -0,0 +1,22 @@
+require 'easy_sax/version'
+require 'easy_sax/parse_error'
+require 'easy_sax/simple_element'
+require 'easy_sax/parser'
+
+# A simple SAX parser that enables parsing of large files without
+# the messy syntax of typical SAX parsers. Currently depends on
+# Nokogiri.
+#
+# Basic Usage:
+# EasySax.parser(io).parse_each(target_element, ignore:, array:)
+#   target_element: is the element you want to parse
+#   ignore: are elements that will be ignored and not parsed
+#   arrays: are the elements that should parsed into arrays
+#
+# You should use a block which returns the parsed target element
+# and it's ancestors if it has one.
+module EasySax
+  def self.parser(io)
+    EasySax::Parser.new(io)
+  end
+end

data/lib/easy_sax/parse_error.rb

@@ -0,0 +1,2 @@
+class EasySax::ParseError < StandardError
+end

data/lib/easy_sax/parser.rb

@@ -0,0 +1,89 @@
+require 'active_support/core_ext/object/blank'
+require 'nokogiri'
+
+class EasySax::Parser < Nokogiri::XML::SAX::Document
+  attr_reader :io,
+    :target_element,
+    :callback,
+    :ignorable_elements,
+    :array_elements,
+    :element_stack,
+    :element_text
+
+  def initialize(io)
+    @io = io
+  end
+
+  def parse_each(target_element, ignore: [], arrays: [], &block)
+    validate_array(:arrays, arrays)
+    @target_element = target_element.to_s
+    @ignorable_elements = validate_array(:ignore, ignore)
+    @array_elements = validate_array(:arrays, arrays)
+    @element_stack = []
+    @callback = block
+    Nokogiri::XML::SAX::Parser.new(self).parse(io)
+  end
+
+  def start_element(name, attrs = [])
+    return if ignorable_elements.include?(name)
+    @element_text = ''
+    parent = element_stack.last
+
+    if parent.nil?
+      element_stack << EasySax::SimpleElement.new(name, attrs.to_h)
+    else
+      add_child(parent, name, attrs)
+    end
+  end
+
+  def characters(string)
+    @element_text << string if element_text
+  end
+
+  def cdata_block(string)
+    characters(string)
+  end
+
+  def end_element(name)
+    return if ignorable_elements.include?(name)
+
+    element = element_stack.pop
+    return if element.kind_of?(Array)
+
+    element.text = element_text.strip if element_text.present?
+    callback.call element, element_stack.first if name == target_element
+  end
+
+  def error(string)
+    raise EasySax::ParseError.new(string)
+  end
+
+  private
+
+  def validate_array(field, array)
+    if array.nil?
+      []
+    elsif array.kind_of?(Array)
+      array.map { |element| element.to_s }
+    else
+      raise ArgumentError, ("%s must be an Array" % field)
+    end
+  end
+
+  def add_child(parent, name, attrs)
+    if array_elements.include?(name)
+      parent[name] = []
+      element_stack << parent[name]
+    else
+      element = EasySax::SimpleElement.new(name, attrs.to_h)
+
+      if parent.kind_of?(Array)
+        parent << element
+      elsif name != target_element
+        parent[name] = element
+      end
+
+      element_stack << element
+    end
+  end
+end

data/lib/easy_sax/simple_element.rb

@@ -0,0 +1,34 @@
+require 'active_support/core_ext/hash/indifferent_access'
+
+class EasySax::SimpleElement
+  attr_accessor :name, :attrs, :elements, :text
+
+  def initialize(name, attrs)
+    @name = name
+    @attrs = HashWithIndifferentAccess.new(attrs || {})
+    @elements = HashWithIndifferentAccess.new
+  end
+
+  def [](key)
+    elements[key]
+  end
+
+  def []=(key, value)
+    elements[key] = value
+  end
+
+  def text_for(key)
+    elements[key]&.text
+  end
+
+  def to_h
+    {}.tap do |hash|
+      hash[:attrs] = attrs unless attrs.empty?
+      hash[:elements] = elements unless elements.empty?
+      hash[:text] = text if text
+    end
+  end
+
+  alias_method :inspect, :to_h
+  alias_method :to_s, :to_h
+end

data/lib/easy_sax/version.rb

@@ -0,0 +1,3 @@
+module EasySax
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,144 @@
+--- !ruby/object:Gem::Specification
+name: easy_sax
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Eric Northam
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-10-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple SAX parser that enables parsing of large files without the messy
+  syntax of typical SAX parsers. Currently depends on Nokogiri.
+email:
+- eric@easybroker.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- easy_sax.gemspec
+- lib/easy_sax.rb
+- lib/easy_sax/parse_error.rb
+- lib/easy_sax/parser.rb
+- lib/easy_sax/simple_element.rb
+- lib/easy_sax/version.rb
+homepage: https://github.com/easybroker/easy_sax
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.3
+signing_key: 
+specification_version: 4
+summary: A simple SAX parser that enables parsing of large files without the messy
+  syntax of typical SAX parsers.
+test_files: []