onigiri 0.1.6

data/.gitignore

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.rspec
+.rvmrc
+*.swp

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in onigiri.gemspec
+gemspec

data/README.md

@@ -0,0 +1,6 @@
+This gem is supposed to replace a tidy-ext for one of our projects.
+
+Tidy-ext has nasty memory leaks, tends to crash and is incompatible with Ruby 1.9.
+So here I am trying to use a japanese saw to make some rice balls.
+
+PS: Worse is better

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/autotest/discover.rb

@@ -0,0 +1 @@
+Autotest.add_discovery { "rspec2" }

data/lib/onigiri.rb

@@ -0,0 +1,51 @@
+require "rubygems"
+require "nokogiri"
+
+module Onigiri
+  extend self
+  @@registry ||= {}
+
+  class OnigiriHandlerTaken < StandardError
+    def description
+      "There was an attempt to override registered handler. This usually indicates a bug in Onigiri."
+    end
+  end
+
+  def clean(data, *params)
+    dupe = Onigiri::Document.parse data
+    params.flatten.each do |method|
+      dupe = dupe.send(method) if @@registry[method]
+    end
+    dupe.to_html
+  end
+
+  class Document < Nokogiri::HTML::DocumentFragment
+    class << self
+      def parse(tags)
+        # Remove formatting whitespaces
+        # Those do not represent any data while messing up the tree
+        tags = tags.gsub(/(\r|\n)/, '').gsub(/> *</, '><') if tags.respond_to? :gsub
+        super
+      end
+    end
+  end
+
+  private
+
+  def register_handler(name)
+    unless @@registry[name]
+      @@registry[name] = true
+    else
+      raise OnigiriHandlerTaken
+    end
+  end
+
+end
+
+require "onigiri/drop_empty_paras"
+require "onigiri/enclose_block_text"
+require "onigiri/enclose_text"
+require "onigiri/fix_backslash"
+require "onigiri/show_body_only"
+require "onigiri/merge_by_tag"
+require "onigiri/hide_comments"

data/lib/onigiri/drop_empty_paras.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+module Onigiri
+  register_handler :drop_empty_paras
+  class Document
+    def drop_empty_paras
+      dupe = dup
+      dupe.css('p').each do |p|
+        p.remove if p.children.empty?
+      end
+      dupe
+    end
+  end
+end

data/lib/onigiri/enclose_block_text.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+module Onigiri
+  register_handler :enclose_block_text
+  class Document
+    def enclose_block_text
+      dupe = dup
+      strict_tags = {"noscript" => 1, "form" => 1, "blockquote" => 1}
+      dupe.traverse do |elem|
+        if strict_tags[elem.name]
+          elem.children.each do |target|
+            if target.text?
+              target.add_previous_sibling "<p>#{target.content.strip}</p>"
+              target.unlink
+            end
+          end
+        end
+      end
+      dupe
+    end
+  end
+end

data/lib/onigiri/enclose_text.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+module Onigiri
+  register_handler :enclose_text
+  class Document
+    def enclose_text
+      dupe = dup
+      wrapper = Onigiri::Document.parse('<p>').child
+      body = dupe.css('body').children
+      body = dupe.children if body.empty?
+      body.each do |target|
+        if target.parent && (target.text? || target.description.inline?)
+          wrap = target.add_previous_sibling(wrapper)
+          wrap << target.unlink
+        end
+      end
+      dupe
+    end
+  end
+end

data/lib/onigiri/fix_backslash.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+module Onigiri
+  register_handler :fix_backslash
+  class Document
+    def fix_backslash
+      dupe = dup
+      attrset = ['src', 'longdesc', 'href', 'action']
+      dupe.css("[#{attrset.join('], [')}]").each do |target|
+        attrset.each do |attr|
+          target[attr] = target[attr].gsub("\\", "/") if target[attr]
+        end
+      end
+      dupe
+    end
+  end
+end

data/lib/onigiri/hide_comments.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+module Onigiri
+  register_handler :hide_comments
+  class Document
+    def hide_comments
+      dupe = dup
+      dupe.traverse {|elem| elem.remove if elem.class == Nokogiri::XML::Comment}
+      dupe
+    end
+  end
+end

data/lib/onigiri/merge_by_tag.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+module Onigiri
+  register_handler :merge_divs
+  register_handler :merge_spans
+  class Document
+
+    def merge_divs
+      self.merge_by_tag('div')
+    end
+
+    def merge_spans
+      self.merge_by_tag('span')
+    end
+
+    # This is going to be ugly
+    def merge_by_tag(tag_name)
+      dupe = dup
+      # First pass. Finding deepest <div>s that require merging upwards.
+      mergers = dupe.find_merger_elements(tag_name)
+      # Second pass. Traverse tree upwards from each merger <div> gathering attributes on our way
+      mergers.each do |merger|
+        data = singular_upverse(merger)
+        merger.children.each do |survivor|
+          data['root'] << survivor
+        end
+        data['deletion_node'].remove
+        data['root']['class'] = data['class'] if data['class']
+        data['root']['style'] = data['style'] if data['style']
+      end
+      dupe
+    end
+
+    def find_merger_elements(tag_name)
+      result = []
+      self.css(tag_name).each do |elem|
+        # !(node.next_sibling || node.previous_sibling) vs. node.parent.children.size
+        result << elem if elem.parent.children.size == 1 && elem.parent.name == tag_name
+      end
+      result
+    end
+
+    def singular_upverse(node)
+      if node.parent.name == node.name && !(node.next_sibling || node.previous_sibling)
+        data = singular_upverse(node.parent)
+        # If we got root node we should set a deletion point for root.
+        # If we have a deletion point - no need to reset it.
+        data['deletion_node'] ||= node if data['root']
+      else
+        data = Hash.new
+        data['root'] = node
+      end
+
+      # Ensuring uglyness
+      data['style'] ? (data['style'] += " #{node['style']}" if node['style']) : data['style'] = node['style']
+      data['class'] ? (data['class'] += " #{node['class']}" if node['class']) : data['class'] = node['class']
+
+      data
+    end
+  end
+end

data/lib/onigiri/show_body_only.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+module Onigiri
+  register_handler :show_body_only
+  class Document
+    def show_body_only
+      dupe = self.css('body').empty? ? dup : Onigiri::Document.parse("")
+      self.css('body').children.each do |child|
+        dupe << child
+      end
+      dupe
+    end
+  end
+end

data/lib/onigiri/version.rb

@@ -0,0 +1,3 @@
+module Onigiri
+  VERSION = "0.1.6"
+end

data/onigiri.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "onigiri/version"
+
+Gem::Specification.new do |s|
+  s.name        = "onigiri"
+  s.version     = Onigiri::VERSION
+  s.authors     = ["Dmitrii Soltis"]
+  s.email       = ["slotos@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Attempt to replicate (at least some) functions of tidy utility}
+  s.description = %q{This gem is supposed to replace a tidy-ext in one of our projects. Tidy-ext has nasty memory leaks, tends to crash and is incompatible with Ruby 1.9. So here I am trying to use a japanese saw to make some rice balls.}
+  s.date        = "2011-07-18"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "nokogiri"
+  s.add_development_dependency "rspec", ">= 2.0.0"
+  s.add_development_dependency "autotest"
+end

data/spec/onigiri_spec.rb

@@ -0,0 +1,214 @@
+# encoding: utf-8
+require 'spec_helper'
+include Onigiri
+
+describe Onigiri do
+  it 'should throw exception when registering two handlers with the same name' do
+    lambda do
+      module Onigiri
+        register_handler :drop_empty_paras
+      end
+    end.should raise_error(OnigiriHandlerTaken)
+  end
+
+  it 'should define "drop_empty_paras" method that removes empty paragraphs from argument string' do
+    input = 'this is text with <p>some </p><p></p> <p>emptyness inside</p>'
+    expectation = 'this is text with <p>some </p><p>emptyness inside</p>'
+    Onigiri::clean(input, :drop_empty_paras).should == expectation
+  end
+
+  it 'should define "enclose_block_text" method that encloses any text inside <form>, <blockquote>, <noscript> in <p> tag with trimmed spaces' do
+    input = '<NOSCRIPT>hello <form>there <blockquote>pretty</blockquote> world</form></NOSCRIPT>'
+    expectation = '<noscript><p>hello</p><form><p>there</p><blockquote><p>pretty</p></blockquote><p>world</p></form></noscript>'
+    Onigiri::clean(input, :enclose_block_text).gsub(/(\r|\n| )/, '').should == expectation
+  end
+
+  it 'should define "enclose_text" method that encloses any text in <body> using <p> tag' do
+    input = '<body>some text<form>some text in form</form><p>some text in p</p><div><blockquote>some text in third level element</blockquote></div></body>'
+    expectation = '<body><p>some text</p><form>some text in form</form><p>some text in p</p><div><blockquote>some text in third level element</blockquote></div></body>'
+    Onigiri::clean(input, :enclose_text).gsub(/(\r|\n)/, '').gsub(/> *</, '><').should == expectation
+  end
+
+  it 'should wrap inline elements with "enclose_text" method' do
+    input = '<body><span>some inline text</span><form>some text in form</form><p>some text in p</p><div><blockquote>some text in third level element</blockquote></div></body>'
+    expectation = '<body><p><span>some inline text</span></p><form>some text in form</form><p>some text in p</p><div><blockquote>some text in third level element</blockquote></div></body>'
+    Onigiri::clean(input, :enclose_text).gsub(/(\r|\n)/, '').gsub(/> *</, '><').should == expectation
+  end
+
+  it 'should work with root element when "enclose_text" method was called on <body>less fragment' do
+    input = '<span>some inline text</span><form>some text in form</form><p>some text in p</p><div><blockquote>some text in third level element</blockquote></div>'
+    expectation = '<p><span>some inline text</span></p><form>some text in form</form><p>some text in p</p><div><blockquote>some text in third level element</blockquote></div>'
+    Onigiri::clean(input, :enclose_text).gsub(/(\r|\n)/, '').gsub(/> *</, '><').should == expectation
+  end
+
+  it 'should work around issue #407 (https://github.com/tenderlove/nokogiri/issues/407)' do
+    input = "<p class='red'><span class='capital'>о</span>тветственный редактор легендарного журнала The Strand, на страницах которого впервые увидели свет рассказы Артура Конан Дойля, Эдит Несбит, Агаты Кристи и Редьярда Киплинга\r\n– Эндрю Ф. Гулли обнаружил 15 ранее не публиковавшихся рассказов классика детектива – Дэшила Хэммета.</p>\nСвое открытие Гулли сделал, изучая онлайн-библиотеку и архивы центра Рэнсома при Техасском университете. «Я знал, что Лиллиан Хеллман, бывшая в тесных отношениях с Хэмметом, передала Центру большое количество его бумаг, и надеялся найти среди них что-то интересное, - рассказывает редактор The Strand. – Каково же было мое удивление, когда после тщательной проверки, на которую ушло около 100 часов работы, я обнаружил 15 совершенно новых, неизвестных читателю рассказов»\r\n\r\n<div>\r\n<br/></div>\r\n\r\n<div>Среди рассказов есть как детективы, так и «психологические» истории. Неизвестно, почему автор отказался публиковать их при жизни, ведь, по словам Эндрю Гулли, написаны они идеально: «Некоторые писатели неспособны адекватно оценивать свои произведения: им кажется, что они недоработаны или неталантливо написаны. Возможно, Хэммет тоже так посчитал, потому что, несмотря на отличный слог и композицию, найденные его рассказы отличаются от «традиционных» его произведений. Хотя как раз это и поможет читателям оценитьмногогранность таланта автора».</div>\r\n\r\n<div>\r\n<br/></div>\r\n\r\n<div>Вначале рассказы выйдут в журнале The Strand, а потом будут изданы отдельной книгой. Ранее в The Strand были напечатаны неизвестные произведения других классиков литературы - Марка Твена, П.Дж. Вудхауза, Агаты Кристи и Грэма Грина.</div>"
+    pending("Rspec itself leads to `#<NoMethodError: undefined method `call' for #<String>>` error when Onigiri returns string (as it should).\r\nStill useful for testing tho.")
+    Onigiri.clean(input, :enclose_text).should_not raise_error
+  end
+
+  describe 'should define "fix_backslash" method that fixes "\" for "/" in urls' do
+    it 'is fixing href attributes' do
+      input = '<a href="http:\\\\google.com/">http:\\\\google.com/</a><link rel="stylesheet" type="text/css" href="http:\\\\bing.com\\">'
+      expectation = '<a href="http://google.com/">http:\\\\google.com/</a><link rel="stylesheet" type="text/css" href="http://bing.com/">'
+      Onigiri::clean(input, :fix_backslash).should == expectation
+    end
+
+    it 'is fixing src attributes' do
+      input = '<img src="http:\\/imagehosting.com/3\\image.png">'
+      expectation = '<img src="http://imagehosting.com/3/image.png">'
+      Onigiri::clean(input, :fix_backslash).should == expectation
+    end
+
+    it 'is fixing longdesc attributes' do
+      input = '<img src="http://imagehosting.com/3/image.png" longdesc="http:\\/alt.com\\desc.txt">'
+      expectation = '<img src="http://imagehosting.com/3/image.png" longdesc="http://alt.com/desc.txt">'
+      Onigiri::clean(input, :fix_backslash).should == expectation
+    end
+
+    it 'is fixing form action attributes' do
+      input = '<form action="\\application.php">\\application.php</form>'
+      expectation = '<form action="/application.php">\\application.php</form>'
+      Onigiri::clean(input, :fix_backslash).should == expectation
+    end
+
+    it 'should all work together' do
+      input = '<a href="http:\\\\google.com/">link</a><link rel="stylesheet" type="text/css" href="http:\\\\bing.com\\"><img src="http://imagehosting.com\\3/image.png" longdesc="http:\\/alt.com\\desc.txt"><form action="\\application.php">русский текст</form>'
+      expectation = '<a href="http://google.com/">link</a><link rel="stylesheet" type="text/css" href="http://bing.com/"><img src="http://imagehosting.com/3/image.png" longdesc="http://alt.com/desc.txt"><form action="/application.php">русский текст</form>'
+      Onigiri::clean(input, :fix_backslash).should == expectation
+    end
+  end
+
+  it 'should provide a "show_body_only" method that extracts contents of a <body> element for incorporation' do
+    input = '<body>some text<form>some text in form</form><p>some text in p</p><div><blockquote>some text in third level element</blockquote></div></body>'
+    expectation = 'some text<form>some text in form</form><p>some text in p</p><div><blockquote>some text in third level element</blockquote></div>'
+    Onigiri::clean(input, :show_body_only).gsub(/(\r|\n)/, '').gsub(/> *</, '><').should == expectation
+  end
+
+  it 'should not do anything with "show_body_only" if there is not body' do
+    input = 'some text<form>some text in form</form><p>some text in p</p><div><blockquote>some text in third level element</blockquote></div>'
+    expectation = 'some text<form>some text in form</form><p>some text in p</p><div><blockquote>some text in third level element</blockquote></div>'
+    Onigiri::clean(input, :show_body_only).gsub(/(\r|\n)/, '').gsub(/> *</, '><').should == expectation
+  end
+
+  it 'should provide a "merge_divs" method that will merge nested <div> such as "<div><div>...</div></div>" into top-level div discarding inner <div>s attributes except for "class" and "style"' do
+    input = <<HTML
+<div class="first">
+  <div class="top">
+    <div id ="!hoho" class="test">
+      <div data-remote="true" style="color: black;" class="tost">
+        <p>data</p>
+        <div>
+          <div class="yopo">
+            another text
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+HTML
+    expectation = <<HTML
+<div class="first top test tost" style="color: black;">
+<p>data</p>
+<div class="yopo">another text</div>
+</div>
+HTML
+    Onigiri::clean(input, :merge_divs).gsub(/(\r|\n)/, '').gsub(/> *</, '><').gsub(/(>) +| +(<)/, '\1\2').should == expectation.gsub(/(\r|\n)/, '').gsub(/> *</, '><').gsub(/(>) +| +(<)/, '\1\2')
+  end
+
+  it 'should provide a "merge_spans" method that replicates "merge_divs" for <span> tag' do
+    input = <<HTML
+<span class="first">
+  <span class="top">
+    <span id ="!hoho" class="test">
+      <span data-remote="true" style="color: black;" class="tost">
+        data
+        <span>
+          <span class="yopo">
+            another text
+          </span>
+        </span>
+      </span>
+    </span>
+  </span>
+</span>
+HTML
+    expectation = <<HTML
+<span class="first top test tost" style="color: black;">
+data
+<span class="yopo">another text</span>
+</span>
+HTML
+    Onigiri::clean(input, :merge_spans).gsub(/(\r|\n)/, '').gsub(/> *</, '><').gsub(/(>) +| +(<)/, '\1\2').should == expectation.gsub(/(\r|\n)/, '').gsub(/> *</, '><').gsub(/(>) +| +(<)/, '\1\2')
+  end
+
+  it 'should provide "hide_comments" method that will remove all comments from the string' do
+    input = <<HTML
+<span class="first">
+  <span class="top">
+    <span id="!hoho" class="test">
+    <!------ hello world! -->
+      <span data-remote="true" style="color: black;" class="tost">
+        data
+        <span>
+          <span class="yopo">
+            another text
+          </span>
+        </span>
+      </span>
+    <!-- another comment -->
+    </span>
+  </span>
+</span>
+HTML
+    expectation = <<HTML
+<span class="first">
+  <span class="top">
+    <span id="!hoho" class="test">
+      <span data-remote="true" style="color: black;" class="tost">
+        data
+        <span>
+          <span class="yopo">
+            another text
+          </span>
+        </span>
+      </span>
+    </span>
+  </span>
+</span>
+HTML
+    Onigiri::clean(input, :hide_comments).gsub(/(\r|\n)/, '').gsub(/> *</, '><').gsub(/(>) +| +(<)/, '\1\2').should == expectation.gsub(/(\r|\n)/, '').gsub(/> *</, '><').gsub(/(>) +| +(<)/, '\1\2')
+  end
+
+  # Noted pending jobs.
+  it 'should provide a "automerge_divs" method that will merge nested <div> such as "<div><div>...</div></div>" into top-level div moving inner <div>s attributes into outer one; however it shouldnt merge together <div>s that have valid id attributes (id attribute serves as a down-top merge breakpoint)' do
+    input = <<HTML
+<div class="first">
+  <div class="top" id="fff">
+    <div id ="!hoho" class="test">
+      <div data-remote="true" style="color: black;" class="tost">
+        <p>data</p>
+        <div>
+          <div id="yopo">
+            another text
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+HTML
+    expectation = <<HTML
+<div class="first top" id="fff">
+<div id="!hoho" class="test tost c1" data-remote="true">
+<p>data</p>
+<div id="yopo">another text</div>
+</div>
+</div>
+HTML
+    pending('Noted the difference between merge-divs: yes/auto, but the latter one doesn\'t get priority')
+  end
+
+  it 'should provide "drop-proprietary-attributes" method that will drop all attributes, not defined in W3C standard or applied to mismatched element'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+$KCODE = 'u'
+require 'rubygems'
+require 'bundler'
+Bundler.require(:default, :development)

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: onigiri
+version: !ruby/object:Gem::Version
+  version: 0.1.6
+  prerelease: 
+platform: ruby
+authors:
+- Dmitrii Soltis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-07-18 00:00:00.000000000 +03:00
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &72106630 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *72106630
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &72106380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *72106380
+- !ruby/object:Gem::Dependency
+  name: autotest
+  requirement: &72106170 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *72106170
+description: This gem is supposed to replace a tidy-ext in one of our projects. Tidy-ext
+  has nasty memory leaks, tends to crash and is incompatible with Ruby 1.9. So here
+  I am trying to use a japanese saw to make some rice balls.
+email:
+- slotos@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- autotest/discover.rb
+- lib/onigiri.rb
+- lib/onigiri/drop_empty_paras.rb
+- lib/onigiri/enclose_block_text.rb
+- lib/onigiri/enclose_text.rb
+- lib/onigiri/fix_backslash.rb
+- lib/onigiri/hide_comments.rb
+- lib/onigiri/merge_by_tag.rb
+- lib/onigiri/show_body_only.rb
+- lib/onigiri/version.rb
+- onigiri.gemspec
+- spec/onigiri_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Attempt to replicate (at least some) functions of tidy utility
+test_files: []