dq-readability 0.2.0

data/Gemfile

@@ -0,0 +1,10 @@
+source "http://rubygems.org"
+
+gem 'fastimage', '~> 1.2.13'
+gem 'rake'
+
+group :test do
+  gem "fakeweb",      "~> 1.3.0"
+end
+
+gemspec

data/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1 @@
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/readability

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby 
+require 'rubygems'
+require 'open-uri'
+require 'optparse'
+require File.dirname(__FILE__) + '/../lib/dq-readability'
+
+options = { :debug => false, :images => false }
+options_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename($0)} [options] URL"
+
+  opts.on("-d", "--debug", "Show debug output") do |v|
+    options[:debug] = v
+  end
+
+  opts.on("-i", "--images", "Keep images and links") do |i|
+    options[:images] = i
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end
+options_parser.parse!
+
+if ARGV.length != 1
+  STDERR.puts options_parser
+  exit 1
+end
+
+text = open(ARGV.first).read
+if options[:images]
+  puts Readability::Document.new(text, :tags => %w[div p img a], 
+                                       :attributes => %w[src href], 
+                                       :remove_empty_nodes => false, 
+                                       :debug => options[:debug]).content
+else
+  puts Readability::Document.new(text, :debug => options[:debug]).content
+end

data/dq-readability.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "dq-readability"
+  s.version     = '0.2.0'
+  s.authors     = ["Prateek Papriwal"]
+  s.email       = ["papriwalprateek@gmail.com"]
+  s.homepage    = "http://github.com/DaQwest/dq-readability"
+  s.summary     = %q{Port of arc90's readability project to ruby}
+  s.description = %q{Port of arc90's readability project to ruby. The base code is derived from https://github.com/cantino/ruby-readability}
+
+  s.rubyforge_project = "dq-readability"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rspec", ">= 2.8"
+  s.add_development_dependency "rspec-expectations", ">= 2.8"
+  s.add_development_dependency "rr", ">= 1.0"
+  s.add_dependency 'nokogiri', '>= 1.4.2'
+  s.add_dependency 'guess_html_encoding', '>= 0.0.4'
+end

data/lib/dq-readability.rb

@@ -0,0 +1,515 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'nokogiri'
+require 'guess_html_encoding'
+
+module Readability
+  class Document
+    DEFAULT_OPTIONS = {
+      :retry_length               => 250,
+      :min_text_length            => 25,
+      :remove_unlikely_candidates => true,
+      :weight_classes             => true,
+      :clean_conditionally        => true,
+      :remove_empty_nodes         => true,
+      :min_image_width            => 130,
+      :min_image_height           => 80,
+      :ignore_image_format        => []
+    }.freeze
+    
+    REGEXES = {
+        :unlikelyCandidatesRe => /combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i,
+        :okMaybeItsACandidateRe => /and|article|body|column|main|shadow/i,
+        :positiveRe => /article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i,
+        :negativeRe => /combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i,
+        :divToPElementsRe => /<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i,
+        :replaceBrsRe => /(<br[^>]*>[ \n\r\t]*){2,}/i,
+        :replaceFontsRe => /<(\/?)font[^>]*>/i,
+        :trimRe => /^\s+|\s+$/,
+        :normalizeRe => /\s{2,}/,
+        :killBreaksRe => /(<br\s*\/?>(\s|&nbsp;?)*){1,}/,
+        :videoRe => /http:\/\/(www\.)?(youtube|vimeo)\.com/i
+    }
+    
+    attr_accessor :options, :html, :best_candidate, :candidates, :best_candidate_has_image
+
+    def initialize(input, options = {})
+      @options = DEFAULT_OPTIONS.merge(options)
+      @input = input
+
+      if RUBY_VERSION =~ /^(1\.9|2)/ && !@options[:encoding]
+        @input = GuessHtmlEncoding.encode(@input, @options[:html_headers]) unless @options[:do_not_guess_encoding]
+        @options[:encoding] = @input.encoding.to_s
+      end
+
+      @input = @input.gsub(REGEXES[:replaceBrsRe], '</p><p>').gsub(REGEXES[:replaceFontsRe], '<\1span>')
+      @remove_unlikely_candidates = @options[:remove_unlikely_candidates]
+      @weight_classes = @options[:weight_classes]
+      @clean_conditionally = @options[:clean_conditionally]
+      @best_candidate_has_image = true
+      make_html
+    end
+
+    def prepare_candidates
+      @html.css("script, style").each { |i| i.remove }
+      remove_unlikely_candidates! if @remove_unlikely_candidates
+      transform_misused_divs_into_paragraphs!
+      
+      @candidates     = score_paragraphs(options[:min_text_length])
+      @best_candidate = select_best_candidate(@candidates)
+    end
+
+    def make_html
+      @html = Nokogiri::HTML(@input, nil, @options[:encoding])
+      # In case document has no body, such as from empty string or redirect
+      @html = Nokogiri::HTML('<body />', nil, @options[:encoding]) if @html.css('body').length == 0
+
+      # Remove html comment tags
+      @html.xpath('//comment()').each { |i| i.remove }
+    end
+
+    def images(content=nil, reload=false)
+      begin
+        require 'fastimage'
+      rescue LoadError
+        raise "Please install fastimage in order to use the #images feature."
+      end
+
+      @best_candidate_has_image = false if reload
+
+      prepare_candidates
+      list_images   = []
+      tested_images = []
+      content       = @best_candidate[:elem] unless reload
+
+      return list_images if content.nil?
+      elements = content.css("img").map(&:attributes)
+
+        elements.each do |element|
+          next unless element["src"]
+
+          url     = element["src"].value
+          height  = element["height"].nil?  ? 0 : element["height"].value.to_i
+          width   = element["width"].nil?   ? 0 : element["width"].value.to_i
+          
+          if url =~ /\Ahttps?:\/\//i && (height.zero? || width.zero?)
+            image   = get_image_size(url) 
+            next unless image
+          else
+            image = {:width => width, :height => height}
+          end
+          
+          image[:format] = File.extname(url).gsub(".", "")
+          
+          if tested_images.include?(url)
+            debug("Image was tested: #{url}")
+            next
+          end
+
+          tested_images.push(url)
+          if image_meets_criteria?(image)
+            list_images << url
+          else
+            debug("Image discarded: #{url} - height: #{image[:height]} - width: #{image[:width]} - format: #{image[:format]}")
+          end
+        end
+
+      (list_images.empty? and content != @html) ? images(@html, true) : list_images
+    end
+    
+    def images_with_fqdn_uris!(source_uri)
+      images_with_fqdn_uris(@html, source_uri)
+    end
+    
+    def images_with_fqdn_uris(document = @html.dup, source_uri)
+      uri = URI.parse(source_uri)
+      host = uri.host
+      scheme = uri.scheme
+      port = uri.port # defaults to 80
+
+      base = "#{scheme}://#{host}:#{port}/"
+
+      images = []
+      document.css("img").each do |elem|
+        begin
+          elem['src'] = URI.join(base,elem['src']).to_s if URI.parse(elem['src']).host == nil 
+          images << elem['src'].to_s
+        rescue URI::InvalidURIError => exc
+          elem.remove
+        end
+      end
+
+      images(document,true)
+    end
+
+    def get_image_size(url)
+      begin
+        w, h = FastImage.size(url)
+        raise "Couldn't get size." if w.nil? || h.nil?
+        {:width => w, :height => h}
+      rescue => e
+        debug("Image error: #{e}")
+        nil
+      end
+    end
+
+    def image_meets_criteria?(image)
+      return false if options[:ignore_image_format].include?(image[:format].downcase)
+      image[:width] >= (options[:min_image_width] || 0) && image[:height] >= (options[:min_image_height] || 0)
+    end
+
+    def title
+      title = @html.css("title").first
+      title ? title.text : nil
+    end
+
+    # Look through the @html document looking for the author
+    # Precedence Information here on the wiki: (TODO attach wiki URL if it is accepted)
+    # Returns nil if no author is detected
+    def author
+      # Let's grab this author:
+      # <meta name="dc.creator" content="Finch - http://www.getfinch.com" />
+      author_elements = @html.xpath('//meta[@name = "dc.creator"]')
+      unless author_elements.empty?
+        author_elements.each do |element|
+          if element['content']
+            return element['content'].strip
+          end
+        end
+      end
+
+      # Now let's try to grab this
+      # <span class="byline author vcard"><span>By</span><cite class="fn">Austin Fonacier</cite></span>
+      # <div class="author">By</div><div class="author vcard"><a class="url fn" href="http://austinlivesinyoapp.com/">Austin Fonacier</a></div>
+      author_elements = @html.xpath('//*[contains(@class, "vcard")]//*[contains(@class, "fn")]')
+      unless author_elements.empty?
+        author_elements.each do |element|
+          if element.text
+            return element.text.strip
+          end
+        end
+      end
+
+      # Now let's try to grab this
+      # <a rel="author" href="http://dbanksdesign.com">Danny Banks (rel)</a>
+      # TODO: strip out the (rel)?
+      author_elements = @html.xpath('//a[@rel = "author"]')
+      unless author_elements.empty?
+        author_elements.each do |element|
+          if element.text
+            return element.text.strip
+          end
+        end
+      end
+
+      author_elements = @html.xpath('//*[@id = "author"]')
+      unless author_elements.empty?
+        author_elements.each do |element|
+          if element.text
+            return element.text.strip
+          end
+        end
+      end
+    end
+
+    def content(remove_unlikely_candidates = :default)
+      @remove_unlikely_candidates = false if remove_unlikely_candidates == false
+
+      prepare_candidates
+      article = get_article(@candidates, @best_candidate)
+
+      cleaned_article = sanitize(article, @candidates, options)
+      if article.text.strip.length < options[:retry_length]
+        if @remove_unlikely_candidates
+          @remove_unlikely_candidates = false
+        elsif @weight_classes
+          @weight_classes = false
+        elsif @clean_conditionally
+          @clean_conditionally = false
+        else
+          # nothing we can do
+          return cleaned_article
+        end
+
+        make_html
+        content
+      else
+        cleaned_article
+      end
+    end
+
+    def get_article(candidates, best_candidate)
+      # Now that we have the top candidate, look through its siblings for content that might also be related.
+      # Things like preambles, content split by ads that we removed, etc.
+
+      sibling_score_threshold = [10, best_candidate[:content_score] * 0.2].max
+      output = Nokogiri::XML::Node.new('div', @html)
+      best_candidate[:elem].parent.children.each do |sibling|
+        append = false
+        append = true if sibling == best_candidate[:elem]
+        append = true if candidates[sibling] && candidates[sibling][:content_score] >= sibling_score_threshold
+
+        if sibling.name.downcase == "p"
+          link_density = get_link_density(sibling)
+          node_content = sibling.text
+          node_length = node_content.length
+
+          if node_length > 80 && link_density < 0.25
+            append = true
+          elsif node_length < 80 && link_density == 0 && node_content =~ /\.( |$)/
+            append = true
+          end
+        end
+
+        if append
+          sibling_dup = sibling.dup # otherwise the state of the document in processing will change, thus creating side effects
+          sibling_dup.name = "div" unless %w[div p].include?(sibling.name.downcase)
+          output << sibling_dup
+        end
+      end
+
+      output
+    end
+
+    def select_best_candidate(candidates)
+      sorted_candidates = candidates.values.sort { |a, b| b[:content_score] <=> a[:content_score] }
+
+      debug("Top 5 candidates:")
+      sorted_candidates[0...5].each do |candidate|
+        debug("Candidate #{candidate[:elem].name}##{candidate[:elem][:id]}.#{candidate[:elem][:class]} with score #{candidate[:content_score]}")
+      end
+
+      best_candidate = sorted_candidates.first || { :elem => @html.css("body").first, :content_score => 0 }
+      debug("Best candidate #{best_candidate[:elem].name}##{best_candidate[:elem][:id]}.#{best_candidate[:elem][:class]} with score #{best_candidate[:content_score]}")
+
+      best_candidate
+    end
+
+    def get_link_density(elem)
+      link_length = elem.css("a").map(&:text).join("").length
+      text_length = elem.text.length
+      link_length / text_length.to_f
+    end
+
+    def score_paragraphs(min_text_length)
+      candidates = {}
+      @html.css("p,td").each do |elem|
+        parent_node = elem.parent
+        grand_parent_node = parent_node.respond_to?(:parent) ? parent_node.parent : nil
+        inner_text = elem.text
+
+        # If this paragraph is less than 25 characters, don't even count it.
+        next if inner_text.length < min_text_length
+
+        candidates[parent_node] ||= score_node(parent_node)
+        candidates[grand_parent_node] ||= score_node(grand_parent_node) if grand_parent_node
+
+        content_score = 1
+        content_score += inner_text.split(',').length
+        content_score += [(inner_text.length / 100).to_i, 3].min
+
+        candidates[parent_node][:content_score] += content_score
+        candidates[grand_parent_node][:content_score] += content_score / 2.0 if grand_parent_node
+      end
+
+      # Scale the final candidates score based on link density. Good content should have a
+      # relatively small link density (5% or less) and be mostly unaffected by this operation.
+      candidates.each do |elem, candidate|
+        candidate[:content_score] = candidate[:content_score] * (1 - get_link_density(elem))
+      end
+
+      candidates
+    end
+
+    def class_weight(e)
+      weight = 0
+      return weight unless @weight_classes
+
+      if e[:class] && e[:class] != ""
+        if e[:class] =~ REGEXES[:negativeRe]
+          weight -= 25
+        end
+
+        if e[:class] =~ REGEXES[:positiveRe]
+          weight += 25
+        end
+      end
+
+      if e[:id] && e[:id] != ""
+        if e[:id] =~ REGEXES[:negativeRe]
+          weight -= 25
+        end
+
+        if e[:id] =~ REGEXES[:positiveRe]
+          weight += 25
+        end
+      end
+
+      weight
+    end
+
+    def score_node(elem)
+      content_score = class_weight(elem)
+      case elem.name.downcase
+        when "div"
+          content_score += 5
+        when "blockquote"
+          content_score += 3
+        when "form"
+          content_score -= 3
+        when "th"
+          content_score -= 5
+      end
+      { :content_score => content_score, :elem => elem }
+    end
+
+    def debug(str)
+      puts str if options[:debug]
+    end
+
+    def remove_unlikely_candidates!
+      @html.css("*").each do |elem|
+        str = "#{elem[:class]}#{elem[:id]}"
+        if str =~ REGEXES[:unlikelyCandidatesRe] && str !~ REGEXES[:okMaybeItsACandidateRe] && (elem.name.downcase != 'html') && (elem.name.downcase != 'body')
+          debug("Removing unlikely candidate - #{str}")
+          elem.remove
+        end
+      end
+    end
+
+    def transform_misused_divs_into_paragraphs!
+      @html.css("*").each do |elem|
+        if elem.name.downcase == "div"
+          # transform <div>s that do not contain other block elements into <p>s
+          if elem.inner_html !~ REGEXES[:divToPElementsRe]
+            debug("Altering div(##{elem[:id]}.#{elem[:class]}) to p");
+            elem.name = "p"
+          end
+        else
+          # wrap text nodes in p tags
+#          elem.children.each do |child|
+#            if child.text?
+#              debug("wrapping text node with a p")
+#              child.swap("<p>#{child.text}</p>")
+#            end
+#          end
+        end
+      end
+    end
+
+    def sanitize(node, candidates, options = {})    
+      node.css("h1, h2, h3, h4, h5, h6").each do |header|
+        header.remove if class_weight(header) < 0 || get_link_density(header) > 0.33
+      end
+
+      node.css("form, object, iframe, embed").each do |elem|
+        elem.remove
+      end
+
+      if @options[:remove_empty_nodes]
+        # remove <p> tags that have no text content - this will also remove p tags that contain only images.
+        node.css("p").each do |elem|
+          elem.remove if elem.content.strip.empty?
+        end
+      end
+
+      # Conditionally clean <table>s, <ul>s, and <div>s
+      clean_conditionally(node, candidates, "table, ul, div")
+
+      # We'll sanitize all elements using a whitelist
+      base_whitelist = @options[:tags] || %w[div p]
+      # We'll add whitespace instead of block elements,
+      # so a<br>b will have a nice space between them
+      base_replace_with_whitespace = %w[br hr h1 h2 h3 h4 h5 h6 dl dd ol li ul address blockquote center]
+
+      # Use a hash for speed (don't want to make a million calls to include?)
+      whitelist = Hash.new
+      base_whitelist.each {|tag| whitelist[tag] = true }
+      replace_with_whitespace = Hash.new
+      base_replace_with_whitespace.each { |tag| replace_with_whitespace[tag] = true }
+
+      ([node] + node.css("*")).each do |el|
+        # If element is in whitelist, delete all its attributes
+        if whitelist[el.node_name]
+          el.attributes.each { |a, x| el.delete(a) unless @options[:attributes] && @options[:attributes].include?(a.to_s) }
+
+          # Otherwise, replace the element with its contents
+        else
+          # If element is root, replace the node as a text node
+          if el.parent.nil?
+            node = Nokogiri::XML::Text.new(el.text, el.document)
+            break
+          else
+            if replace_with_whitespace[el.node_name]
+              el.swap(Nokogiri::XML::Text.new(' ' << el.text << ' ', el.document))
+            else
+              el.swap(Nokogiri::XML::Text.new(el.text, el.document))
+            end
+          end
+        end
+
+      end
+
+      s = Nokogiri::XML::Node::SaveOptions
+      save_opts = s::NO_DECLARATION | s::NO_EMPTY_TAGS | s::AS_XHTML
+      html = node.serialize(:save_with => save_opts)
+
+      # Get rid of duplicate whitespace
+      return html.gsub(/[\r\n\f]+/, "\n" )
+    end
+
+    def clean_conditionally(node, candidates, selector)
+      return unless @clean_conditionally
+      node.css(selector).each do |el|
+        weight = class_weight(el)
+        content_score = candidates[el] ? candidates[el][:content_score] : 0
+        name = el.name.downcase
+        
+        if weight + content_score < 0
+          el.remove
+          debug("Conditionally cleaned #{name}##{el[:id]}.#{el[:class]} with weight #{weight} and content score #{content_score} because score + content score was less than zero.")
+        elsif el.text.count(",") < 10
+          counts = %w[p img li a embed input].inject({}) { |m, kind| m[kind] = el.css(kind).length; m }
+          counts["li"] -= 100
+
+          # For every img under a noscript tag discount one from the count to avoid double counting
+          counts["img"] -= el.css("noscript").css("img").length
+                
+          content_length = el.text.strip.length  # Count the text length excluding any surrounding whitespace
+          link_density = get_link_density(el)
+          to_remove = false
+          reason = ""
+          
+          if (counts["img"] > counts["p"]) && (counts["img"] > 1)
+            reason = "too many images"
+            to_remove = true
+          elsif counts["li"] > counts["p"] && name != "ul" && name != "ol"
+            reason = "more <li>s than <p>s"
+            to_remove = true
+          elsif counts["input"] > (counts["p"] / 3).to_i
+            reason = "less than 3x <p>s than <input>s"
+            to_remove = true
+          elsif (content_length < options[:min_text_length]) && (counts["img"] != 1)
+            reason = "too short a content length without a single image"
+            to_remove = true
+          elsif weight < 25 && link_density > 0.2
+            reason = "too many links for its weight (#{weight})"
+            to_remove = true
+          elsif weight >= 25 && link_density > 0.5
+            reason = "too many links for its weight (#{weight})"
+            to_remove = true
+          elsif (counts["embed"] == 1 && content_length < 75) || counts["embed"] > 1
+            reason = "<embed>s with too short a content length, or too many <embed>s"
+            to_remove = true
+          end
+
+          if to_remove
+            debug("Conditionally cleaned #{name}##{el[:id]}.#{el[:class]} with weight #{weight} and content score #{content_score} because it has #{reason}.")
+            el.remove
+          end
+        end
+      end
+    end
+  end
+end