auto_excerpt 0.6.2 → 0.6.3

data/.gitignore

@@ -1,3 +1,5 @@
 *.php
 *.zip
 extra
+*.gemspec
+pkg

data/CHANGELOG

@@ -0,0 +1,8 @@
+== 0.6.3 (2010-01-26)
+  * Removed limit by :characters for the time being
+  * Improved limit by characters to be more accurate
+  * Changed method of stripping HTML
+  * Changed Regexp used for closing unclosed HTML tags
+  
+== 0.6.2 (2010-01-20)
+  * Original cleanup of what was already written and turning it into a gem.

data/README.textile

@@ -1,5 +1,7 @@
 h1. AutoExcerpt
 
+pre. [sudo] gem install auto_excerpt
+
 Creates Automatic excerpts of html formatted text.
 
 pre. AutoExcerpt.new("<span>This is <strong>some</strong> fancy html formatted text homie</span>", {:words => 5}) 
@@ -8,37 +10,42 @@ pre. AutoExcerpt.new("<span>This is <strong>some</strong> fancy html formatted t
 h3. Features
 
   * There are 4 different ways to limit the length of an excerpt: *characters*, *words*, *sentences*, *paragraphs*
-  * If limiting by *characters*, the gem will ensure that the excerpt does not cutoff in the middle of a word. If the excerpt would be shorter than the limit that is set, the entire text will be shown.
+  * If the excerpt would be shorter than the limit that is set, the entire text will be shown.
   * HTML can be stripped
   * HTML tags are automatically closed.
 
 h3. Options
 
-*:characters*
+@:characters@
 The number of characters to display from the text.
 Default: 150 (does not need to be reset to 0 if you choose another option)
 
-*:words*, *:sentences*, *:paragraphs*
+If you need to be 100% accurate in your character count, then remove the @:ending@
+
+pre. AutoExcerpt.new("<h1>Hello World!</h1>", {:characters => 5, :ending => nil}) 
+     # => <h1>Hello</h1>
+
+@:words@, @:sentences@, @:paragraphs@
 The number of [which] to display from the text.
 
-*:ending*
+@:ending@
 The text that will be displayed at the end of the excerpt when generating the excerpt by length or words. Set to @nil@ if you don't want it.
 Default: "..."
 
 pre. AutoExcerpt.new("This is cool stuff man!", :ending => ". Srsly!", :words => 3)
     # => "This is cool. Srsly!"
     
-*:strip_html*
+@:strip_html@
 Strips HTML tags from the excerpt that is displayed.
 Default: false
 
-*:strip_paragraphs*
+@:strip_paragraphs@
 Strip all paragraph tags from the html.
 Default: false
 
-*:skip_characters*, *:skip_words*, *:skip_sentences*, *:skip_paragraphs*
+@:skip_words@, @:skip_sentences@, @:skip_paragraphs@
 The number of [which] to skip at the beginning of the html when returned.
-Default 0
+Default: 0
 
 h2. Help out on Github!
 

data/Rakefile

@@ -11,6 +11,9 @@ begin
     gemspec.email = "kabari@gmail.com"
     gemspec.homepage = "http://github.com/kabari/auto_excerpt"
     gemspec.authors = ["Kabari Hendrick"]
+    
+    gemspec.add_development_dependency "rspec", ">= 1.2.9"
+    gemspec.add_development_dependency "yard", ">= 0"
   end
   Jeweler::GemcutterTasks.new
 rescue LoadError
@@ -21,7 +24,7 @@ end
 desc "spec"
 Spec::Rake::SpecTask.new do |t|
   t.libs << "spec"
-  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.spec_files = FileList['spec/*_spec.rb']
 end
 
 
@@ -29,7 +32,7 @@ begin
   require 'yard'
   YARD::Rake::YardocTask.new
 rescue LoadError
-  task :yardoc do
+  task :yard do
     abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
   end
 end

data/VERSION

@@ -1 +1 @@
-0.6.2
+0.6.3

data/lib/auto_excerpt.rb

@@ -9,50 +9,60 @@ class String
 end
 
 # TODO allow for default options to be set.
-class AutoExcerpt < String
-  VERSION = '0.6'
-  
+class AutoExcerpt < String  
   DEFAULTS = {
      :characters => 0,
      :words => 0,
      :sentences => 0, 
      :paragraphs => 0,
-     :skip_characters => 0,
+     # :skip_characters => 0,
      :skip_words => 0,
      :skip_sentences => 0,
      :skip_paragraphs => 0,
      :ending => '...',
-     :strip_html => false, #:allowed_tags => [],
+     :strip_html => false, :allowed_tags => [],
      :strip_breaks_tabs => false,
      :strip_paragraphs => false
-  }.freeze
+  }
   
   # TODO add and allowwed tags option
-  HTMLTAGS = /<(.|\n)+?>/
   PUNCTUATION_MARKS = /\!\s|\.\s|\?\s/
   NO_CLOSE = %w( br hr img input ) # tags that do not have opposite closing tags
-
+  OPENING_TAG = /<([a-z0-9]{1,})\b[^>]*>/im
+  CLOSING_TAG = /<\/([a-z0-9]{1,})>/im
+  
   # @param [String] text The text to be excerpted
   # @param [Hash] settings The settings for creating the excerpt
-  # @option settings [Integer] :characters (0)
-  # 
-  # 
+  # @option settings [Integer] :characters (0) The number of characters to limit the html by
+  # @option settings [Integer] :words (0) The number of words to limit the html by
+  # @option settings [Integer] :sentences (0) The number of sentences to limit the html by
+  # @option settings [Integer] :paragraphs (0) The number of paragraphs to limit the html by
+  # @option settings [Integer] :skip_characters (0) The number of characters to skip from the start of the html
+  # @option settings [Integer] :skip_words (0) The number of words to skip from the start of the html
+  # @option settings [Integer] :skip_sentences (0) The number of sentences to skip from the start of the html
+  # @option settings [Integer] :skip_paragraphs (0) The number of paragraphs to skip from the start of the html
+  # @option settings [String] :ending ('...') A string added to the end of the excerpt
+  # @option settings [Boolean] :strip_html (false) Strip all HTML from the text before creating the excerpt
+  # @option settings [Boolean] :strip_paragraphs (false) Strip all <p> tags from the HTML before creating the excerpt
   def initialize(text, settings = {})
-    @settings = DEFAULTS.dup.merge(settings)
+    @settings = Marshal.load(Marshal.dump(DEFAULTS)).merge(settings)
     
     # make our copy
     @body = text.dup.strip
     @excerpt = ""
-    @body.gsub!(HTMLTAGS, "") if @settings[:strip_html]
-    @body.clean! if @settings[:strip_breaks_tabs]
+    
+    if @settings[:strip_html]
+      (@settings[:allowed_tags] << "p") if @settings[:paragraphs] > 0 # don't stip P tags if that is the limiter
+      @body = strip_html(@body) 
+    end
+    @body = @body.clean if @settings[:strip_breaks_tabs]
     # TODO replace this with better regex
     @body.replace(@body.gsub(/<(\/|)p>/,'')) if @settings[:strip_paragraphs]
-    # @charcount = @body.gsub(HTMLTAGS, "").length
-    @charcount = @body.length
-    @wordcount = @body.gsub(HTMLTAGS, "").scan(/\w+/).size
+    @charcount = strip_html(@body).length
+    @wordcount = strip_html(@body).scan(/\w+/).size
     @sencount  = @body.split(PUNCTUATION_MARKS).size
     @pghcount  = @body.split("</p>").size
-    @settings[:characters] = 150 if @settings.values_at(:characters, :words, :sentences, :paragraphs).all?{|val| val.zero?  }
+    @settings[:characters] = 150 if @settings.values_at(:characters, :words, :sentences, :paragraphs).all?{|val| val.zero? || val.nil?  }
     
     create_excerpt
     super(@excerpt)
@@ -60,35 +70,42 @@ class AutoExcerpt < String
       
   
   protected
+  
   attr_reader :charcount, :wordcount, :sencount, :pghcount
   attr_accessor :settings, :body, :excerpt
   
  # close html tags
  # TODO make this work with new strip_html method. Improve regex
   def close_tags(text)
-    tagstoclose = ""
-    tags = []
-    opentags = text.scan( /<(([A-Z]|[a-z]).*?)(( )|(>))/is ).transpose[0] || []
-    opentags.reverse!
-    closedtags = text.scan(/<\/(([A-Z]|[a-z]).*?)(( )|(>))/is).transpose[0] || []
+    # Don't bother closing tags if html is stripped since there are no tags.
+    if @settings[:strip_html] && @settings[:allowed_tags].empty?
+      tagstoclose = nil
+    else
+      tagstoclose = ""
+      tags = []
+      # /<(([A-Z]|[a-z]).*?)(( )|(>))/is
+      # /<\/(([A-Z]|[a-z]).*?)(( )|(>))/is
+      opentags = text.scan(OPENING_TAG).transpose[0] || []
+      opentags.reverse!
+      closedtags = text.scan(CLOSING_TAG).transpose[0] || []
   
-    opentags.each do |ot|
-      if closedtags.include?(ot)
-        closedtags.delete_at(closedtags.index(ot))
-      else
-        tags << ot
+      opentags.each do |ot|
+        if closedtags.include?(ot)
+          closedtags.delete_at(closedtags.index(ot))
+        else
+          tags << ot
+        end
       end
-    end
     
-    tags.each do |tag|
-      tagstoclose << "</#{tag.strip.downcase}>" unless NO_CLOSE.include?(tag)
+      tags.each do |tag|
+        tagstoclose << "</#{tag.strip.downcase}>" unless NO_CLOSE.include?(tag)
+      end      
     end
     
-    text << (@settings[:ending] || "") + tagstoclose
-    @excerpt = text
+    @excerpt = [text, @settings[:ending], tagstoclose].compact.join
   end
     
-  def create_excerpt #:nodoc:
+  def create_excerpt
     return characters unless @settings[:characters].zero?
     return words      unless @settings[:words].zero?
     return sentences  unless @settings[:sentences].zero?
@@ -99,13 +116,28 @@ class AutoExcerpt < String
     @settings[:ending] = nil
     close_tags(@body)
   end
-  
+
   # limit by characters
+  # @todo make this work with skip characters
   def characters
     return non_excerpted_text if @charcount < @settings[:characters]
-    text = @body[@settings[:skip_characters]...@settings[:characters]].split(" ")
-    text.pop if text.length > 1
-    text = text.join(" ")
+    text = ""
+    html_count = char_count = 0
+    start_end_tags = /#{Regexp.union(/(<[a-z0-9]{1,}\b[^>]*>)/,/(<\/[a-z0-9]{1,}>)/)}/io
+    @body.split(start_end_tags).each do |piece|
+      if piece =~ start_end_tags
+        html_count += piece.length
+      else
+        chars_left = @settings[:characters] - char_count
+        # TODO don't clip the middle of a word
+        # unless piece[0...(chars_left+1)] =~ /(\s|\W)$/
+        #   chars_left += 1 until piece[0...chars_left] =~ /(\s|\W)$/
+        # end
+        char_count += piece[0...chars_left].length
+      end
+      break if (char_count >= @settings[:characters])
+    end 
+    text = @body[0...(html_count+char_count)]
     close_tags(text)
   end
   
@@ -119,6 +151,7 @@ class AutoExcerpt < String
   # limit by sentences
   def sentences
     return non_excerpted_text if @sencount < @settings[:sentences]
+    # TODO don't change punctuation
     text = @body.split(PUNCTUATION_MARKS).slice(@settings[:skip_sentences], @settings[:sentences]).join(". ")
     close_tags(text)
   end
@@ -128,24 +161,24 @@ class AutoExcerpt < String
     return non_excerpted_text if @pghcount < @settings[:paragraphs]
     text = @body.split("</p>").slice(@settings[:skip_paragraphs], @settings[:paragraphs])
     @settings[:ending] = nil
-    # text.last.replace(text.last.rstrip.concat(@settings.delete(:ending)))
     text = text.join("</p>")
     close_tags(text)
   end
   
   # Removes HTML tags from a string. Allows you to specify some tags to be kept.
   # @see http://codesnippets.joyent.com/posts/show/1354#comment-293
-  # def strip_html(html)    
-  #   reg = if @settings[:allowed_tags].any?
-  #     Regexp.new(
-  #       %(<(?!(\\s|\\/)*(#{
-  #         @settings[:allowed_tags].map {|tag| Regexp.escape( tag )}.join( "|" )
-  #       })( |>|\\/|'|"|<|\\s*\\z))[^>]*(>+|\\s*\\z)),
-  #       Regexp::IGNORECASE | Regexp::MULTILINE, 'u'
-  #     )
-  #   else
-  #     /<[^>]*(>+|\s*\z)/m
-  #   end
-  #   html.gsub(reg,'')
-  # end
+  def strip_html(html)
+    allowed = @settings[:allowed_tags]
+    reg = if allowed.any?
+      Regexp.new(
+        %(<(?!(\\s|\\/)*(#{
+          allowed.map {|tag| Regexp.escape( tag )}.join( "|" )
+        })( |>|\\/|'|"|<|\\s*\\z))[^>]*(>+|\\s*\\z)),
+        Regexp::IGNORECASE | Regexp::MULTILINE, 'u'
+      )
+    else
+      /<[^>]*(>+|\s*\z)/m
+    end
+     @stripped_html ||= html.gsub(reg,'')
+  end
 end

data/spec/auto_excerpt_spec.rb

@@ -1,49 +1,60 @@
 require File.expand_path(File.dirname(__FILE__) + "/spec_helper")
-# require File.join(File.dirname(__FILE__), "shared_strip_tags")
+require File.join(File.dirname(__FILE__), *%w[shared strip_html_spec])
 
 # I definitely need more tests
 describe AutoExcerpt do
-  include AutoExcerptHelpers
   
- it "should limit characters" do
-   text = html_excerpt({:characters => 5})
+  it "should limit characters" do
+   text = html_excerpt({:characters => 5, :ending => nil})
    stripped_text(text).length.should eql(5)
-   
-   text = heavy_excerpt({:characters => 5})
-   stripped_text(text).length.should eql(5)
- end
- 
- it "should default to 150 characters" do
-   text = html_excerpt
-   text.length.should be_close(150, 4)   
- end
  
- it "should limit words" do
+   text = heavy_excerpt({:characters => 5, :ending => nil})
+   stripped_text(text).length.should eql(5)
+  end
+
+  it "should default to 150 characters" do
+   text = html_excerpt(:ending => nil)
+   stripped_text(text).length.should eql(150)
+  end
+
+  it "does not include html tags in character count" do
+    AutoExcerpt.new("<h1>Hello World!</h1>", {:characters => 5, :ending => nil}).should == "<h1>Hello</h1>"
+  end
+  
+  it "should not cutoff in the middle of a word" do
+    pending("this does not work yet") do
+      AutoExcerpt.new("<h1>Hello World!</h1>", {:characters => 4, :ending => nil}).should == "<h1>Hello</h1>"
+    end
+  end
+  
+  it "should limit words" do
    text = html_excerpt({:words => 5})
    stripped_text(text).split(" ").length.should eql(5)
-   
+ 
    text = heavy_excerpt({:words => 5})
    stripped_text(text).split(" ").length.should eql(5)
-   
- end
  
- it "should limit sentences" do
+  end
+
+  it "should limit sentences" do
    text = html_excerpt({:sentences => 3})
-   stripped_text(text).split(AutoExcerpt::PUNCTUATION_MARKS).length.should eql(3)
-   
-   text = heavy_excerpt({:sentences => 3})
-   stripped_text(text).split(AutoExcerpt::PUNCTUATION_MARKS).length.should eql(3)
-   
- end
+   text.should == %{<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur...</p>}
  
- it "should limit paragraphs" do
+   # text = heavy_excerpt({:sentences => 3})
+   # text.should == %{<p>Alright&hellip;ok&hellip;that title is a bold faced lie. I don&rsquo;t give a damn about <acronym title="Cascading Style Sheets">CSS</acronym> validation! Being a designer for a living, you have to know when to ditch some of these &lsquo;web 2.0&rsquo; type fads...</p>} 
+  end
+
+  it "should limit paragraphs" do
    text = html_excerpt({:paragraphs => 1})
    stripped_text(text).split("</p>").length.should eql(1)
+  end
+
+  it "should not include the :ending when limited by paragraph" do
+   text = html_excerpt({:paragraphs => 1})
    stripped_text(text).split("</p>").last[-3..-1].should_not == '...'
- end
-     
- 
- it "should close the effin B tag" do
+  end
+
+  it "should close the effin B tag" do
    t = %{
      <strong>
      left a greeting on Kroogi page of <a href="http://localhost:3000/user/sasha/" frwrd_type="user" title="sasha">sasha</a>
@@ -55,7 +66,15 @@ describe AutoExcerpt do
    }
    text = AutoExcerpt.new(t,{:characters => 270})
    text.match(/(<(\/|)b>)/).captures.length.should eql(2)
- end
- 
- # it_should_behave_like "HTML stripper"
+  end
+  
+end
+
+describe AutoExcerpt, "when stripping HTML" do
+  
+  it_should_behave_like "an HTML stripper"
+  
+  it "should not strip P tags if :paragraphs option is set" do
+    AutoExcerpt.new("<p>this is a paragraph.</p><p>this is also a paragraph.</p>",{:paragraphs => 1, :strip_html => true}).should eql("<p>this is a paragraph.</p>")
+  end
 end

data/spec/{shared_strip_tags.rb → shared/strip_html_spec.rb}

@@ -1,11 +1,13 @@
-describe "HTML stripper", :shared => true do
+describe "an HTML stripper", :shared => true do
+  
   before(:all) do
     @html = "foo < BAR> <><a href=\"www.beer.com\">xyzzy</a>>><br /><p><br/><p><br///><p></><br ///><p></br><p></ br> <cheeky"
+    
   end
-  
   it "should strip html tags" do
     AutoExcerpt.new(@html, {:strip_html => true}).should == "foo  xyzzy "
   end
+  
   # cheeky keeper
   it "should allow given tags" do
     AutoExcerpt.new(@html, {:strip_html => true, :allowed_tags => %w(bar br ) }).should == "foo < BAR> xyzzy<br /><br/><br///><br ///></br></ br> " 
@@ -14,8 +16,8 @@ describe "HTML stripper", :shared => true do
   it "should treat unclosed tags at the end of the document as tags to be safe" do
     AutoExcerpt.new(@html, {:strip_html => true, :allowed_tags => %w(cheeky monkey) }).should == "foo  xyzzy <cheeky"
     AutoExcerpt.new("pass< cheeky", {:strip_html => true, :allowed_tags => %w(cheeky) }).should == "pass< cheeky"
-    AutoExcerpt.new("pass< cheeky ", {:strip_html => true, :allowed_tags => %w(bar br ) }).should == "pass< cheeky "
-    AutoExcerpt.new("pass< cheeky  ", {:strip_html => true, :allowed_tags => %w(cheeky) }).should == "pass< cheeky  "
+    AutoExcerpt.new("pass< cheeky ", {:strip_html => true, :allowed_tags => %w(cheeky) }).should == "pass< cheeky"
+    AutoExcerpt.new("pass< cheeky  ", {:strip_html => true, :allowed_tags => %w(cheeky) }).should == "pass< cheeky"
   end
 
   it "should treat quotes, less thans and shashes as tag word terminators to be conservative" do
@@ -30,6 +32,7 @@ describe "HTML stripper", :shared => true do
   it "should know the difference between a tag called a and one called alpha" do
     difference_test = "passed<alpha> the test"
     AutoExcerpt.new(difference_test, {:strip_html => true, :allowed_tags => %w(a) }).should == "passed the test"
-    AutoExcerpt.new(difference_test, {:strip_html => true, :allowed_tags => %w(alpha) }).should == difference_test
+    AutoExcerpt.new(difference_test, {:strip_html => true, :allowed_tags => %w(alpha) }).should == "passed<alpha> the test</alpha>"
   end
+  
 end

data/spec/spec.opts

@@ -1,3 +1,2 @@
 --colour
---loadby random
 --backtrace

data/spec/spec_helper.rb

@@ -1,12 +1,22 @@
 require "rubygems"
 require "spec"
-require File.join(File.dirname(__FILE__), *%w[.. lib auto_excerpt])
+require "pp"
+require "webrick/htmlutils"
 
-Spec::Runner.configure do |config|
-  
+Object.class_eval do
+  alias_method :old_pp, :pp
+
+  def pp(str)
+    str = WEBrick::HTMLUtils.escape(str) if str.is_a?(String)
+    old_pp(str)
+  end
 end
 
+require File.join(File.dirname(__FILE__), *%w[.. lib auto_excerpt])
+
 module AutoExcerptHelpers
+  
+  
   def html_excerpt(opts = {})
    AutoExcerpt.new(HTML_BLOCK, opts)
   end
@@ -20,8 +30,10 @@ module AutoExcerptHelpers
   end
 
   def stripped_text(t)
-   t.gsub(AutoExcerpt::HTMLTAGS, "")
+   t.gsub(/<[^>]*(>+|\s*\z)/m, "")
   end
+    
+  CRAP_HTML = ""
   
   NORMAL_TEXT = %{Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
     
@@ -43,4 +55,8 @@ module AutoExcerptHelpers
 
   	<p>Use the validation tool on your basic style sheets if your having a hard time figuring out what may be causing an issue. At this stage in development they are helpful tools for just that. By no means should you be running them as the final judge on your styles, or checking everyone else&rsquo;s sites to &lsquo;see if it validates&rsquo; so you can feel all professional, or posting the &lsquo;valid CSS&rsquo; emblem at the bottom of your web pages like it&rsquo;s a flag on the moon.</p>
   }
-end
+end
+
+Spec::Runner.configure do |config|
+  config.include AutoExcerptHelpers
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: auto_excerpt
 version: !ruby/object:Gem::Version 
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors: 
 - Kabari Hendrick
@@ -9,10 +9,29 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-20 00:00:00 -06:00
+date: 2010-01-26 00:00:00 -06:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.9
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: yard
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
 description: Create excerpts from html formatted text. HTML tags are automatically closed.
 email: kabari@gmail.com
 executables: []
@@ -32,7 +51,7 @@ files:
 - browser_test/browser_test.rb
 - lib/auto_excerpt.rb
 - spec/auto_excerpt_spec.rb
-- spec/shared_strip_tags.rb
+- spec/shared/strip_html_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
 has_rdoc: true
@@ -65,5 +84,5 @@ specification_version: 3
 summary: Create excerpts from html formatted text.
 test_files: 
 - spec/auto_excerpt_spec.rb
-- spec/shared_strip_tags.rb
+- spec/shared/strip_html_spec.rb
 - spec/spec_helper.rb