htmlclipping 0.1.0

data/lib/htmlclipping.rb

@@ -0,0 +1,243 @@
+# HtmlClipping generates excerpts from an HTML page that has a link pointing to 
+# a particular URI. It removes most HTML markup, bolds the link text, and 
+# trims the resulting text to a fixed number of characters. I developed it to 
+# help me track referers to my website, though I suppose it might have other 
+# uses.
+#
+# For example, the following script gets the HTML at http://rubyforge.org/credits/, and forms an excerpt around the link to http://www.rubycentral.org/pledge/.
+#
+#   require 'htmlclipping'
+#   require 'net/http'
+#
+#   contents = ''
+#   Net::HTTP.start( 'rubyforge.org' ) do |http|
+#     response = http.get '/credits/'
+#     contents = response.body
+#   end
+#   clipping = HtmlClipping.new(
+#     contents, 'http://www.rubycentral.org/pledge/', 500
+#   )
+#   puts clipping.to_s
+#
+#   => "… RubyForge takes time, effort, and money. Many thanks to the
+#      folks listed below who are making it possible! <br /> If RubyForge has
+#      been helpful to you, and you want to give something back to the Ruby
+#      community, please consider supporting <strong>RubyCentral</strong>.
+#      Thanks! <br /> InfoEther, Inc purchased the RubyForge hardware and
+#      provides system administration support. <br /> Several folks provide
+#      file mirrors to help share the bandwidth load: <br /> Evan Webb <br />
+#      Dennis Oelkers <br /> Austin &#8230;"
+#
+# The Rubyforge project page can be found at http://rubyforge.org/projects/htmlclipping.
+
+require 'iconv'
+
+class HtmlClipping
+	Version = '0.1.0'
+	
+	# html:: The HTML of the referring web page.
+	# referred_uri:: The URI that is being referred to.
+	# excerpt_limit:: The maximum size of the resulting clipping
+	def initialize( html, referred_uri, excerpt_limit )
+		@contents = html
+		if @contents =~ %r{<meta.*text/html; charset=(.*?)('|")}i
+			@converter = Iconv.new( 'UTF-8', $1 )
+		end
+		@referred_uri = referred_uri
+		@excerpt_limit = excerpt_limit
+	end
+	
+	def convert( str ) # :nodoc:
+		@converter ? @converter.iconv( str ) : str
+	end
+	
+	# Returns the clipping as a string suitable for use as XML text.
+	def to_s
+		if @contents =~ %r{<body[^>]*>(.*)</body>}mi
+			to_detokenize = $1
+		else
+			@contents =~ %r{<body[^>]*>(.*)}mi
+			to_detokenize = $1
+		end
+		excerpt = Detokenizer.new( to_detokenize, @referred_uri ).execute
+		excerpt = Clipper.new( excerpt, @excerpt_limit ).execute
+		excerpt.gsub( /[\200-\377]/ ) { |c| "&#%04d;" % c[0] }
+		convert( excerpt )
+	end
+
+	class ArrayOfWordsAndBrTags < Array # :nodoc:
+		def initialize( text )
+			super()
+			text.split( %r{(<br />)} ).each { |br_or_between_br|
+				if br_or_between_br == '<br />'
+					self << br_or_between_br
+				else
+					self.concat( br_or_between_br.split )
+				end
+			}
+		end
+	end
+
+	class Clipper # :nodoc:
+		def initialize( excerpt, excerpt_limit )
+			@excerpt = excerpt
+			@excerpt_limit = excerpt_limit
+		end
+		
+		def add_next_word
+			if @words_before.empty?
+				add_from_words_after
+			elsif @words_after.empty?
+				add_from_words_before
+			elsif @str_after.size >= @str_before.size
+				add_from_words_before
+			else
+				add_from_words_after
+			end
+		end
+		
+		def add_next_word?
+			( excerpt_parts_under_limit &&
+			  !( @words_before.empty? && @words_after.empty? ) )
+		end
+
+		def add_from_words_after
+			if @str_after == ''
+				@str_after = @words_after.shift
+			else
+				@str_after += ' ' + @words_after.shift
+			end
+		end
+
+		def add_from_words_before
+			if @str_before == ''
+				@str_before = @words_before.pop
+			else
+				@str_before = @words_before.pop + ' ' + @str_before
+			end
+		end
+		
+		def excerpt_parts_under_limit
+			( @str_before.size + @strong_text.size + @str_after.size + 2 ) <
+			  @excerpt_limit
+		end
+
+		def execute
+			@excerpt =~ %r{(.*)(<strong>.*?</strong>\S*)(.*)}m
+			@text_before = $1
+			@strong_text = $2
+			@text_after = $3
+			get_elts.join( ' ' )
+		end
+
+		def get_elts
+			@words_before = ArrayOfWordsAndBrTags.new( @text_before )
+			@words_after = ArrayOfWordsAndBrTags.new( @text_after )
+			@str_before = ''
+			@str_after = ''
+			add_next_word while add_next_word?
+			ellipses = '&#8230;'
+			elts = []
+			elts << ellipses unless @words_before.empty?
+			elts << @str_before
+			elts << @strong_text
+			elts << @str_after
+			elts << ellipses unless @words_after.empty?
+			elts
+		end
+	end
+
+	class Detokenizer # :nodoc:
+		def initialize( contents, referred_uri )
+			@contents, @referred_uri = contents, referred_uri;
+		end
+
+		def compact_brs
+			result = 'something'
+			until result.nil?
+				result = @detokenized.gsub!( %r{<br />\s*<br />}, '<br /> ' )
+			end
+		end
+		
+		def execute
+			@detokenized = HtmlWithFixedAttributes.new( @contents )
+			@detokenized.gsub!( /<!--.*?-->/m, '' )
+			@detokenized.gsub!(
+				%r{</?(h\d|p|blockquote|table|tr|br|div|form|ul|li|center|ol|dl|dd|dt|fieldset|option|select|object|o:p).*?>}i,
+				' <br /> '
+			)
+			@detokenized.gsub!(
+				%r{</?(acronym|abbr|strong|td|tt|small|em|img|font|span|input|hr|noscript|legend|address).*?>}im, ''
+			)
+			@detokenized.gsub!( %r{</?(b|i)(\s+.*?)?>}i, '' )
+			substitute_links
+			compact_brs
+			@detokenized.gsub!( /\s+/, ' ' )
+			@detokenized
+		end
+		
+		def substitute_links
+			@detokenized.gsub!( %r{<a\s+(.*?)>(.*?)</(xhtml:)?a.*?>}mi ) { |s|
+				a_att = $1
+				a_body = $2
+				if ( a_att =~ /href=('|")?([^'" ]*)("|')?/i )
+					href = $2
+					href.gsub!( /("|')$/, '' )
+					matches_self = ( href == @referred_uri )
+				else
+					matches_self = false
+				end
+				matches_self ? "<strong>#{ a_body }</strong>" : a_body
+			}
+		end
+	end
+
+	class HtmlWithFixedAttributes < String # :nodoc:
+		def initialize( contents )
+			super( '' )
+			script_stack = []
+			contents.split( %r{(</?script[^>]*>)}i ).each { |script_or_between|
+				if script_or_between =~ /^<script/i
+					script_stack.push( script_or_between )
+				elsif script_or_between =~ %r{</script}i
+					script_stack.pop
+				elsif script_stack.empty?
+					self << fix_attributes( script_or_between )
+				else
+					self << ' '
+				end
+			}
+		end
+
+		def fix_attributes( html_part )
+			fixed = ''
+			html_part.split( /(<.*?)(?=<)/ ).each { |part|
+				part = fix_bracket_plus( part ) if part != '' and part =~ /('|")/
+				fixed << part
+			}
+			fixed
+		end
+		
+		def fix_bracket_plus( part )
+			inside_attribute = false
+			between_tags = true
+			last_attribute_quote = nil
+			fixed = ''
+			part.scan( /([^'"<>]*)("|'|<|>)/ ) { |pre_delimiter, delimiter|
+				delimiter.gsub!( />/, '&gt;' ) if inside_attribute
+				if %w( ' " ).include? delimiter and !between_tags
+					if last_attribute_quote == delimiter or !inside_attribute
+						inside_attribute = !inside_attribute
+						last_attribute_quote = delimiter
+					end
+				end
+				between_tags = !between_tags if %w( < > ).include? delimiter
+				fixed << pre_delimiter
+				fixed << delimiter
+			}
+			post_scan = ( part =~  /.*('|"|<|>)(.*)/m ) ? $2 : part
+			fixed << post_scan
+			fixed
+		end
+	end
+end

data/lib/htmlclipping.rb~

@@ -0,0 +1,208 @@
+require 'iconv'
+
+class HtmlClipping
+	Version = '0.1.0'
+	
+	def initialize( html, referred_uri, excerpt_limit )
+		@contents = html
+		if @contents =~ %r{<meta.*text/html; charset=(.*?)('|")}i
+			@converter = Iconv.new( 'UTF-8', $1 )
+		end
+		@referred_uri = referred_uri
+		@excerpt_limit = excerpt_limit
+	end
+	
+	def convert( str )
+		@converter ? @converter.iconv( str ) : str
+	end
+	
+	def to_s
+		if @contents =~ %r{<body[^>]*>(.*)</body>}mi
+			to_detokenize = $1
+		else
+			@contents =~ %r{<body[^>]*>(.*)}mi
+			to_detokenize = $1
+		end
+		excerpt = Detokenizer.new( to_detokenize, @referred_uri ).execute
+		excerpt = Clipper.new( excerpt, @excerpt_limit ).execute
+		excerpt.gsub( /[\200-\377]/ ) { |c| "&#%04d;" % c[0] }
+		convert( excerpt )
+	end
+	
+
+	class ArrayOfWordsAndBrTags < Array
+		def initialize( text )
+			super()
+			text.split( %r{(<br />)} ).each { |br_or_between_br|
+				if br_or_between_br == '<br />'
+					self << br_or_between_br
+				else
+					self.concat( br_or_between_br.split )
+				end
+			}
+		end
+	end
+
+	class Clipper
+		def initialize( excerpt, excerpt_limit )
+			@excerpt = excerpt
+			@excerpt_limit = excerpt_limit
+		end
+		
+		def add_next_word
+			if @words_before.empty?
+				add_from_words_after
+			elsif @words_after.empty?
+				add_from_words_before
+			elsif @str_after.size >= @str_before.size
+				add_from_words_before
+			else
+				add_from_words_after
+			end
+		end
+		
+		def add_next_word?
+			( excerpt_parts_under_limit &&
+			  !( @words_before.empty? && @words_after.empty? ) )
+		end
+
+		def add_from_words_after
+			if @str_after == ''
+				@str_after = @words_after.shift
+			else
+				@str_after += ' ' + @words_after.shift
+			end
+		end
+
+		def add_from_words_before
+			if @str_before == ''
+				@str_before = @words_before.pop
+			else
+				@str_before = @words_before.pop + ' ' + @str_before
+			end
+		end
+		
+		def excerpt_parts_under_limit
+			( @str_before.size + @strong_text.size + @str_after.size + 2 ) <
+			  @excerpt_limit
+		end
+
+		def execute
+			@excerpt =~ %r{(.*)(<strong>.*?</strong>\S*)(.*)}m
+			@text_before = $1
+			@strong_text = $2
+			@text_after = $3
+			get_elts.join( ' ' )
+		end
+
+		def get_elts
+			@words_before = ArrayOfWordsAndBrTags.new( @text_before )
+			@words_after = ArrayOfWordsAndBrTags.new( @text_after )
+			@str_before = ''
+			@str_after = ''
+			add_next_word while add_next_word?
+			ellipses = '&#8230;'
+			elts = []
+			elts << ellipses unless @words_before.empty?
+			elts << @str_before
+			elts << @strong_text
+			elts << @str_after
+			elts << ellipses unless @words_after.empty?
+			elts
+		end
+	end
+
+	class Detokenizer
+		def initialize( contents, referred_uri )
+			@contents, @referred_uri = contents, referred_uri;
+		end
+
+		def compact_brs
+			result = 'something'
+			until result.nil?
+				result = @detokenized.gsub!( %r{<br />\s*<br />}, '<br /> ' )
+			end
+		end
+		
+		def execute
+			@detokenized = HtmlWithFixedAttributes.new( @contents )
+			@detokenized.gsub!( /<!--.*?-->/m, '' )
+			@detokenized.gsub!(
+				%r{</?(h\d|p|blockquote|table|tr|br|div|form|ul|li|center|ol|dl|dd|dt|fieldset|option|select|object|o:p).*?>}i,
+				' <br /> '
+			)
+			@detokenized.gsub!(
+				%r{</?(acronym|abbr|strong|td|tt|small|em|img|font|span|input|hr|noscript|legend|address).*?>}im, ''
+			)
+			@detokenized.gsub!( %r{</?(b|i)(\s+.*?)?>}i, '' )
+			substitute_links
+			compact_brs
+			@detokenized.gsub!( /\s+/, ' ' )
+			@detokenized
+		end
+		
+		def substitute_links
+			@detokenized.gsub!( %r{<a\s+(.*?)>(.*?)</(xhtml:)?a.*?>}mi ) { |s|
+				a_att = $1
+				a_body = $2
+				if ( a_att =~ /href=('|")?([^'" ]*)("|')?/i )
+					href = $2
+					href.gsub!( /("|')$/, '' )
+					matches_self = ( href == @referred_uri )
+				else
+					matches_self = false
+				end
+				matches_self ? "<strong>#{ a_body }</strong>" : a_body
+			}
+		end
+	end
+
+	class HtmlWithFixedAttributes < String
+		def initialize( contents )
+			super( '' )
+			script_stack = []
+			contents.split( %r{(</?script[^>]*>)}i ).each { |script_or_between|
+				if script_or_between =~ /^<script/i
+					script_stack.push( script_or_between )
+				elsif script_or_between =~ %r{</script}i
+					script_stack.pop
+				elsif script_stack.empty?
+					self << fix_attributes( script_or_between )
+				else
+					self << ' '
+				end
+			}
+		end
+
+		def fix_attributes( html_part )
+			fixed = ''
+			html_part.split( /(<.*?)(?=<)/ ).each { |part|
+				part = fix_bracket_plus( part ) if part != '' and part =~ /('|")/
+				fixed << part
+			}
+			fixed
+		end
+		
+		def fix_bracket_plus( part )
+			inside_attribute = false
+			between_tags = true
+			last_attribute_quote = nil
+			fixed = ''
+			part.scan( /([^'"<>]*)("|'|<|>)/ ) { |pre_delimiter, delimiter|
+				delimiter.gsub!( />/, '&gt;' ) if inside_attribute
+				if %w( ' " ).include? delimiter and !between_tags
+					if last_attribute_quote == delimiter or !inside_attribute
+						inside_attribute = !inside_attribute
+						last_attribute_quote = delimiter
+					end
+				end
+				between_tags = !between_tags if %w( < > ).include? delimiter
+				fixed << pre_delimiter
+				fixed << delimiter
+			}
+			post_scan = ( part =~  /.*('|"|<|>)(.*)/m ) ? $2 : part
+			fixed << post_scan
+			fixed
+		end
+	end
+end

metadata

@@ -0,0 +1,41 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.6
+specification_version: 1
+name: htmlclipping
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2005-05-15
+summary: HtmlClipping generates excerpts from an HTML page that has a link pointing to a particular URI.
+require_paths: 
+  - lib
+email: sera@fhwang.net
+homepage: http://htmlclipping.rubyforge.org/
+rubyforge_project: 
+description: "HtmlClipping generates excerpts from an HTML page that has a link pointing to a
+  particular URI. It removes most HTML markup, bolds the link text, and trims the
+  resulting text to a fixed number of characters. I developed it to help me track
+  referers to my website, though I suppose it might have other uses."
+autorequire: htmlclipping
+default_executable: 
+bindir: bin
+has_rdoc: false
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+  version: 
+platform: ruby
+authors: 
+  - Francis Hwang
+files: 
+  - lib/htmlclipping.rb
+  - lib/htmlclipping.rb~
+test_files: []
+rdoc_options: []
+extra_rdoc_files: []
+executables: []
+extensions: []
+requirements: []
+dependencies: []