xapian-indexer 1.2.3.2 → 1.2.3.4

data/lib/xapian/indexer/extractors/html.rb

@@ -20,6 +20,9 @@ module Xapian
 		module Extractors
 			# Represents a resource that will be indexed
 			class HTML
+				NBSP = Nokogiri::HTML(" ").text
+				WHITESPACE = /(\s|#{NBSP})+/
+				
 				def initialize(options = {})
 					@options = options
 					
@@ -40,7 +43,7 @@ module Xapian
 						first_paragraph = html.search("p").first
 					
 						if first_paragraph
-							result[:description] = first_paragraph.inner_text
+							result[:description] = first_paragraph.inner_text.gsub(WHITESPACE, " ")
 						end
 					end
 				
@@ -76,15 +79,15 @@ module Xapian
 					title_tag = html.at('html/head/title')
 					h1_tag = html.search('h1').first
 					if title_tag
-						result[:title] = title_tag.inner_text
+						result[:title] = title_tag.inner_text.gsub(WHITESPACE, " ")
 					elsif h1_tag
-						result[:title] = h1_tag.inner_text
+						result[:title] = h1_tag.inner_text.gsub(WHITESPACE, " ")
 					end
 				
 					# Extract keywords
 					meta_keywords = html.css("meta[name='keyword']").first
 					if meta_keywords
-						result[:keywords] = meta_keywords['content']
+						result[:keywords] = meta_keywords['content'].gsub(WHITESPACE, " ")
 					end
 					
 					# Remove junk elements from the html
@@ -99,7 +102,8 @@ module Xapian
 					body = html.at('html/body')
 					
 					if body
-						result[:content] = body.inner_text.gsub(/\s+/, " ")
+						# We also convert NBSP characters to inner space.
+						result[:content] = body.inner_text.gsub(WHITESPACE, " ")
 					end
 				
 					return result

data/lib/xapian/indexer/version.rb

@@ -19,7 +19,7 @@ module Xapian
 			MAJOR = 1
 			MINOR = 2
 			TINY  = 3
-			REV   = 2
+			REV   = 4
 			
 			STRING = [MAJOR, MINOR, TINY, REV].join('.')
 		end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification 
 name: xapian-indexer
 version: !ruby/object:Gem::Version 
-  hash: 71
+  hash: 75
   prerelease: false
   segments: 
   - 1
   - 2
   - 3
-  - 2
-  version: 1.2.3.2
+  - 4
+  version: 1.2.3.4
 platform: ruby
 authors: 
 - Samuel Williams
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-22 00:00:00 +13:00
+date: 2011-01-24 00:00:00 +13:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -33,6 +33,20 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
 description: 
 email: samuel.williams@oriontransfer.co.nz
 executables: []