forum_post 0.2 → 0.3

data/Rakefile

@@ -6,7 +6,7 @@ require 'rake/rdoctask'
 require 'fileutils'
 include FileUtils
 
-version="0.2"
+version="0.3"
 name="forum_post"
 
 spec = Gem::Specification.new do |s|

data/lib/forum_post.rb

@@ -9,7 +9,7 @@ module ForumPost
 
     REGEXES = {
         :unlikelyCandidatesRe => /combx|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i,
-        :maybe_post => /article|body|column|main|content|post|topic|text|info|message|item/i,
+        :maybe_post => /article|body|column|main|content|post|topic|text|info|message|item|bord|forum/i,
         :not_post => /author|head|avatar|profile|rank|user|uid/i,
         :divToPElementsRe => /<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i,
         :replaceBrsRe => /(<br[^>]*>[ \n\r\t]*){2,}/i,
@@ -32,19 +32,24 @@ module ForumPost
       remove_script_and_style!#去除scrpt和style
       remove_unlikely_candidates!#去除不可能是发帖内容的标签
       transform_misused_divs_into_p!#把一些噪音的div转换成p标签
-			bests=[]
 			better_post = likely_posts
 			if better_post.size > 0
 			  debug("post>4")
-     		@html =  most_likely_posts(better_post)
+     		handle_html=most_likely_posts(better_post)
 			else
+				handle_html=@html
 			  debug("post<4")
 		  end	
-	    bests=@html.css("div,tr,td") 
-			return @html.text.gsub(/\s(\s+)/,"")	if bests.size==0  
+	    bests=handle_html.css("div,tr,td") 
+			if bests.size==0 
+				debug("best_one:#{bests.name}.#{bests[:class]} #{bests[:id]}")
+				return handle_html.text.gsub(/\s(\s+)/,"")	 
+			else
+				bests.map{|best| debug("bests:#{best.name}.#{best[:class]} #{best[:id]}")}
+			end
      	candidates=score_elem(bests)
       best_elem=select_best(candidates)
-      best_elem.text
+      best_elem.text.gsub(/\s(\s+)/,"") 
     end
 
     def score_elem(bests)
@@ -56,6 +61,7 @@ module ForumPost
         base_score-=8  if elem_size(elem)<DEFAULT_OPTIONS[:min_text_length]
         elem["score"]=base_score.to_s
       end
+			bests.map{|best| debug("#{best.name}.#{best[:class]} #{best[:id]}---score:#{best['score']}")}
       bests
     end
 
@@ -74,10 +80,14 @@ module ForumPost
         end
         last_candidates<<p if flag==0
       end
-      return last_candidates.first if last_candidates.size==1
+			if last_candidates.size==1
+				debug("best_one:#{last_candidates.first.name}.#{last_candidates.first[:class]} #{last_candidates.first[:id]}")
+      	return last_candidates.first 
+			end
       last_candidates.each do |lc|
         lc["text_rate"] = (elem_size(lc)/elem_size(lc,'inner_html').to_f).to_s
       end
+			last_candidates.map{|lc| debug("best_one:#{lc.name}.#{lc[:class]} #{lc[:id]}---text_rate:#{lc['text_rate']}")}
       last_candidates.sort{|a,b| b["text_rate"].to_f<=>a["text_rate"].to_f}.first
     end
 
@@ -95,7 +105,6 @@ module ForumPost
             debug("most_likelys:#{q.name}.#{q[:class]}.#{q[:id]}") 
           end
         end
-			debug("most_likely_posts:#{most_likely_posts.size}")	
       most_likely_posts.sort{|m,n| elem_size(n)<=>elem_size(m)}.first
     end
     
@@ -111,18 +120,24 @@ module ForumPost
           str = "#{elem[:class]}#{elem[:id]}"
           if str =~ REGEXES[:maybe_post]
             flag="#{elem.name},#{elem[:class]}"
-            if h.keys.include?(flag)
-               h[flag] << elem
-            else
-               h[flag] = ([] << elem)
-            end
+            collect_likely_elem(h,flag,elem)
           end
        end
        h.delete_if{|k,v| v.size < DEFAULT_OPTIONS[:min_length]}
        h.map{|k,v| likely_posts << v.first}
+			 likely_posts.map{|lp| debug("likely_posts:#{lp.name}.#{lp[:class]} #{lp[:id]}")}
        likely_posts
     end
 
+		def collect_likely_elem(h,flag,elem)
+				flag.split(/ /).each do |item|
+					h.map do |k,v|
+						h[k] << elem if k =~ Regexp.new(item)
+					end
+				end
+				h[flag] = ([] << elem)
+		end
+
     def remove_script_and_style!
       @html.css("script, style").each { |i| i.remove }
     end

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: forum_post
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 13
   prerelease: false
   segments: 
   - 0
-  - 2
-  version: "0.2"
+  - 3
+  version: "0.3"
 platform: ruby
 authors: 
 - ltl3884
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-04 00:00:00 +08:00
+date: 2011-08-09 00:00:00 +08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency