forum_post 0.1

data/COPYING

@@ -0,0 +1 @@
+No copying restrictions/license given.

data/LICENSE

@@ -0,0 +1 @@
+No license given.

data/README

@@ -0,0 +1,5 @@
+== Forum_post GemPlugin
+
+You should document your project here.
+
+

data/Rakefile

@@ -0,0 +1,62 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/clean'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'fileutils'
+include FileUtils
+
+version="0.1"
+name="forum_post"
+
+spec = Gem::Specification.new do |s|
+  s.name = name
+  s.version = version
+  s.description = s.summary = "Get the forum post"
+  s.author = "ltl3884"
+	s.email = "ltl3884@gmail.com"
+  s.add_dependency('nokogiri', '>= 1.4')
+  
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README"]
+  
+  s.files = %w(COPYING LICENSE README Rakefile) +
+    Dir.glob("{bin,doc/rdoc,test,lib}/**/*") + 
+    Dir.glob("ext/**/*.{h,c,rb}") +
+    Dir.glob("examples/**/*.rb") +
+    Dir.glob("tools/*.rb") +
+    Dir.glob("resources/**/*")
+  
+  s.require_path = "lib"
+  s.bindir = "bin"
+end
+
+Rake::GemPackageTask.new(spec) do |p|
+  p.need_tar = true if RUBY_PLATFORM !~ /mswin/
+end
+
+task :install => [:test, :package] do
+  sh %{sudo gem install pkg/#{name}-#{version}.gem}
+end
+
+task :uninstall => [:clean] do
+  sh %{sudo gem uninstall #{name}}
+end
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/test*.rb']
+  t.verbose = true
+end
+
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'doc/rdoc'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.add ['README', 'LICENSE', 'COPYING', 'lib/**/*.rb', 'doc/**/*.rdoc']
+end
+
+task :default => [:test, :package]
+
+CLEAN.include ['build/*', '**/*.o', '**/*.so', '**/*.a', 'lib/*-*', '**/*.log', 'pkg', 'lib/*.bundle', '*.gem', '.config']
+

data/lib/forum_post/init.rb

@@ -0,0 +1,6 @@
+require 'gem_plugin'
+
+# give this class the name you want for your command forum_post
+class ChangeME < GemPlugin::Plugin "/somecategory"
+end
+

data/lib/forum_post.rb

@@ -0,0 +1,174 @@
+require 'rubygems'
+require 'nokogiri'
+module ForumPost
+  class Document
+    DEFAULT_OPTIONS = {
+      :min_length => 5,
+      :min_text_length => 15
+    }.freeze
+
+    REGEXES = {
+        :unlikelyCandidatesRe => /combx|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i,
+        :maybe_post => /article|body|column|main|content|post|topic|text|info|message/i,
+        :not_post => /author|head|avatar|profile|rank|user|uid/i,
+        :divToPElementsRe => /<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i,
+        :replaceBrsRe => /(<br[^>]*>[ \n\r\t]*){2,}/i,
+        :replaceFontsRe => /<(\/?)font[^>]*>/i
+    }
+
+    attr_accessor :options, :html
+
+    def initialize(input, options = {})
+      @input = input.gsub(REGEXES[:replaceBrsRe], '</p><p>').gsub(REGEXES[:replaceFontsRe], '<\1span>')
+      @options = DEFAULT_OPTIONS.merge(options)
+      make_html
+    end
+
+    def make_html
+      @html = Nokogiri::HTML(@input, nil, 'UTF-8')
+    end
+
+    def content
+      remove_script_and_style!#去除scrpt和style
+      remove_unlikely_candidates!#去除不可能是发帖内容的标签
+      transform_misused_divs_into_p!#把一些噪音的div转换成p标签
+      if likely_posts.size > 0
+        #回帖数大于5
+        most_likely_post = most_likely_posts(likely_posts)
+        candidates=score_elem(most_likely_post)
+      else
+        #回帖数小于5
+        candidates=score_elem(@html)
+      end
+        best_elem=select_best(candidates)
+        best_elem.text
+    end
+
+    def score_elem(most_likely_post)
+      candidates=most_likely_post.css("div,tr,td") 
+      candidates.each do |elem|
+        base_score=100
+        str = "#{elem[:class]}#{elem[:id]}"
+        base_score+=10 if str =~ REGEXES[:maybe_post]
+        base_score-=20 if str =~ REGEXES[:not_post]
+        base_score-=8  if elem_size(elem)<DEFAULT_OPTIONS[:min_text_length]
+        elem["score"]=base_score.to_s
+      end
+      candidates
+    end
+
+    def select_best(candidates)
+      last_candidates=[]
+      candidates=candidates.sort{|a,b| b["score"].to_i<=>a["score"].to_i}
+      best_score=candidates.first["score"]
+      candidates.delete_if{|c| c["score"]!=best_score}
+      return candidates.first if candidates.size==1
+      candidates.each do |p|
+        flag=0
+        candidates.each do |q|
+          if is_contain(p,q) == true && p != q
+            flag+=1
+          end
+        end
+        last_candidates<<p if flag==0
+      end
+      return last_candidates.first if last_candidates.size==1
+      last_candidates.each do |lc|
+        lc["text_rate"] = (elem_size(lc)/elem_size(lc,'inner_html').to_f).to_s
+      end
+      last_candidates.sort{|a,b| b["text_rate"].to_f<=>a["text_rate"].to_f}.first
+    end
+
+    def most_likely_posts(likely_posts)
+      most_likely_posts=[]
+      likely_posts.each do |q|
+          flag=0
+          likely_posts.each do |p|
+               if is_contain(p,q) == true && p != q
+                  flag+=1
+               end
+          end
+          if flag == 0
+            most_likely_posts << q
+            debug("most_likelys:#{q.name}.#{q[:class]}.#{q[:id]}") 
+          end
+        end
+      most_likely_posts.sort{|m,n| elem_size(n)<=>elem_size(m)}.first
+    end
+    
+    def elem_size(elem,type='inner_text')
+       return elem.text.gsub(/\s(\s+)/,"").size  if type=='inner_text'
+       return elem.inner_html.gsub(/\s(\s+)/,"").size if type=='inner_html' 
+    end
+
+    def likely_posts
+      h={}
+      likely_posts=[]
+      @html.css("div,tr,td").each do |elem|
+          str = "#{elem[:class]}#{elem[:id]}"
+          if str =~ REGEXES[:maybe_post]
+            flag="#{elem.name},#{elem[:class]}"
+            if h.keys.include?(flag)
+               h[flag] << elem
+            else
+               h[flag] = ([] << elem)
+            end
+          end
+       end
+       h.delete_if{|k,v| v.size < DEFAULT_OPTIONS[:min_length]}
+       h.map{|k,v| likely_posts << v.first}
+       likely_posts
+    end
+
+    def remove_script_and_style!
+      @html.css("script, style").each { |i| i.remove }
+    end
+
+    def remove_unlikely_candidates!
+      @html.css("*").each do |elem|
+        str = "#{elem[:class]}#{elem[:id]}"
+        if str =~ REGEXES[:unlikelyCandidatesRe] && str !~ REGEXES[:maybe_post] && elem.name.downcase != 'body'
+          debug("Removing unlikely candidate - #{str}")
+          elem.remove
+        end
+      end
+    end
+    
+    def transform_misused_divs_into_p!
+      @html.css("*").each do |elem|
+        if elem.name.downcase == "div"
+          if elem.inner_html !~ REGEXES[:divToPElementsRe]
+            debug("Altering div(##{elem[:id]}.#{elem[:class]}) to p");
+            elem.name = "p"
+          end
+        end
+      end
+    end
+
+    def is_contain(p,q)
+      p.css("div,tr,td").each do |item|
+        return true if item.name == q.name && item[:class] == q[:class]
+      end
+      return false
+    end
+
+    def debug(str)
+      puts str if options[:debug]
+    end
+    
+   end
+end
+
+
+#require 'open-uri'
+#include ForumPost
+#uri = 'http://topic.csdn.net/u/20110718/17/f1a523fb-8c65-4510-a094-daf7bd2698cf.html?50656'
+#uri = 'http://bbs.lampchina.net/thread-29528-1-1.html'
+#uri = 'http://bbs.55bbs.com/thread-5662509-1-1.html'
+
+#uri='http://topic.csdn.net/u/20110803/11/7ef08712-129d-4971-88a4-4bd0b712c804.html?90401'
+#f = open(uri).read
+#charset = adjudge_html_charset(f)
+#source = convert_to_utf8(f,charset)
+#d = Document.new(source)
+#puts d.content

data/resources/defaults.yaml

@@ -0,0 +1,2 @@
+--- 
+:debug: false

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification 
+name: forum_post
+version: !ruby/object:Gem::Version 
+  hash: 9
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  version: "0.1"
+platform: ruby
+authors: 
+- ltl3884
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-04 00:00:00 +08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 1
+        - 4
+        version: "1.4"
+  type: :runtime
+  version_requirements: *id001
+description: Get the forum post
+email: ltl3884@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- COPYING
+- LICENSE
+- README
+- Rakefile
+- lib/forum_post.rb
+- lib/forum_post/init.rb
+- resources/defaults.yaml
+has_rdoc: true
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Get the forum post
+test_files: []
+