BoilerpipeArticle 0.0.4 → 0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 86e9801c9b5e6eacf758aa4bbff2ec298313e163
-  data.tar.gz: 58bdb8a6a7624e08f5f28f8bbbc094da2a3fc02a
+  metadata.gz: fccf00da5423dc69b01d5f7b43a0932574ac16e9
+  data.tar.gz: a9d9ae5187108e1d20d7d9a45926b811cde2f3cf
 SHA512:
-  metadata.gz: f8dcba151737a254fb7f0840d24d187a757575ed87c8b0817ffb24dd3d3e3b28bebacf7ac4ee980bf8bc085317c9f3ae94a651c7c5b77fee43511f1a615077ab
-  data.tar.gz: e3a3ed339a688ae3ed778cd4f4c2e2e426ea070b94e07dcc156d945731b38961540f7a46fa4d421dbdac21aa47022c70df6586f6458bf4f620b95e3628278cd5
+  metadata.gz: 3c85b972589fd947fe9e2dd7606935fcd292e874e7e19ba1936d128eae26e03e8dc0e8e35afd40b82dac598925ba2e4de3d72da4d43f6a13ed36982a27c7ff7c
+  data.tar.gz: 10642f8bbf5573f8d4608945201cca270022b5e000bba26998670a75d235e5c234d5fd0047039d2a2afff38b5a696d4c82c10c68d2b9148104ce4cd24f09dcc3

data/lib/boilerpipe_article.rb

@@ -1,90 +1,189 @@
+#Encoding: UTF-8
+
 require 'nokogiri'
+require 'mida'
 
 class BoilerpipeArticle
   def initialize(html)
-    @html = html
+    @html = html.gsub(/\s\s+/,' ')
+    @articlesStats = Hash.new
   end
-
-  def getText(html = @html)
+  def removeBadHtmlTags(html = @html)
     html =  Nokogiri::HTML.parse(html).to_s
     html.gsub!(/<!-[\s\S]*?->/, '')
     html.gsub!(/\r?\n|\r/, '')
 
+    unwantedTags = ['strong','bold','i']
+    unwantedTags.each do |tag|
+      html.gsub!("<#{tag}>",'')
+      html.gsub!("</#{tag}>",'')
+    end
+
+
     doc = Nokogiri::HTML(html)
-    badHtmlTags = ['nav','head','script','style','a','img']
+
+    badHtmlTags = ['script','style','head','nav','iframe','img','footer','ol','ul','li','a']
+    doc.css('*').each do |node|
+      node.remove if node.text.length < 3
+    end
     badHtmlTags.each do |tag|
       doc.search(tag).each do |src|
         src.remove
       end
     end
-
+    # doc.css('a').each do |atag|
+    #   atag = "#{atag.text}"
+    #   puts atag
+    # end
     html = doc.to_html.to_s
-    selfClosingTags = ['<area','<base','<br','<col','<command','<embed','<hr','<img','<input','<keygen','<link','<meta','<param','<source','<track','<wbr']
-    time = Time.now.to_f
-    depth = 1
-    i = 0
-    start = 0
-    close = 0
+
+    return html
+  end
+  def calculateDepth(html = @html)
     articlesStats = Hash.new
-    inPtag = false
-    content = ''
-    html.length.times do
-      char = html[i]
-      if char.eql? '<'
-        start = i
-        ii = start
-        html.length.times do
-          char2 = html[ii]
-          if char2.eql? '>'
-            tag = html[start..ii]
-            tagname = "#{tag}"
-            inPtag = true if tagname.eql?('<p>') || tagname.split(' ')[0].eql?('<p')
-            content = html[close..start].gsub(/[<>]/,'')
-            tagname = "#{tag}"
-            text = ''
-            text = content if inPtag
-            articlesStats.store(i,[text,depth,tagname]) if content.gsub(/[^a-zA-Z]+/,'').length > 1
-            close = ii
-            inPtag = false if tagname.eql? '</p>'
-            if !selfClosingTags.include?(tag.split(" ")[0]) && !tag.include?('<br')
-              tag.gsub!(/"[\s\S]*?"/,'')
-              tag.gsub!(/[^<>\/]+/,'')
-              if tag.eql? '<>'
-                depth+=1
-              else
-                depth-=1
-              end
+    doc = Nokogiri::HTML(html)
+    i = 0
+    doc.xpath('//text()').each do |node|
+      text = node.to_s
+      articlesStats.store(i,[node.text.to_s,node.ancestors.length.to_i,node.parent.name])
+      i+=1
+    end
+    return articlesStats
+  end
+  def removeSamePatterns(html)
+    doc = Nokogiri::HTML(html)
+    paths = Array.new
+    doc.css('*').each do |node|
+      s = node.path.gsub(/\[[\s\S]*?\]/, '')
+      paths.push(s)
+    end
+    final = []
+    (7..30).each do |i|
+      all = []
+      paths.each_with_index do |seq,a|
+        se = []
+        paths[a..-1].each_with_index do |s,ii|
+          se << s
+          break if ii == i-1
+        end
+        all << se
+      end
+      final << all
+    end
+    allDoubles = Hash.new
+    final.each_with_index do |seq,i|
+      counts = Hash.new(0)
+      seq.each do |name|
+        counts[name] += 1
+      end
+      counts = counts.sort_by{|k,v|v}.reverse.to_h
+      allDoubles.store(i,counts)
+    end
+    allDoubles.each do |i,doubles|
+      doubles.each do |path,count|
+        if count >= 7
+          doc.css('*').each do |node|
+            s = node.path.gsub(/\[[\s\S]*?\]/, '')
+            if path.include? s
+              node.remove
             end
-            break
           end
-          ii+=1
         end
       end
-      i+=1
     end
+    return doc.to_s
+  end
+  def calculateBestDepth(articlesStats)
     bestDepth = Hash.new(0)
     articlesStats.each do |line,stats|
-      bestDepth[stats[1]]+=stats[0].gsub(/[^a-zA-Z]+/,'').length
+      bestDepth[stats[1]]+=stats[0].length
     end
-    best = bestDepth.sort_by {|key,value|value}.reverse.to_h.keys[0]
+    bestvalues = bestDepth.sort_by {|key,value|value}.reverse.to_h
+    average = 0.0
+    bestDepth.each {|l,v|average+=v/bestDepth.keys.length.to_f}
+    texts = 0
+    bestDepth.each{|l,v|texts +=1 if v > average}
+
+    doubleTexts = false
+    doubleTexts = true if texts >= 2
+    best = bestvalues.keys[0]
+
+    return best,doubleTexts
+  end
+
+  def getTextOfBestDepth(articlesStats,best)
     text = ''
     articlesStats.each do |line,stats|
-      text = "#{text} #{stats[0]}" if stats[1] == best
+      if stats[1] == best && (stats[-1].eql?('h1') || stats[-1].eql?('h2') || stats[-1].eql?('p'))
+        text = "#{text} <#{stats[-1]}>#{stats[0]}</#{stats[-1]}>" if stats[0].strip.length > 2
+      end
     end
-    return Nokogiri::HTML.parse(text).text
+    return text
   end
 
-  def getOgMetas(html = @html)
+  def getMetas(html = @html)
     metas = Hash.new
     doc = Nokogiri.parse(html)
-    properties = ['title','type','url','description','image','type','updated_time','locale','url','site_name']
-    properties.each do |prop|
-      if doc.at("meta[property=\"og:#{prop}\"]") != nil
-        metas.store(prop,doc.at("meta[property=\"og:#{prop}\"]")['content'])
-      else
-        metas.store(prop,' ')
-      end
+    doc.xpath("//meta").each do |node|
+      name = node[node.attributes.keys[1]]
+      name = node[node.attributes.keys[0]] if node.attributes.keys[0] != 'content' &&  node.attributes.keys[0] != 'value'
+      content = node['content']
+      content = node['value'] if content == nil
+
+      metas.store(name,content)
     end
     return metas
   end
+  def getOtherHTMLDescriptions(html = @html)
+    doc = Nokogiri.parse(html)
+    images = Array.new
+    headlines = Hash.new
+    links = Hash.new
+    5.times do |i|
+      hs = doc.xpath("//h#{i+1}")
+      texts = []
+      hs.each {|node| texts.push(node.text.to_s)}
+      headlines.store("h#{i+1}",texts)
+    end
+
+    imgs = doc.xpath('//img/@src')
+    imgs.each do |source|
+      images.push(source.text) if source.text.include?('http')
+    end
+
+    plinks = doc.xpath('//a/@href')
+    plinks.each do |source|
+      links.store(source.text,1) if source.text.strip.length > 2
+    end
+
+    return {'headlines'=>headlines,'images'=>images, 'links' => links.keys}
+  end
+  def getMicroData(html = @html)
+    doc = Mida::Document.new(html, "")
+    topLevel = Array.new
+    doc.items.each do |item|
+      topLevel.push(item.to_h)
+    end
+    return topLevel
+  end
+  def getAllText(html = @html)
+    doc = Nokogiri.parse(html)
+    doc.search('script').remove
+    doc.search('style').remove
+    return doc.text.gsub(/\s\s+/,' ')
+  end
+  def getArticle(html = @html)
+    html = removeBadHtmlTags(html)
+    articlesStats = calculateDepth(html)
+    best,doubleTexts = calculateBestDepth(articlesStats)
+    if doubleTexts
+      html = removeSamePatterns(html)
+      articlesStats,d = calculateDepth(html)
+
+    end
+    bestDepth,doubles = calculateBestDepth(articlesStats)
+    plainText = getTextOfBestDepth(articlesStats,bestDepth)
+    return plainText
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: BoilerpipeArticle
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: '0.1'
 platform: ruby
 authors:
 - David Layer-Reiss
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-31 00:00:00.000000000 Z
+date: 2016-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -24,9 +24,24 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.6.8
+- !ruby/object:Gem::Dependency
+  name: mida
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.9
 description: This gem removes the surplus “clutter” (boilerplate, templates) around
   the main textual content of a web page (pure Ruby implementation). BoilerpipeArticle
-  can be also used to parse open graph meta data. Check GitHub for usage examples.
+  can be also used to parse (open graph) meta data and microdata. Check GitHub for
+  usage examples.
 email: layerreiss@gmail.com
 executables: []
 extensions: []