BoilerpipeArticle 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a2cbf7a8c4c979aecefeb76935ec831f8f7e010b
+  data.tar.gz: 427bb161caf9302c93739c52ffdda87795d08ab1
+SHA512:
+  metadata.gz: d7268bf78155ffba0641a79c2aa83b36953f965587fe404ea96b41fbf7b960b7c70189a22690eedd9993ef2cd0f04d12408718a84e2a3ef5db445636be7f6fe7
+  data.tar.gz: 8366fd8c1ad057b7bb3718109c6e9070e60dd68e06fc4372e9a16f1ed72c9dd7527039e46c841a4f6aafa86d8ae72829f8f1775c1d85d76f1ad9a8ddd40fd24a

data/lib/boilerpipe_article.rb

@@ -0,0 +1,80 @@
+require 'nokogiri'
+
+class BoilerpipeArticle
+  def initialize(html)
+    @html = html
+  end
+
+  def run
+    html =  Nokogiri::HTML.parse(@html).to_s
+    html.gsub!(/<!-[\s\S]*?->/, '')
+    html.gsub!(/\r?\n|\r/, '')
+
+    doc = Nokogiri::HTML(html)
+    badHtmlTags = ['li','ol','ul','head','script','style','a','img']
+    badHtmlTags.each do |tag|
+      doc.search(tag).each do |src|
+        src.remove
+      end
+    end
+
+    html = doc.to_html.to_s
+
+    selfClosingTags = ['<area','<base','<br','<col','<command','<embed','<hr','<img','<input','<keygen','<link','<meta','<param','<source','<track','<wbr']
+
+
+    time = Time.now.to_f
+    depth = 1
+    i = 0
+    start = 0
+    close = 0
+    articlesStats = Hash.new
+    inPtag = false
+    content = ''
+    html.length.times do
+      char = html[i]
+      if char.eql? '<'
+        start = i
+        ii = start
+        html.length.times do
+          char2 = html[ii]
+          if char2.eql? '>'
+            tag = html[start..ii]
+            tagname = "#{tag}"
+            inPtag = true if tagname.eql?('<p>') || tagname.split(' ')[0].eql?('<p')
+            content = html[close..start].gsub(/[<>]/,'')
+            tagname = "#{tag}"
+            text = ''
+            text = content if inPtag
+            articlesStats.store(i,[text,depth,tagname]) if content.gsub(/[^a-zA-Z]+/,'').length > 1
+            close = ii
+            inPtag = false if tagname.eql? '</p>'
+            if !selfClosingTags.include?(tag.split(" ")[0]) && !tag.include?('<br')
+              tag.gsub!(/"[\s\S]*?"/,'')
+              tag.gsub!(/[^<>\/]+/,'')
+              if tag.eql? '<>'
+                depth+=1
+              else
+                depth-=1
+              end
+            end
+            break
+          end
+          ii+=1
+        end
+      end
+      i+=1
+    end
+    bestDepth = Hash.new(0)
+    articlesStats.each do |line,stats|
+      bestDepth[stats[1]]+=stats[0].gsub(/[^a-zA-Z]+/,'').length
+    end
+    best = bestDepth.sort_by {|key,value|value}.reverse.to_h.keys[0]
+    text = ''
+    articlesStats.each do |line,stats|
+      text = "#{text} #{stats[0]}" if stats[1] == best
+    end
+
+    return Nokogiri::HTML.parse(text).text
+  end
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: BoilerpipeArticle
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- David Layer-Reiss
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-07-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+description: This gem removes the surplus “clutter” (boilerplate, templates) around
+  the main textual content of a web page (pure Ruby implementation). Check GitHub
+  for usage examples.
+email: layerreiss@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/boilerpipe_article.rb
+homepage: https://github.com/davidlr99/BoilerpipeArticle
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '1.9'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: This gem extract the main textual content of a HTML page (e.g. news articles).
+test_files: []