artext 0.0.2 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 379fd2eab84219f4a82393817354930f7e6cbacd
-  data.tar.gz: cee44b5160c701f51c1836d6cd0a27d655ecf4bb
+SHA256:
+  metadata.gz: 2b5fad951364d5b4d72143b5306bc59138b8ecbc120f34acd826aa7a64694e7c
+  data.tar.gz: 71c65f3cb3d0a7451dab3703050cd788454bdfba65981a8e633dc5f62808b059
 SHA512:
-  metadata.gz: e505ce5c26c90ecd95ffee18656626954276fc5333e607c158b650f2ded704a7b69a2181904555ae106f25e252488ca878b3aa58bccbd7fda21145c013290ac2
-  data.tar.gz: a6ffbd0ebe58681e9ecb9e0554e3aa0fc9ff446fe082ce8ab463362e16a83b1d91ddfdc30e6ecb08adf1a1b1e679ee363d2fcebf443b1a27b58fc260aa09d02b
+  metadata.gz: ef346ac934aeeb5ad7960b343ab7fe07206ba97da57dd16c02f686028c3ff4678b21be3852dcde619188bbaf5881ef63d89540f7686e924c3bc9267e5b9ca96b
+  data.tar.gz: 99b81500ee6ba0aac5a43cf54a3991f07e5856232a6cac49567c242b8195eaae72c49d9d6a4e8f390ae3836f5667bb977adfa944b11d01f1af6be88f57ad936b

data/.gitignore

@@ -0,0 +1 @@
+*.DS_STORE

data/Gemfile.lock

@@ -1,45 +1,42 @@
 PATH
   remote: .
   specs:
-    artext (0.0.1)
-      addressable (~> 2.3)
-      fastimage (~> 1.6)
-      httparty (~> 0.13)
-      mini_magick (~> 3.7)
-      nokogiri (~> 1.6)
+    artext (0.0.4)
+      addressable (~> 2.5)
+      fastimage (~> 2.1)
+      httparty (~> 0.14)
+      mini_magick (~> 4.8)
+      nokogiri (~> 1.8)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.3.8)
-    diff-lcs (1.2.5)
-    fastimage (1.7.0)
-      addressable (~> 2.3, >= 2.3.5)
-    httparty (0.13.5)
-      json (~> 1.8)
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    diff-lcs (1.3)
+    fastimage (2.1.4)
+    httparty (0.16.2)
       multi_xml (>= 0.5.2)
-    json (1.8.3)
-    mini_magick (3.8.1)
-      subexec (~> 0.2.1)
-    mini_portile (0.6.2)
-    multi_xml (0.5.5)
-    nokogiri (1.6.6.2)
-      mini_portile (~> 0.6.0)
-    rake (10.4.2)
-    rspec (3.3.0)
-      rspec-core (~> 3.3.0)
-      rspec-expectations (~> 3.3.0)
-      rspec-mocks (~> 3.3.0)
-    rspec-core (3.3.2)
-      rspec-support (~> 3.3.0)
-    rspec-expectations (3.3.1)
+    mini_magick (4.9.2)
+    mini_portile2 (2.3.0)
+    multi_xml (0.6.0)
+    nokogiri (1.8.5)
+      mini_portile2 (~> 2.3.0)
+    public_suffix (3.0.3)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.3.0)
-    rspec-mocks (3.3.2)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.3.0)
-    rspec-support (3.3.0)
-    subexec (0.2.3)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
 
 PLATFORMS
   ruby
@@ -51,4 +48,4 @@ DEPENDENCIES
   rspec (~> 3.3)
 
 BUNDLED WITH
-   1.10.6
+   1.16.3

data/Rakefile

@@ -6,4 +6,6 @@ RSpec::Core::RakeTask.new
 task :default => :spec
 task :test => :spec
 
-
+task :console do
+  exec "irb -r mygem -I ./lib"
+end

data/artext.gemspec

@@ -23,9 +23,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec", "~> 3.3"
 
 
-  spec.add_dependency "addressable", "~> 2.3"
-  spec.add_dependency "httparty", "~> 0.13"
-  spec.add_dependency "fastimage", "~> 1.6"
-  spec.add_dependency "mini_magick", "~> 3.7"
-  spec.add_dependency "nokogiri", "~> 1.6"
+  spec.add_dependency "addressable", "~> 2.5"
+  spec.add_dependency "httparty", "~> 0.14"
+  spec.add_dependency "fastimage", "~> 2.1"
+  spec.add_dependency "mini_magick", "~> 4.8"
+  spec.add_dependency "nokogiri", "~> 1.8"
 end

data/lib/artext.rb

@@ -4,6 +4,7 @@ require "httparty"
 require "nokogiri"
 require "mini_magick"
 require "fastimage"
+require "addressable"
 
 module Artext
 
@@ -121,7 +122,7 @@ module Artext
     if (article.count > 1)
       article = get_correct_article(article)
       score = 0.8
-    end 
+    end
     if (is_blank?(article))
       article = find_article(doc)
       score = 0.6
@@ -173,7 +174,7 @@ module Artext
     end
     return rel, score
   end
-  
+
   def self.iteratively_clean(element, html, images, score)
     html = ""
     imgs = []
@@ -227,15 +228,17 @@ module Artext
     elsif (element.name == "p")
       p_elem, ti = extractp(element, score)
       tv = "<p>#{p_elem}</p>" if (!is_blank?(p_elem))
+      tv = tv.gsub("<p></p>", "")
       images = images + ti if (!is_blank?(ti))
     elsif (element.name == "figure")
-      cap = element.search("figcaption").text.split.join(" ")
+      cap = element.search("figcaption").inner_html
       cap = is_blank?(cap) ? "" : "<figcaption>#{cap}</figcaption>"
       tv, ti = figurehandle(element, "", [])
       tv = "<figure>#{tv}#{cap}</figure>" if (!is_blank?(tv))
       images = images + ti
     elsif (element.name == "text")
       tv = element.text.split.join(" ")
+      tv = nil if tv == "advertisement"
       tv = "<p class\"inline\">#{tv}</p>" if (!is_blank?(tv))
     elsif (element.name == "i")
       tv = element.text.split.join(" ")
@@ -292,7 +295,7 @@ module Artext
   end
 
   def self.phandle(element, html, images)
-    if (!is_blank?(element.children) && !(element.name == "a" && is_blank?(element.search("img"))))
+    if (!is_blank?(element.children) && !((element.name == "a" || element.name == "figure") && is_blank?(element.search("img"))))
       element.children.each do |elem|
         html, images = phandle(elem, html, images)
       end
@@ -303,6 +306,12 @@ module Artext
         html = "</p><figure><img src=\"#{img}\"></figure><p>"
         images << img
       end
+    elsif (element.name == "figure")
+      cap = element.search("figcaption").inner_html
+      cap = is_blank?(cap) ? "" : "<figcaption>#{cap}</figcaption>"
+      tv, ti = figurehandle(element, "", [])
+      html = "</p><figure>#{tv}#{cap}</figure><p>" if (!is_blank?(tv))
+      images << ti if (!is_blank?(ti))
     elsif (element.name == "a")
       html = html + " <a href=\"#{element.attribute("href").value if (!is_blank?(element.attribute("href")))}\">#{element.text.split.join(" ")}</a> "
     elsif (element.name == "text")

data/lib/artext/version.rb

@@ -1,3 +1,3 @@
 module Artext
-  VERSION = "0.0.2"
+  VERSION = "0.0.4"
 end

metadata

@@ -1,127 +1,127 @@
 --- !ruby/object:Gem::Specification
 name: artext
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.4
 platform: ruby
 authors:
 - Anindya Mondal
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-14 00:00:00.000000000 Z
+date: 2018-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: httparty
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: fastimage
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: mini_magick
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '4.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '4.8'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.8'
 description: Extract article and other metadata from websites.
 email:
 - anindyamondal@mazdigital.com
@@ -129,6 +129,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -147,17 +148,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Extract article from websites.