GDNewsScraper 3.0.1 → 3.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06e468f8771feccf3609fa31369c22b865c5645b
-  data.tar.gz: 1936f2409c7f4c7bfb33cc58d3feaa5243e47203
+  metadata.gz: 692d674f129613f9b5fa1b379abf86c588d7f3f1
+  data.tar.gz: 3653b5992703ae9e7027e75b07ef0ed7818313a9
 SHA512:
-  metadata.gz: 5a90733cf5b4403b607154fcc354153ae26aa4ca9cd57334258e59a88e9ffdf7c913cc9dddb7e4e5cb643a6481cb24ad874bdb1aad91b583923a45507ffb1686
-  data.tar.gz: 310056309d60efe507b8e5ca92025f55945cc8dc4d17f934444f77e94281c20f129ab29242275ace4de86a0ddbd381b665ada074937c793b7dd0b0b592ff52db
+  metadata.gz: 580d4967034bed31b74e80b72993fda91614f6b8bb91c2c3f924590448e6f1c22c490ee6292722528434b619ed9f8a8c0625eb24c8dcaa767c4176adebebe184
+  data.tar.gz: fe10ea48908f0e012a14a78de2b340bb6ac21956475d7efdb4235a6d3bf0fc7e39e7a8a812c9cbb0e4844242d3b8ae63cff358b989e4a4c2fa6811e19e0105c6

data/lib/GDNewsScraper/scrapers/polygon_com/news.rb

@@ -1,109 +1,108 @@
-require 'pry'
 require 'base64'
 require 'json'
 
 module GDNewsScraper::Scrapers
-  
-  HEADERS ||= {
-    "User-Agent" => "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
-  }
-
-  STREAM_URI ||= 'https://www.polygon.com'
-
-  WHITELIST ||= {
-    default: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'figure', 'blockquote', 'ul', 'ol'],
-    inner:   ['strong', 'em', 'li']
-  }
-
-  DOM = {
-    article: {
-      wrapper:               '.c-compact-river',
-      container:             '.c-compact-river__entry',
-      inner_container:       '.c-entry-box--compact',
-      inner_container_video: '.c-entry-box--compact--video',
-      title:                 '.c-entry-box--compact__title',
-      cover:                 '.c-entry-box--compact__image',
-      meta:                  '.c-byline'
-    },
-
-    pagination: {
-      previous: '.c-pagination__prev',
-      info:     '.c-pagination__text',
-      next:     '.c-pagination__next'
+  module PolygonCOM
+    HEADERS ||= {
+      "User-Agent" => "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
+    }
+
+    STREAM_URI ||= 'https://www.polygon.com'
+
+    WHITELIST ||= {
+      default: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'figure', 'blockquote', 'ul', 'ol'],
+      inner:   ['strong', 'em', 'li']
+    }
+
+    DOM ||= {
+      article: {
+        wrapper:               '.c-compact-river',
+        container:             '.c-compact-river__entry',
+        inner_container:       '.c-entry-box--compact',
+        inner_container_video: '.c-entry-box--compact--video',
+        title:                 '.c-entry-box--compact__title',
+        cover:                 '.c-entry-box--compact__image',
+        meta:                  '.c-byline'
+      },
+
+      pagination: {
+        previous: '.c-pagination__prev',
+        info:     '.c-pagination__text',
+        next:     '.c-pagination__next'
+      }
     }
-  }
 
-  module PolygonCOM
     class News
       attr_accessor :stream
 
       def initialize(offset = 0)
-        uri = "#{ STREAM_URI }/news/archives/#{ offset }"
+        unless offset.nil?
+          uri = "#{ GDNewsScraper::Scrapers::PolygonCOM::STREAM_URI }/news/archives/#{ offset }"
 
-        @page   ||= Nokogiri::HTML(open(uri, HEADERS))
-        @stream = Hash.new
+          @page   = Nokogiri::HTML(open(uri, GDNewsScraper::Scrapers::PolygonCOM::HEADERS))
+          @stream = Hash.new
 
-        stream[:stream] = Hash.new
-        stream[:stream][:size]  = @page.css(DOM[:pagination][:info]).text.split[0].gsub(/\D/, '').to_i
-        stream[:stream][:pages] = @page.css(DOM[:pagination][:info]).text.split[6].gsub(/\D/, '').to_i
-        stream[:stream][:prev]  = @page.css(DOM[:pagination][:previous])&.first&.attr('href')&.split('/')&.last.to_i
-        stream[:stream][:next]  = @page.css(DOM[:pagination][:next])&.first&.attr('href')&.split('/')&.last.to_i
+          stream[:stream] = Hash.new
+          stream[:stream][:size]  = @page.css(GDNewsScraper::Scrapers::PolygonCOM::DOM[:pagination][:info]).text.split[0].gsub(/\D/, '').to_i
+          stream[:stream][:pages] = @page.css(GDNewsScraper::Scrapers::PolygonCOM::DOM[:pagination][:info]).text.split[6].gsub(/\D/, '').to_i
+          stream[:stream][:prev]  = @page.css(GDNewsScraper::Scrapers::PolygonCOM::DOM[:pagination][:previous])&.first&.attr('href')&.split('/')&.last.to_i
+          stream[:stream][:next]  = @page.css(GDNewsScraper::Scrapers::PolygonCOM::DOM[:pagination][:next])&.first&.attr('href')&.split('/')&.last.to_i
 
-        stream[:feed] = Hash.new
-        stream[:feed][:url] = STREAM_URI
-        stream[:feed][:source] = 'polygon'
-        stream[:feed][:label] = 'Polygon'
+          stream[:feed] = Hash.new
+          stream[:feed][:url] = GDNewsScraper::Scrapers::PolygonCOM::STREAM_URI
+          stream[:feed][:source] = 'polygon'
+          stream[:feed][:label] = 'Polygon'
 
-        stream[:articles] = Array.new
+          stream[:articles] = Array.new
 
-        perform
-      rescue
-        return 'There was a problem initializing the PolygonCOM::News Service'
+          perform
+        end
       end
 
       def perform
-        @page.css(DOM[:article][:container]).each do |article|
+        @page.css(GDNewsScraper::Scrapers::PolygonCOM::DOM[:article][:container]).each do |article|
           stream[:articles] << parse(article)
         end
-      rescue
-        return 'There was a problem performing the initial task in the PolygonCOM::News Service'
+      end
+
+      def refresh(article_url)
+        parse_article_body(article_url)
       end
 
       def parse(article)
         pulse = Hash.new
 
-        is_a_video = !article.at(DOM[:article][:inner_container_video]).nil?
+        is_a_video = !article.at(GDNewsScraper::Scrapers::PolygonCOM::DOM[:article][:inner_container_video]).nil?
 
-        key   = article.css(DOM[:article][:inner_container]).first.attr('data-chorus-optimize-id').to_i
-        url   = article.css(DOM[:article][:title]).children.first.attr('href')
-        title = article.css(DOM[:article][:title]).children.first.text
+        key   = article.css(GDNewsScraper::Scrapers::PolygonCOM::DOM[:article][:inner_container]).first.attr('data-chorus-optimize-id').to_i
+        url   = article.css(GDNewsScraper::Scrapers::PolygonCOM::DOM[:article][:title]).children.first.attr('href')
+        title = article.css(GDNewsScraper::Scrapers::PolygonCOM::DOM[:article][:title]).children.first.text
 
         pulse[:id]   = key
         pulse[:hash] = ::Base64.encode64("#{ title } - #{ key }")
 
         begin
-          pulse[:cover] = article.children.css(DOM[:article][:cover]).children.children.first.attr('src')
+          pulse[:cover] = article.children.css(GDNewsScraper::Scrapers::PolygonCOM::DOM[:article][:cover]).children.children.first.attr('src')
         rescue
           pulse[:cover] = nil
         end
 
         pulse[:url]     = url
         pulse[:title]   = title
-        pulse[:author]  = article.css(DOM[:article][:meta]).first.children[1].children[1].text
-        pulse[:date]    = JSON.parse(article.css(DOM[:article][:meta]).first.attr('data-cdata'))['timestamp'].to_i
+        pulse[:author]  = article.css(GDNewsScraper::Scrapers::PolygonCOM::DOM[:article][:meta]).first.children[1].children[1]&.text
+        pulse[:date]    = JSON.parse(article.css(GDNewsScraper::Scrapers::PolygonCOM::DOM[:article][:meta]).first.attr('data-cdata'))['timestamp'].to_i
         pulse[:content] = parse_article_body(url, is_a_video)
         pulse[:tags]    = title.downcase.split
 
-
         return pulse
-      rescue
-        return 'There was a problem creating the article in the PolygonCOM::News Service'
+      rescue => e
+        "There was a problem while parsing Article for '#{ title }' => #{ e }"
       end
 
     private
 
       def parse_article_body(article_url, is_a_video = false)
-        article_page = Nokogiri::HTML(open(article_url, HEADERS))
+        article_page = Nokogiri::HTML(open(article_url, GDNewsScraper::Scrapers::PolygonCOM::HEADERS))
         article_container = article_page.css('.c-entry-content')
 
         article_body = {
@@ -169,6 +168,12 @@ module GDNewsScraper::Scrapers
             if twitdget
               article_body[:body] << twitdget.to_html
             end
+
+            redditget = node.at('.reddit-card')
+
+            if redditget
+              article_body[:body] << redditget.to_html
+            end
           end
 
           # First ensure the node is an actual element. This removes random HTML elements
@@ -181,7 +186,7 @@ module GDNewsScraper::Scrapers
           # 
           # => WHITELIST[:default].include?(node.name)
           # 
-          if node.element? && WHITELIST[:default].include?(node.name)
+          if node.element? && GDNewsScraper::Scrapers::PolygonCOM::WHITELIST[:default].include?(node.name)
             case node.name
             when 'figure'
 
@@ -253,8 +258,8 @@ module GDNewsScraper::Scrapers
         end
 
         return article_body
-      rescue
-        return 'There was a problem parsing the article body in the PolygonCOM::News Service'
+      rescue => e
+        "There was a problem while parsing the Article body for '#{ title }' => #{ e }"
       end
 
       def attr(attribute)

data/lib/GDNewsScraper/version.rb

@@ -1,5 +1,5 @@
 module GDNewsScraper
-  VERSION ||= '3.0.1'
+  VERSION ||= '3.0.4'
 
   # => major: A new Source has been added or removed
   # => minor: A Source code has changed drastically to a point where it's not
@@ -27,4 +27,9 @@ module GDNewsScraper
   #          widget
   # v3.0.1 - Change the placeholder from [] to {{:}} which makes it a lot easier
   #          to scan and replace with Regex using scan(/\{{(.*?)\}}/)
+  # v3.0.2 - Parse Reddit inline widgets in the same way as Twitter widget
+  # v3.0.3 - Added a new method which will refresh the content of an Article
+  # v3.0.4 - Fixed an issue caused by Featured Articles which have a different
+  #          DOM structure
+  #          
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: GDNewsScraper
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.0.4
 platform: ruby
 authors:
 - Vlad Radulescu
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-27 00:00:00.000000000 Z
+date: 2017-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri