GDNewsScraper 2.0.2 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 165229c71b29b8ea97c5a08c981421798d7a8d53
-  data.tar.gz: 86e96cc7035010913d220632bacbece653bb75df
+  metadata.gz: 300b4b0c79a91907c9031699c874cd2339dda2e7
+  data.tar.gz: d452da316c641f609d3cb83c5d88601d5abca271
 SHA512:
-  metadata.gz: 931a89d894e137d571e27162f8c2e3cab9358e8ca8ea0de46569a68c7ce39dcbb4ef70d4bcc7f1c7c17d8d3e7f473c95929218f216b5aef42024b319a5c24fd0
-  data.tar.gz: a26324d325f91bdeb60c57ae42b67a82ce4df74a86cc8ab30d5451219a3b0c50f000e7542af36b5d230220c7ebeb2b51d8e571a74e40a14cd34b057d8b64bdc6
+  metadata.gz: 7023abbec8ca015b9152737cd72385cf01aec37b2de5dc6e53b6d94ae5fd433e5c27940eaecbe37f1ceb4e6549312740a9d422fc6873144f8d877a2739d8469e
+  data.tar.gz: 72260107eab5febed2a934117869e815ff46cf721cc5bc7a830d4c78e754cb260aa0df173ac5349862bc44528e00c2d596b2a81ea13c734899863b32944991e9

data/GDNewsScraper.gemspec

@@ -4,27 +4,23 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'GDNewsScraper/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "GDNewsScraper"
+  spec.name          = 'GDNewsScraper'
   spec.version       = GDNewsScraper::VERSION
-  spec.authors       = ["Vlad Radulescu"]
-  spec.email         = ["pacMakaveli90@gmail.co.uk"]
+  spec.authors       = ['Vlad Radulescu']
+  spec.email         = ['pacMakaveli90@gmail.co.uk']
 
   spec.summary       = %q{A Ruby web scraper for gaming News and Reviews}
   spec.description   = %q{A Ruby Scraper created for games.directory to crawl the web for gaming News and Reviews.}
-  spec.homepage      = "https://github.com/games-directory/scraper"
-  spec.license       = "MIT"
+  spec.homepage      = 'https://github.com/games-directory/scraper'
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_dependency "nokogiri"
-  spec.add_dependency "httparty"
-  spec.add_dependency "activesupport"
+  spec.add_dependency 'nokogiri'
 
-  spec.add_development_dependency "bundler", "~> 1.12"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "pry"
+  spec.add_development_dependency 'bundler', '~> 1.12'
+  spec.add_development_dependency 'rake', '~> 10.0'
 end

data/Gemfile

@@ -1,10 +1,3 @@
 source 'https://rubygems.org'
 
-gem 'nokogiri'
-gem 'httparty'
-gem 'activesupport'
-gem 'sanitize'
-
-gem 'pry'
-
 gemspec

data/lib/GDNewsScraper/scrapers/polygon_com/news.rb

@@ -1,72 +1,269 @@
-require 'active_support/hash_with_indifferent_access'
+require 'pry'
+require 'base64'
+require 'json'
 
-module GDNewsScraper
-  module Scrapers
-    module PolygonCOM
+module GDNewsScraper::Scrapers
+  
+  HEADERS ||= {
+    "User-Agent" => "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
+  }
 
-      class News
-        attr_accessor :page, :articles
+  STREAM_URI ||= 'https://www.polygon.com'
 
-        def initialize(offset = 0)
-          headers  = { "User-Agent" => "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36" }
-          uri      = "http://www.polygon.com/news/#{offset}"
+  WHITELIST ||= {
+    default: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'figure', 'blockquote', 'ul', 'ol'],
+    inner:   ['strong', 'em', 'li']
+  }
 
-          @page   ||= Nokogiri::HTML(HTTParty.get(uri, headers: headers))
-          @articles = HashWithIndifferentAccess.new
+  DOM = {
+    article: {
+      wrapper:               '.c-compact-river',
+      container:             '.c-compact-river__entry',
+      inner_container:       '.c-entry-box--compact',
+      inner_container_video: '.c-entry-box--compact--video',
+      title:                 '.c-entry-box--compact__title',
+      cover:                 '.c-entry-box--compact__image',
+      meta:                  '.c-byline'
+    },
 
-          pagination = container.css('.pagination').children[3].children.text.split
+    pagination: {
+      previous: '.c-pagination__prev',
+      info:     '.c-pagination__text',
+      next:     '.c-pagination__next'
+    }
+  }
 
-          articles[:stream_size] = pagination.last.to_i
-          articles[:offset]      = pagination[3].to_i
+  module PolygonCOM
+    class News
+      attr_accessor :stream
 
-          articles[:feed] = HashWithIndifferentAccess.new
-          articles[:feed][:source] = 'polygon'
-          articles[:feed][:label]  = 'Polygon'
+      def initialize(offset = 0)
+        uri = "#{ STREAM_URI }/news/archives/#{ offset }"
 
-          articles[:stream] = get_all_news
+        @page   ||= Nokogiri::HTML(open(uri, HEADERS))
+        @stream = Hash.new
+
+        stream[:stream] = Hash.new
+        stream[:stream][:size]  = @page.css(DOM[:pagination][:info]).text.split[0].gsub(/\D/, '').to_i
+        stream[:stream][:pages] = @page.css(DOM[:pagination][:info]).text.split[6].gsub(/\D/, '').to_i
+        stream[:stream][:prev]  = @page.css(DOM[:pagination][:previous])&.first&.attr('href')&.split('/')&.last.to_i
+        stream[:stream][:next]  = @page.css(DOM[:pagination][:next])&.first&.attr('href')&.split('/')&.last.to_i
+
+        stream[:feed] = Hash.new
+        stream[:feed][:url] = STREAM_URI
+        stream[:feed][:source] = 'polygon'
+        stream[:feed][:label] = 'Polygon'
+
+        stream[:articles] = Array.new
+
+        perform
+      rescue
+        return 'There was a problem initializing the PolygonCOM::News Service'
+      end
+
+      def perform
+        @page.css(DOM[:article][:container]).each do |article|
+          stream[:articles] << parse(article)
+        end
+      rescue
+        return 'There was a problem performing the initial task in the PolygonCOM::News Service'
+      end
+
+      def parse(article)
+        pulse = Hash.new
+
+        is_a_video = !article.at(DOM[:article][:inner_container_video]).nil?
+
+        key   = article.css(DOM[:article][:inner_container]).first.attr('data-chorus-optimize-id').to_i
+        url   = article.css(DOM[:article][:title]).children.first.attr('href')
+        title = article.css(DOM[:article][:title]).children.first.text
+
+        pulse[:id]   = key
+        pulse[:hash] = ::Base64.encode64("#{ title } - #{ key }")
+
+        begin
+          pulse[:cover] = article.children.css(DOM[:article][:cover]).children.children.first.attr('src')
+        rescue
+          pulse[:cover] = nil
+        end
+
+        pulse[:url]     = url
+        pulse[:title]   = title
+        pulse[:author]  = article.css(DOM[:article][:meta]).first.children[1].children[1].text
+        pulse[:date]    = JSON.parse(article.css(DOM[:article][:meta]).first.attr('data-cdata'))['timestamp'].to_i
+        pulse[:content] = parse_article_body(url, is_a_video)
+        pulse[:tags]    = title.downcase.split
+
+
+        return pulse
+      rescue
+        return 'There was a problem creating the article in the PolygonCOM::News Service'
+      end
+
+    private
+
+      def parse_article_body(article_url, is_a_video = false)
+        article_page = Nokogiri::HTML(open(article_url, HEADERS))
+        article_container = article_page.css('.c-entry-content')
+
+        article_body = {
+          galleries: { },
+          videos: { },
+          images: { },
+
+          anchors: { },
+          figures: { },
+
+          body: [ ]
+        }
+
+        if is_a_video
+          iframe = article_page.at('.c-video-embed--media').at('iframe')
+          iframe_id = random_string
+
+          article_body[:videos][iframe_id] = {}
+          article_body[:videos][iframe_id][:url] = iframe.attr('src')
+
+          article_body[:body] << iframe.replace("[video]#{ iframe_id }").to_html
         end
 
-        def get_all_news
-          news = HashWithIndifferentAccess.new
-
-          container.css('.m-block').each do |article|
-            body       = article.css('.pinned_wrapper').css('.m-block__body')
-            article_id = article.attributes['data-entry-id'].value.to_i
-
-            news[article_id]           = HashWithIndifferentAccess.new
-            news[article_id][:id]      = article_id
-            news[article_id][:hash]    = Base64.encode64(body.children[3].children[1].children.children.text)
-            news[article_id][:url]     = body.children[1].attributes['href'].value
-            news[article_id][:title]   = body.children[3].children[1].children.children.text
-            news[article_id][:author]  = body.children[3].children[3].children[1].children.text
-            news[article_id][:content] = body.children[3].css('.copy').text
-            news[article_id][:tags]    = body.children[3].children[1].children.children.text.split('/').last.scan(/[[:alpha:]]{4,}/).uniq
-            news[article_id][:date]    = body.children[3].children[3].css('.long_date').children.text.strip
-
-            begin
-              cover = body.children[1].attributes['data-original']
-
-              if cover.nil?
-                news[article_id][:cover] = "https://cdn#{body.children[1].children[1].attributes['data-original'].value.split('/cdn').last}"
-              else
-                news[article_id][:cover] = "https://cdn#{body.children[1].attributes['data-original'].value.split('/cdn').last}"
+        article_container.children.each do |node|
+          if node.name == 'div'
+
+            # Check to see if the div contains a embeded video
+            # 
+            iframe = node.at('iframe')
+
+            if iframe # YouTube videos
+              iframe_id = random_string
+
+              article_body[:videos][iframe_id] = {}
+              article_body[:videos][iframe_id][:url] = iframe.attr('src')
+
+              article_body[:body] << iframe.replace("[video]#{ iframe_id }").to_html
+            end
+
+            # Check to see if the div contains a gallery
+            # 
+            gallery = node.at('.c-image-gallery')
+
+            if gallery
+              gallery_container = gallery.at('.c-image-gallery__thumbs-viewport')
+
+              gallery_id = random_string
+              article_body[:galleries][gallery_id] = []
+
+              gallery_container.children.children.each do |image_container|
+                image = image_container.at('a')
+
+                if image
+                  article_body[:galleries][gallery_id] << image.attr('href')
+                end
               end
-            rescue
-              news[article_id][:cover] = nil
+
+              article_body[:body] << gallery.replace("[gallery]#{ gallery_id }").to_html
+            end
+
+            twitdget = node.at('.twitter-tweet')
+
+            if twitdget
+              article_body[:body] << twitdget.to_html
             end
           end
 
-          return news
-        end
+          # First ensure the node is an actual element. This removes random HTML elements
+          # 
+          # => node.element?
+          #
+          # Secondly, ensure the node is what we actual want. We don't want <div>'s
+          # which are usualy used for placing inline advertisments or content specific
+          # only to that website
+          # 
+          # => WHITELIST[:default].include?(node.name)
+          # 
+          if node.element? && WHITELIST[:default].include?(node.name)
+            case node.name
+            when 'figure'
+
+              image = node.css('.e-image__image').first
+              image_url = image.attr('data-original')
+
+              begin
+                if image_url.split('.').last == 'gif'
+                  image_id = random_string
+
+                  article_body[:images][image_id] = {}
+                  article_body[:images][image_id][:url] = image_url
+
+                  article_body[:body] << node.replace("[image]#{ image_id }").to_html
+                else
+                  image_alt = image.children.at('img').attr('alt')
+                  image_title = image.children.at('img').attr('title')
 
-      private
+                  image_meta = node.css('.e-image__meta')
 
-        def container
-          page.css('.m-grouptown')
+                  figure_id = random_string
+
+                  article_body[:figures][figure_id] = {}
+
+                  article_body[:figures][figure_id][:image] = image_url
+                  article_body[:figures][figure_id][:title] = image_title
+                  article_body[:figures][figure_id][:alt]   = image_alt
+
+                  unless image_meta.empty?
+                    article_body[:figures][figure_id][:caption] = image_meta.first.at('figcaption')&.text
+                    article_body[:figures][figure_id][:cite]    = image_meta.first.at('cite')&.text
+                  end
+                  
+                  article_body[:body] << node.replace("[figure]#{ figure_id }").to_html
+                end
+              rescue
+                raise 'Unknown format, please review.'
+              end
+            else
+
+              node.children.each do |url|
+                begin
+                  if url.name == 'a'
+                    url_id = random_string
+                    
+                    article_body[:anchors][url_id.to_sym] = {
+                      text: url.children.text,
+                      url: url.attributes['href'].value
+                    }
+
+                    url.replace("[anchor]#{ url_id }")
+                  end
+                rescue
+                  raise 'Unknown format, please review.'
+                end
+              end
+
+              # Remove all attributes
+              # 
+              parsed_node = node.xpath('.//@*').remove
+
+              # Return clean HTML, including HTML elements and text
+              # 
+              parsed_node = node.to_html
+            end
+
+            article_body[:body] << parsed_node
+          end
         end
 
-      end # News
+        return article_body
+      rescue
+        return 'There was a problem parsing the article body in the PolygonCOM::News Service'
+      end
+
+      def attr(attribute)
+        attributes&.fetch(attribute, nil)&.value
+      end
 
-    end # PolygonCOM
-  end # Scrapers
-end # GDNewsScraper
+      def random_string
+        (0...50).map { (65 + rand(25)).chr }.join
+      end
+    end # News
+  end # PolygonCOM
+end # GDNewsScraper::Scrapers

data/lib/GDNewsScraper/scrapers/polygon_com/reviews.rb

@@ -1,5 +1,3 @@
-require 'active_support/hash_with_indifferent_access'
-
 module GDNewsScraper
   module Scrapers
     module PolygonCOM
@@ -12,7 +10,7 @@ module GDNewsScraper
           uri      = "https://www.polygon.com/games/reviewed/#{offset}"
 
           @page   ||= Nokogiri::HTML(HTTParty.get(uri, headers: headers))
-          @articles = HashWithIndifferentAccess.new
+          @articles = Hash.new
 
           pagination = container.css('.pagination').children[3].children.text.split
 
@@ -20,7 +18,7 @@ module GDNewsScraper
           articles[:offset]      = pagination[3].to_i
           articles[:total]       = get_all_reviews.size
 
-          articles[:feed] = HashWithIndifferentAccess.new
+          articles[:feed] = Hash.new
           articles[:feed][:source] = 'polygon'
           articles[:feed][:label]  = 'Polygon'
 
@@ -28,7 +26,7 @@ module GDNewsScraper
         end
 
         def get_all_reviews
-          reviews = HashWithIndifferentAccess.new
+          reviews = Hash.new
 
           page.css('.m-game--index__list').children.each do |review|
 

data/lib/GDNewsScraper/version.rb

@@ -1,6 +1,12 @@
 module GDNewsScraper
-  VERSION = "2.0.2"
+  VERSION ||= '3.0.0'
 
+  # => major: A new Source has been added or removed
+  # => minor: A Source code has changed drastically to a point where it's not
+  # backwards compatible anymore
+  # => patch: Small addition to an existing Source. The new code shouldn't break
+  # any existing implementation. If it does, it needs backwards compatibility
+  #
   # CHANGELOG
   #
   # v1.0.0 - Initial Gem Setup
@@ -13,5 +19,10 @@ module GDNewsScraper
   #          would cause the script to fail when requesting the photo for the
   #          Article
   # v2.0.2 - Fix minor bug in PolygonCOM News scraper
-
+  # v2.1.0 - Updated PolygonCOM to reflect their new UI changes
+  # v2.1.1 - Fixed PolygonCOM Scraper to include videos if there are any
+  # v3.0.0 - Major overhaul to how an Article is parsed and returned to the User
+  #          In a 'Wordpress' kind of style, various html elements are now
+  #          returned in a way that an app can recognize and translate it into a
+  #          widget
 end

data/lib/GDNewsScraper.rb

@@ -1,6 +1,6 @@
 require 'GDNewsScraper/version'
-require 'httparty'
 require 'nokogiri'
+require 'open-uri'
 
 module GDNewsScraper
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: GDNewsScraper
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 3.0.0
 platform: ruby
 authors:
 - Vlad Radulescu
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-06-22 00:00:00.000000000 Z
+date: 2017-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -24,34 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: httparty
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -80,34 +52,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: A Ruby Scraper created for games.directory to crawl the web for gaming
   News and Reviews.
 email:
@@ -151,8 +95,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: A Ruby web scraper for gaming News and Reviews
 test_files: []
+has_rdoc: