hongkong-news-scrapers 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c55a3d970d1d5352121e572030ebca5de8ba0700
-  data.tar.gz: fb6f758b612e98e23de79f03285dd55dbc760fd7
+  metadata.gz: 325eddb50876330eefa2711493ca8070705319ce
+  data.tar.gz: ef032d1cda4305049bdfacbe2b02db85bc1127c8
 SHA512:
-  metadata.gz: 609c19b1a287c3f339d1902692a1f883862ddda4474d3c1b49d933a6e82da298998291032a72c6700bddfe73492b189b1c29edd6dc8426f442bb1437680893f3
-  data.tar.gz: 02e67ae4872db3dd33bce4d5f09dbcf0a34dd3ea5e765f2a9343d4615bb22118a84602000c2680f34996ae48a58fa1b763220547d7ef4c7b0024c9294eb61946
+  metadata.gz: 36f874ccb99d3ef14c0d2cf2db420369c16117c43f3d4753e0af4d0082a948dbbceec7776e5f8523ac7598e701fa54d2334a606ae506a73fadecbfa5359a512c
+  data.tar.gz: 32e782416317efbbb91e7947957d3d3683cec51d23db94f387fea7554765c76fb5cf708823b58320e3ec19ed607b1efb529ef8810d2669c710e191b797e45d1c

data/hongkong-news-scrapers.gemspec

@@ -26,4 +26,5 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'capybara'
   spec.add_dependency 'poltergeist'
+  spec.add_dependency 'nokogiri'
 end

data/lib/hongkong/news/models/document.rb

@@ -1,6 +1,6 @@
 module Hongkong
   module News
-    class Document < Struct.new(:id, :source, :title, :url, :html, :content, :screenshot_data)
+    class Document < Struct.new(:id, :source, :title, :url, :html, :content, :screenshot_data, :image_url)
     end
   end
 end

data/lib/hongkong/news/scrapers/apple_daily_scraper.rb

@@ -5,6 +5,10 @@ module Hongkong
     module Scrapers
       class AppleDailyScraper
         include PhantomScraper
+        
+        def name
+          "appledaily"
+        end
 
         # Extract all news links from Apple Daily
         def news_links
@@ -15,7 +19,7 @@ module Hongkong
             link.title = option.text
             link.url = option["value"]
             link
-          end
+          end.reject { |l| l.url.nil? }
         end
 
         # Extract article from page from Apple Daily
@@ -23,13 +27,13 @@ module Hongkong
           visit url
 
           document = Document.new
-          document.source = 'appledaily'
-          document.title = first("#articleContent h1").text.strip
+          document.source = name
+          document.title = doc.search("#articleContent h1").text.strip
           document.url = url
           document.html = html
           document.content = page.evaluate_script("HongKongNews.getInnerText('#masterContent')")
           document.screenshot_data = screenshot_data
-
+          document.image_url = doc.search("//meta[@property='og:image']/@content").first.text rescue nil
           document
         end
       end

data/lib/hongkong/news/scrapers/mingpao_scraper.rb

@@ -9,6 +9,10 @@ module Hongkong
 
         LIST_URL = "http://news.mingpao.com/pns/%E6%96%B0%E8%81%9E%E7%B8%BD%E8%A6%BD/web_tc/archive/latest"
 
+        def name
+          "mingpao"
+        end
+
         # Extract all news links from Mingpao
         def news_links
           visit LIST_URL
@@ -29,13 +33,13 @@ module Hongkong
           first("article p")
           
           document = Document.new
-          document.source = 'mingpao'
-          document.title = first("h1").text
+          document.source = name
+          document.title = doc.search("h1").text
           document.url = url
           document.html = html
           document.content = page.evaluate_script("HongKongNews.getInnerText('article')")
           document.screenshot_data = screenshot_data
-
+          document.image_url = doc.search("//meta[@property='og:image']/@content").first.text rescue nil
           document
         end
       end

data/lib/hongkong/news/scrapers/oriental_daily_scraper.rb

@@ -0,0 +1,51 @@
+require_relative './phantom_scraper'
+require 'uri'
+
+module Hongkong
+  module News
+    module Scrapers
+      class OrientalDailyScraper
+        include PhantomScraper
+
+        LIST_URL = "http://orientaldaily.on.cc/"
+
+        def name
+          "orientaldaily"
+        end
+
+        # Extract all news links
+        def news_links
+          visit LIST_URL
+
+          all("#articleListSELECT option").collect do |option|
+            link = Link.new
+            link.title = option.text
+            link.url = URI::join(LIST_URL, option["value"]).to_s
+            link
+          end.reject { |l| l.url.to_s.end_with?("#") }
+        end
+
+        # Extract article from page
+        def news(url)
+          visit url
+
+          # wait for content to be loaded
+          first("#contentCTN-right")
+          
+          document = Document.new
+          document.source = name
+          document.title = doc.search("h1").text
+          document.url = url
+          document.html = html
+          document.content = page.evaluate_script("HongKongNews.getInnerText('#contentCTN-top')") + "\n" + page.evaluate_script("HongKongNews.getInnerText('#contentCTN-right')")
+          document.screenshot_data = screenshot_data
+
+          image = doc.search("#contentCTN .photo img").first
+          document.image_url = URI::join(url, image["src"]).to_s if image
+
+          document
+        end
+      end
+    end
+  end
+end

data/lib/hongkong/news/scrapers/phantom_scraper.rb

@@ -1,4 +1,5 @@
 require 'capybara/poltergeist'
+require 'nokogiri'
 require 'tempfile'
 
 module Hongkong
@@ -45,6 +46,14 @@ module Hongkong
         def html
           page.html
         end
+
+        # Get a Nokogiri Document for current page
+        def doc
+          unless @doc
+            @doc = Nokogiri::HTML(html)
+          end
+          @doc
+        end
       end
     end
   end

data/lib/hongkong/news/scrapers/version.rb

@@ -1,7 +1,7 @@
 module Hongkong
   module News
     module Scrapers
-      VERSION = "0.3.1"
+      VERSION = "0.4.0"
     end
   end
 end

data/lib/hongkong/news/scrapers.rb

@@ -4,4 +4,5 @@ require "hongkong/news/models/document"
 require "hongkong/news/models/link"
 
 require "hongkong/news/scrapers/apple_daily_scraper"
-require "hongkong/news/scrapers/mingpao_scraper"
+require "hongkong/news/scrapers/mingpao_scraper"
+require "hongkong/news/scrapers/oriental_daily_scraper"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hongkong-news-scrapers
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Francis Chong
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Scrape Hong Kong news for good.
 email:
 - francis@ignition.hk
@@ -118,6 +132,7 @@ files:
 - lib/hongkong/news/scrapers.rb
 - lib/hongkong/news/scrapers/apple_daily_scraper.rb
 - lib/hongkong/news/scrapers/mingpao_scraper.rb
+- lib/hongkong/news/scrapers/oriental_daily_scraper.rb
 - lib/hongkong/news/scrapers/phantom_scraper.rb
 - lib/hongkong/news/scrapers/phantom_scraper_extension.js
 - lib/hongkong/news/scrapers/version.rb