hongkong-news-scrapers 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 95d75b7d38e8eb25ea08a743393301fb76f59179
-  data.tar.gz: 18944ea56db252f3f3a21f475cf03ec43db4eca5
+  metadata.gz: bd2a3a978a37ffcf1f0edafaca7b69d379f81983
+  data.tar.gz: 69974d9afb715fda8b72b6a8901e274fbd081dba
 SHA512:
-  metadata.gz: 92e25a760477e6b7d3f74cc33f327b8eb22212befaaa22a74bd8d5b2b3f614c8430ebf517e9bd47827eb3f40a944b9ff81ee21c58f7b1cf3834af2f89474ff37
-  data.tar.gz: 96683750052447cd9f724c2fa10a2a5842aad7365ba70e5302c992d6dc705168d16f7cf3f1c5fa28701dee18a910417750dd0ede2b234e57dc211fa97cc0dda1
+  metadata.gz: 71e08113fbe2282382b4c6260d4b3dcf7c0ec89797f98bfd886d409a5840d516e0cc5660ffdf80e15e2bab05764598800882ff68b5c7c48d52771b9bd8abf2d6
+  data.tar.gz: 46e8d11e0aaa8c875a18a7f723ac07e90b634734d51ca2e096ef61cf6c663e3da65aca36df4d42f606ad11dab2244adcd1b7cc76787448d75b831d4db98507b7

data/lib/hongkong/news/scrapers/apple_daily_scraper.rb

@@ -14,12 +14,15 @@ module Hongkong
         def news_links
           visit "http://hk.apple.nextmedia.com/"
 
-          all("#article_ddl option").collect do |option|
+          links = all("#article_ddl option").collect do |option|
             link = Link.new
             link.title = option.text
             link.url = option["value"]
             link
           end.reject { |l| l.url.nil? }
+
+          cleanup
+          links
         end
 
         # Extract article from page from Apple Daily
@@ -34,6 +37,8 @@ module Hongkong
           document.content = page.evaluate_script("HongKongNews.getInnerText('#masterContent')")
           document.screenshot_data = screenshot_data
           document.image_url = doc.search("//meta[@property='og:image']/@content").first.text rescue nil
+
+          cleanup
           document
         end
       end

data/lib/hongkong/news/scrapers/mingpao_scraper.rb

@@ -17,12 +17,15 @@ module Hongkong
         def news_links
           visit LIST_URL
 
-          all(".listing ul li a").collect do |anchor|
+          links = all(".listing ul li a").collect do |anchor|
             link = Link.new
             link.title = anchor.text
             link.url = URI::join(LIST_URL, anchor["href"]).to_s
             link
           end
+
+          cleanup
+          links
         end
 
         # Extract article from page from Mingpao
@@ -40,6 +43,8 @@ module Hongkong
           document.content = page.evaluate_script("HongKongNews.getInnerText('article')")
           document.screenshot_data = screenshot_data
           document.image_url = doc.search("//meta[@property='og:image']/@content").first.text rescue nil
+
+          cleanup
           document
         end
       end

data/lib/hongkong/news/scrapers/oriental_daily_scraper.rb

@@ -17,12 +17,15 @@ module Hongkong
         def news_links
           visit LIST_URL
 
-          all("#articleListSELECT option").collect do |option|
+          links = all("#articleListSELECT option").collect do |option|
             link = Link.new
             link.title = option.text
             link.url = URI::join(LIST_URL, option["value"]).to_s
             link
           end.reject { |l| l.url.to_s.end_with?("#") }
+
+          cleanup
+          links
         end
 
         # Extract article from page
@@ -43,6 +46,7 @@ module Hongkong
           image = doc.search("#contentCTN .photo img").first
           document.image_url = URI::join(url, image["src"]).to_s if image
 
+          cleanup
           document
         end
       end

data/lib/hongkong/news/scrapers/phantom_scraper.rb

@@ -58,6 +58,30 @@ module Hongkong
           end
           @doc
         end
+
+        # call when shutdown phantomjs
+        def cleanup
+          wait_for_ajax
+          page.driver.reset!
+        end
+
+        private
+
+        # workaround for hang phantomjs
+        def wait_for_ajax
+          Timeout.timeout(Capybara.default_wait_time) do
+            loop until finished_all_ajax_requests?
+          end
+        end
+
+        def finished_all_ajax_requests?
+          begin
+            page.evaluate_script("(typeof jQuery !== \"undefined\") ? jQuery.active : 0").zero?
+          rescue Exception => e
+            puts "ignored excpetion wiating ajax: #{e}"
+          end
+        end
+
       end
     end
   end

data/lib/hongkong/news/scrapers/version.rb

@@ -1,7 +1,7 @@
 module Hongkong
   module News
     module Scrapers
-      VERSION = "0.4.1"
+      VERSION = "0.4.2"
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hongkong-news-scrapers
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Francis Chong