news2kindle 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e6bdc099112592b2077a306b7df812677d5bbe0e
-  data.tar.gz: 22859f13bff4dab410b8db1b5b684ee216fd8766
+  metadata.gz: b6359f3d5bd815e2288ae183bdf574a429f64318
+  data.tar.gz: 746ab24028ae050952b3b84a58d78ec8b0e7230c
 SHA512:
-  metadata.gz: 871735c4e40b58d0a86bbde9c81b49fa3b1a59c488f8f634d686a9ad750e5c2a6408fa7635b76a193651f2e180ec2213d4f39237e02c497791e8f53eb118226d
-  data.tar.gz: 2e32d11d43dd96038167e5e8fa402b085485e04e7fcdbb74fa9b8d73e82899b28cd6a61c62509a6634b03542fa4abc66e206065745a28006a6e341afc03456ee
+  metadata.gz: 45d90ea6ee619e73aad9c61ad2cdcca1ecd5ef3493553618bb8ba79e534dc5582b5a6bd62f9c39e20a6420345537a886b1f16025b0dfa01162cce7875998d5ef
+  data.tar.gz: 4a5fc8651f8163b566191c048fe49387599db5b6e6e5c381d68762fb5ed26a6137bb368a52dd5d5eadfc6adb6bedf5d6c151958d5900073a96f4b34273dfd676

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    news2kindle (0.1.5)
+    news2kindle (0.2.0)
       dropbox_api
       kindlegen
       mail
@@ -55,7 +55,7 @@ GEM
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
-    mini_mime (0.1.4)
+    mini_mime (1.0.0)
     mini_portile2 (2.3.0)
     minitest (5.10.3)
     mongo (2.4.3)

data/bin/test-generator

@@ -12,6 +12,7 @@ News2Kindle::DupChecker.setup({
 })
 
 ARGV.each do |task|
+	News2Kindle.logger.level = Logger::DEBUG
 	require "news2kindle/generator/#{task}"
 	gen = News2Kindle::Generator.const_get(task.split(/-/).map{|a|a.capitalize}.join)
 	Dir::mkdir(task)

data/lib/news2kindle/dup_checker.rb

@@ -31,7 +31,7 @@ module News2Kindle
 				else
 					return true
 				end
-			rescue Moped::Errors::ConnectionFailure
+			rescue
 				News2Kindle.logger.error $!
 				@@mongoid_conf = nil
 				return false

data/lib/news2kindle/generator/nikkei-paid.rb

@@ -55,28 +55,25 @@ module News2Kindle
 				# scraping top news
 				#
 				toc_top = ['TOP NEWS']
-				%w(first second third fourth).each do |category|
-					(agent.page / "div.nx-top_news_#{category} h3 a").each do |a|
-						uri = a.attr('href')
-						next if News2Kindle::DupChecker.dup?(uri)
-						toc_top << [canonical( a.text.strip ), uri]
-					end
+				(agent.page / '#JSID_baseRefreshNxTop2 h3 a').each do |a|
+					uri = a.attr('href')
+					next if News2Kindle::DupChecker.dup?(uri)
+					toc_top << [canonical(a.text.strip), uri]
 				end
 				toc << toc_top
 
 				#
 				# scraping all categories
 				#
-				(agent.page / 'div.cmnc-genre').each do |genre|
-					toc_cat = []
-					(genre / 'h4.cmnc-genre_title a.cmnc-title_text').each do |cat|
-						next if /local/ =~ cat.attr( 'href' )
-						toc_cat << cat.text
-						(genre / 'li a').each do |article|
-							uri = article.attr('href')
-							next if News2Kindle::DupChecker.dup?(uri)
-							toc_cat << [canonical( article.text ), uri]
-						end
+				(agent.page / 'div.m-miM11_box').each do |genre|
+					headline = genre / 'div.m-headline h3'
+					toc_cat = [headline.text]
+					agent.get((headline / 'a').attr('href'))
+					(agent.page / '#CONTENTS_MAIN h3 a').each do |article|
+						uri = article.attr('href')
+						next unless article.attr('href') =~ %r|^/article/|
+						next if News2Kindle::DupChecker.dup?(uri)
+						toc_cat << [canonical(article.text), uri]
 					end
 					toc << toc_cat
 				end
@@ -150,7 +147,7 @@ module News2Kindle
 					begin
 						#puts "getting html #{aid}#{sub}"
 						retry_loop( 5 ) do
-							agent.get( "#{TOP}#{uri}" )
+							agent.get("#{TOP}/news/print-article/?ng=#{aid}")
 							html = agent.page.root
 							sleep 1
 						end

data/lib/news2kindle/version.rb

@@ -1,3 +1,3 @@
 module News2Kindle
-  VERSION = "0.1.5"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: news2kindle
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - TADA Tadashi
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-06 00:00:00.000000000 Z
+date: 2017-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kindlegen