news2kindle 0.2.0 → 0.2.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: b6359f3d5bd815e2288ae183bdf574a429f64318
4
- data.tar.gz: 746ab24028ae050952b3b84a58d78ec8b0e7230c
2
+ SHA256:
3
+ metadata.gz: '00639cc8f196c01a7dd9595b204000f4af6ec26c30b7ed268223960373cc52ea'
4
+ data.tar.gz: 93992dce4e4b6ca3a87e28c36323018598c19c4433a82ed9ba79c492c65bc562
5
5
  SHA512:
6
- metadata.gz: 45d90ea6ee619e73aad9c61ad2cdcca1ecd5ef3493553618bb8ba79e534dc5582b5a6bd62f9c39e20a6420345537a886b1f16025b0dfa01162cce7875998d5ef
7
- data.tar.gz: 4a5fc8651f8163b566191c048fe49387599db5b6e6e5c381d68762fb5ed26a6137bb368a52dd5d5eadfc6adb6bedf5d6c151958d5900073a96f4b34273dfd676
6
+ metadata.gz: a312cbe553b155982a92cd423eea344182e3229ad2bfe51a7131e05979936a336a7f25bfa4bd4c398c2fec34942c1a52eb626cc59c3b6052792ef86c17cbcb7c
7
+ data.tar.gz: fd0bf057310b395dd90590fff11a44b16c1157675312088834aaf89e805754de12372802770457a91d49ce609ef777af1354775173b9e3d344c4bcb9879cf3dc
data/.travis.yml CHANGED
@@ -2,11 +2,12 @@ sudo: false
2
2
  cache: bundler
3
3
 
4
4
  before_install:
5
- - gem install bundler -v 1.16.0.pre.3
5
+ - gem update bundler
6
6
 
7
7
  rvm:
8
- - 2.3.5
9
- - 2.4.2
8
+ - 2.6.7
9
+ - 2.7.3
10
+ - 3.0.1
10
11
 
11
12
  script: bundle exec rake
12
13
 
data/Gemfile.lock CHANGED
@@ -1,12 +1,12 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- news2kindle (0.2.0)
4
+ news2kindle (0.2.6)
5
5
  dropbox_api
6
6
  kindlegen
7
7
  mail
8
8
  mechanize
9
- mongoid (~> 6.1)
9
+ mongoid (>= 6.1, < 8.0)
10
10
  nokogiri
11
11
  pit
12
12
  systemu
@@ -14,97 +14,109 @@ PATH
14
14
  GEM
15
15
  remote: https://rubygems.org/
16
16
  specs:
17
- activemodel (5.1.4)
18
- activesupport (= 5.1.4)
19
- activesupport (5.1.4)
17
+ activemodel (6.1.3.1)
18
+ activesupport (= 6.1.3.1)
19
+ activesupport (6.1.3.1)
20
20
  concurrent-ruby (~> 1.0, >= 1.0.2)
21
- i18n (~> 0.7)
22
- minitest (~> 5.1)
23
- tzinfo (~> 1.1)
24
- bson (4.2.2)
25
- coderay (1.1.2)
26
- concurrent-ruby (1.0.5)
27
- diff-lcs (1.3)
28
- domain_name (0.5.20170404)
21
+ i18n (>= 1.6, < 2)
22
+ minitest (>= 5.1)
23
+ tzinfo (~> 2.0)
24
+ zeitwerk (~> 2.3)
25
+ addressable (2.7.0)
26
+ public_suffix (>= 2.0.2, < 5.0)
27
+ bson (4.12.0)
28
+ concurrent-ruby (1.1.8)
29
+ connection_pool (2.2.5)
30
+ diff-lcs (1.4.4)
31
+ domain_name (0.5.20190701)
29
32
  unf (>= 0.0.5, < 1.0.0)
30
- dropbox_api (0.1.10)
31
- faraday (~> 0.9, ~> 0.8)
33
+ dropbox_api (0.1.19)
34
+ faraday (~> 1.0)
32
35
  oauth2 (~> 1.1)
33
- faraday (0.12.2)
36
+ faraday (1.4.1)
37
+ faraday-excon (~> 1.1)
38
+ faraday-net_http (~> 1.0)
39
+ faraday-net_http_persistent (~> 1.1)
34
40
  multipart-post (>= 1.2, < 3)
41
+ ruby2_keywords (>= 0.0.4)
42
+ faraday-excon (1.1.0)
43
+ faraday-net_http (1.0.1)
44
+ faraday-net_http_persistent (1.1.0)
35
45
  http-cookie (1.0.3)
36
46
  domain_name (~> 0.5)
37
- i18n (0.9.1)
47
+ i18n (1.8.10)
38
48
  concurrent-ruby (~> 1.0)
39
- jwt (1.5.6)
40
- kindlegen (3.0.3)
49
+ jwt (2.2.3)
50
+ kindlegen (3.1.1)
41
51
  rake
42
52
  rubyzip
43
- mail (2.7.0)
53
+ mail (2.7.1)
44
54
  mini_mime (>= 0.1.1)
45
- mechanize (2.7.5)
46
- domain_name (~> 0.5, >= 0.5.1)
47
- http-cookie (~> 1.0)
48
- mime-types (>= 1.17.2)
49
- net-http-digest_auth (~> 1.1, >= 1.1.1)
50
- net-http-persistent (~> 2.5, >= 2.5.2)
51
- nokogiri (~> 1.6)
52
- ntlm-http (~> 0.1, >= 0.1.1)
53
- webrobots (>= 0.0.9, < 0.2)
54
- method_source (0.9.0)
55
- mime-types (3.1)
55
+ mechanize (2.8.0)
56
+ addressable (~> 2.7)
57
+ domain_name (~> 0.5, >= 0.5.20190701)
58
+ http-cookie (~> 1.0, >= 1.0.3)
59
+ mime-types (~> 3.0)
60
+ net-http-digest_auth (~> 1.4, >= 1.4.1)
61
+ net-http-persistent (>= 2.5.2, < 5.0.dev)
62
+ nokogiri (~> 1.11, >= 1.11.2)
63
+ rubyntlm (~> 0.6, >= 0.6.3)
64
+ webrick (~> 1.7)
65
+ webrobots (~> 0.1.2)
66
+ mime-types (3.3.1)
56
67
  mime-types-data (~> 3.2015)
57
- mime-types-data (3.2016.0521)
58
- mini_mime (1.0.0)
59
- mini_portile2 (2.3.0)
60
- minitest (5.10.3)
61
- mongo (2.4.3)
62
- bson (>= 4.2.1, < 5.0.0)
63
- mongoid (6.2.1)
64
- activemodel (~> 5.1)
65
- mongo (>= 2.4.1, < 3.0.0)
66
- multi_json (1.12.2)
68
+ mime-types-data (3.2021.0225)
69
+ mini_mime (1.1.0)
70
+ minitest (5.14.4)
71
+ mongo (2.14.0)
72
+ bson (>= 4.8.2, < 5.0.0)
73
+ mongoid (7.2.2)
74
+ activemodel (>= 5.1, < 6.2)
75
+ mongo (>= 2.10.5, < 3.0.0)
76
+ multi_json (1.15.0)
67
77
  multi_xml (0.6.0)
68
- multipart-post (2.0.0)
78
+ multipart-post (2.1.1)
69
79
  net-http-digest_auth (1.4.1)
70
- net-http-persistent (2.9.4)
71
- nokogiri (1.8.1)
72
- mini_portile2 (~> 2.3.0)
73
- ntlm-http (0.1.1)
74
- oauth2 (1.4.0)
75
- faraday (>= 0.8, < 0.13)
76
- jwt (~> 1.0)
80
+ net-http-persistent (4.0.1)
81
+ connection_pool (~> 2.2)
82
+ nokogiri (1.11.3-x86_64-linux)
83
+ racc (~> 1.4)
84
+ oauth2 (1.4.7)
85
+ faraday (>= 0.8, < 2.0)
86
+ jwt (>= 1.0, < 3.0)
77
87
  multi_json (~> 1.3)
78
88
  multi_xml (~> 0.5)
79
89
  rack (>= 1.2, < 3)
80
90
  pit (0.0.7)
81
- pry (0.11.2)
82
- coderay (~> 1.1.0)
83
- method_source (~> 0.9.0)
84
- rack (2.0.3)
85
- rake (12.2.1)
86
- rspec (3.7.0)
87
- rspec-core (~> 3.7.0)
88
- rspec-expectations (~> 3.7.0)
89
- rspec-mocks (~> 3.7.0)
90
- rspec-core (3.7.0)
91
- rspec-support (~> 3.7.0)
92
- rspec-expectations (3.7.0)
91
+ public_suffix (4.0.6)
92
+ racc (1.5.2)
93
+ rack (2.2.3)
94
+ rake (13.0.3)
95
+ rspec (3.10.0)
96
+ rspec-core (~> 3.10.0)
97
+ rspec-expectations (~> 3.10.0)
98
+ rspec-mocks (~> 3.10.0)
99
+ rspec-core (3.10.1)
100
+ rspec-support (~> 3.10.0)
101
+ rspec-expectations (3.10.1)
93
102
  diff-lcs (>= 1.2.0, < 2.0)
94
- rspec-support (~> 3.7.0)
95
- rspec-mocks (3.7.0)
103
+ rspec-support (~> 3.10.0)
104
+ rspec-mocks (3.10.2)
96
105
  diff-lcs (>= 1.2.0, < 2.0)
97
- rspec-support (~> 3.7.0)
98
- rspec-support (3.7.0)
99
- rubyzip (1.2.1)
106
+ rspec-support (~> 3.10.0)
107
+ rspec-support (3.10.2)
108
+ ruby2_keywords (0.0.4)
109
+ rubyntlm (0.6.3)
110
+ rubyzip (2.3.0)
100
111
  systemu (2.6.5)
101
- thread_safe (0.3.6)
102
- tzinfo (1.2.4)
103
- thread_safe (~> 0.1)
112
+ tzinfo (2.0.4)
113
+ concurrent-ruby (~> 1.0)
104
114
  unf (0.1.4)
105
115
  unf_ext
106
- unf_ext (0.0.7.4)
116
+ unf_ext (0.0.7.7)
117
+ webrick (1.7.0)
107
118
  webrobots (0.1.2)
119
+ zeitwerk (2.4.2)
108
120
 
109
121
  PLATFORMS
110
122
  ruby
@@ -112,9 +124,8 @@ PLATFORMS
112
124
  DEPENDENCIES
113
125
  bundler
114
126
  news2kindle!
115
- pry
116
127
  rake
117
128
  rspec
118
129
 
119
130
  BUNDLED WITH
120
- 1.16.0.pre.3
131
+ 2.2.15
@@ -31,9 +31,9 @@ module News2Kindle
31
31
  now = opts[:now]
32
32
  items = []
33
33
 
34
- rdf_file = "http://rss.rssad.jp/rss/internetwatch/internet.rdf"
34
+ rdf_file = "https://internet.watch.impress.co.jp/data/rss/1.0/iw/feed.rdf"
35
35
  rdf = retry_loop( 5 ) do
36
- Nokogiri(open(rdf_file, 'r:utf-8', &:read))
36
+ Nokogiri(URI.open(rdf_file, 'r:utf-8', &:read))
37
37
  end
38
38
  (rdf / 'item' ).each do |item|
39
39
  uri = URI( item.attr( 'rdf:about' ).to_s )
@@ -63,7 +63,7 @@ module News2Kindle
63
63
  next if org =~ /^http/ # skip images on other servers
64
64
  begin
65
65
  img_file = retry_loop( 5 ) do
66
- open( "#{TOP}#{org}", &:read )
66
+ URI.open( "#{TOP}#{org}", &:read )
67
67
  end
68
68
  cache = "#{org.gsub( /\//, '_' ).sub( /^_/, '' )}"
69
69
  open( "#{@dst_dir}/#{cache}", 'w' ){|f| f.write img_file}
@@ -205,7 +205,7 @@ module News2Kindle
205
205
  rescue Errno::ENOENT
206
206
  #puts "getting article: #{uri.path}".encode( Encoding::default_external )
207
207
  html = retry_loop( 5 ) do
208
- open( uri, &:read )
208
+ URI.open( uri, &:read )
209
209
  end
210
210
  open( cache, 'w' ){|f| f.write html }
211
211
  end
@@ -13,7 +13,6 @@ module News2Kindle
13
13
  class IllegalPage < StandardError; end
14
14
 
15
15
  TOP = 'https://www.nikkei.com'
16
- LOGIN = "#{TOP}/etc/accounts/login?dps=3&amp;pageflag=top&amp;url=http%3A%2F%2Fwww.nikkei.com%2F"
17
16
 
18
17
  def initialize( tmpdir )
19
18
  @nikkei_id, @nikkei_pw = auth
@@ -34,28 +33,31 @@ module News2Kindle
34
33
  @now_str = @now.strftime '%Y-%m-%d %H:%M'
35
34
 
36
35
  agent = Mechanize::new
36
+ agent.user_agent_alias = "Windows Chrome"
37
37
  agent.set_proxy( *ENV['HTTP_PROXY'].split( /:/ ) ) if ENV['HTTP_PROXY']
38
38
 
39
39
  toc = []
40
40
  if @nikkei_id and @nikkei_pw
41
- agent.get('https://regist.nikkei.com/ds/etc/accounts/logout')
42
- agent.get( LOGIN )
43
- agent.page.form_with( :name => 'autoPostForm' ).submit
44
- agent.page.form_with( :name => 'LA7010Form01' ) do |form|
45
- form['LA7010Form01:LA7010Email'] = @nikkei_id
46
- form['LA7010Form01:LA7010Password'] = @nikkei_pw
41
+ agent.get('https://id.nikkei.com/lounge/li/main/')
42
+ agent.page.form_with(name: 'LA0120Form01') do |form|
43
+ form['LA0120Form01:LA0120Email'] = @nikkei_id
44
+ form['LA0120Form01:LA0120Password'] = @nikkei_pw
47
45
  form.click_button
48
46
  end
49
- agent.page.forms.first.submit
47
+ begin # skip the confirmation page if exist
48
+ agent.get(TOP)
49
+ agent.page.form_with(name: 'autoPostForm').click_button
50
+ rescue NoMethodError
51
+ end
50
52
  else
51
- agent.get( TOP )
53
+ agent.get(TOP)
52
54
  end
53
55
 
54
56
  #
55
57
  # scraping top news
56
58
  #
57
59
  toc_top = ['TOP NEWS']
58
- (agent.page / '#JSID_baseRefreshNxTop2 h3 a').each do |a|
60
+ (agent.page / 'div.m-miM11_box h3 a').each do |a|
59
61
  uri = a.attr('href')
60
62
  next if News2Kindle::DupChecker.dup?(uri)
61
63
  toc_top << [canonical(a.text.strip), uri]
@@ -21,7 +21,7 @@ module News2Kindle
21
21
  html = title = author = now_str = nil
22
22
  begin
23
23
  retry_loop( 5 ) do
24
- html = Nokogiri(open("#{@top}?date=#{now.strftime '%m%d'}", 'r:utf-8', &:read))
24
+ html = Nokogiri(URI.open("#{@top}?date=#{now.strftime '%m%d'}", 'r:utf-8', &:read))
25
25
  title = (html / 'head title').text
26
26
  author = (html / 'head meta[name="author"]')[0]['content']
27
27
  now_str = now.strftime( '%m-%d' )
@@ -124,7 +124,7 @@ module News2Kindle
124
124
  file_name = "#{SecureRandom.hex}#{uri.to_s.scan(/\.[^\.]+$/)[0]}"
125
125
  begin
126
126
  open("#{@current_dir}/#{file_name}", 'w') do |f|
127
- f.write open(uri, &:read)
127
+ f.write URI.open(uri, &:read)
128
128
  end
129
129
  rescue OpenURI::HTTPError, RuntimeError, Errno::ENOENT
130
130
  News2Kindle.logger.warn "#$!: #{uri}"
@@ -1,3 +1,3 @@
1
1
  module News2Kindle
2
- VERSION = "0.2.0"
2
+ VERSION = "0.2.6"
3
3
  end
data/news2kindle.gemspec CHANGED
@@ -28,10 +28,9 @@ Gem::Specification.new do |spec|
28
28
  spec.add_dependency 'nokogiri'
29
29
  spec.add_dependency 'pit'
30
30
  spec.add_dependency 'dropbox_api'
31
- spec.add_dependency 'mongoid', '~> 6.1'
31
+ spec.add_dependency 'mongoid', '>= 6.1', '< 8.0'
32
32
 
33
33
  spec.add_development_dependency "bundler"
34
34
  spec.add_development_dependency "rake"
35
35
  spec.add_development_dependency "rspec"
36
- spec.add_development_dependency "pry"
37
36
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: news2kindle
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.2.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - TADA Tadashi
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-11-14 00:00:00.000000000 Z
11
+ date: 2021-05-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: kindlegen
@@ -112,16 +112,22 @@ dependencies:
112
112
  name: mongoid
113
113
  requirement: !ruby/object:Gem::Requirement
114
114
  requirements:
115
- - - "~>"
115
+ - - ">="
116
116
  - !ruby/object:Gem::Version
117
117
  version: '6.1'
118
+ - - "<"
119
+ - !ruby/object:Gem::Version
120
+ version: '8.0'
118
121
  type: :runtime
119
122
  prerelease: false
120
123
  version_requirements: !ruby/object:Gem::Requirement
121
124
  requirements:
122
- - - "~>"
125
+ - - ">="
123
126
  - !ruby/object:Gem::Version
124
127
  version: '6.1'
128
+ - - "<"
129
+ - !ruby/object:Gem::Version
130
+ version: '8.0'
125
131
  - !ruby/object:Gem::Dependency
126
132
  name: bundler
127
133
  requirement: !ruby/object:Gem::Requirement
@@ -164,20 +170,6 @@ dependencies:
164
170
  - - ">="
165
171
  - !ruby/object:Gem::Version
166
172
  version: '0'
167
- - !ruby/object:Gem::Dependency
168
- name: pry
169
- requirement: !ruby/object:Gem::Requirement
170
- requirements:
171
- - - ">="
172
- - !ruby/object:Gem::Version
173
- version: '0'
174
- type: :development
175
- prerelease: false
176
- version_requirements: !ruby/object:Gem::Requirement
177
- requirements:
178
- - - ">="
179
- - !ruby/object:Gem::Version
180
- version: '0'
181
173
  description: scrape some news site and deliver to kindle
182
174
  email:
183
175
  - t@tdtds.jp
@@ -188,7 +180,6 @@ extra_rdoc_files: []
188
180
  files:
189
181
  - ".gitignore"
190
182
  - ".rspec"
191
- - ".tachikoma.yml"
192
183
  - ".travis.yml"
193
184
  - Gemfile
194
185
  - Gemfile.lock
@@ -237,8 +228,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
237
228
  - !ruby/object:Gem::Version
238
229
  version: '0'
239
230
  requirements: []
240
- rubyforge_project:
241
- rubygems_version: 2.5.2.1
231
+ rubygems_version: 3.1.6
242
232
  signing_key:
243
233
  specification_version: 4
244
234
  summary: scrape some news site and deliver to kindle
data/.tachikoma.yml DELETED
@@ -1 +0,0 @@
1
- strategy: 'bundler'