web_stat 0.3.14 → 0.3.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +15 -3
  3. data/lib/web_stat.rb +1 -0
  4. data/lib/web_stat/fetch/fetch_as_web.rb +25 -1
  5. data/lib/web_stat/version.rb +1 -1
  6. data/spec/fixtures/pdfs/newsdict.blog.pdf +0 -0
  7. data/spec/fixtures/pdfs/rfc2616.pdf +0 -0
  8. data/spec/spec_helper.rb +17 -2
  9. data/spec/web_stat/fetch_spec.rb +101 -1
  10. data/web_stat.gemspec +2 -1
  11. metadata +22 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 549db2077648ce028b556a72126335f05336155f86c4b18d47856f219c71fff0
4
- data.tar.gz: be8e0cee272fc20013659346608bf9a50a69dc48e777a46214c29f5c1865232d
3
+ metadata.gz: 82732f779004a5a2ef1f7259ce40126de8f51dfb07e46cd4972aafd3ab386ac3
4
+ data.tar.gz: 17c2b99bdeb5db8c134d107fc9d4957dab985c7e680b1c92060bf7090315477b
5
5
  SHA512:
6
- metadata.gz: c78fa085f475c7cdf0747b4c777e357503ce41929bbe5462ccda7b28a6cf4f20a5394deb7853ee11570a4b3338573e392f8697e61484649fe06beadc54aa38a8
7
- data.tar.gz: 941f0de20548a37899ac7610bd95b381ac18ca71761ccfb7f7788c299dd41ff521ae7eea283df64ee624044f919d2951f396ec38c3c4a15ded378ee0acbd0a20
6
+ metadata.gz: d6691a57b0498fcfbb609042cdb7e12b4ac453b5cedaf6dd39671090c97fc19958a4162ba50947392116320ba78c9a21491acab27be2a41287825d3d4d2194d1
7
+ data.tar.gz: 1e2e3d33f5c232532f442bbd16540c47810c93caef436a351124165b5294ddc7354d766ea66aa24ebc052c69432a639a6d5ef7b61a7cde896f6dc4676f95650d
data/Gemfile.lock CHANGED
@@ -1,12 +1,13 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- web_stat (0.3.12)
4
+ web_stat (0.3.15)
5
5
  bundler (>= 2.0.2)
6
6
  cld (>= 0.8.0)
7
7
  mechanize (>= 2.7)
8
8
  natto (>= 1.1.2)
9
9
  nokogiri (>= 1.10.4)
10
+ pdf-reader (= 2.4.0)
10
11
  ruby-readability (>= 0.7)
11
12
  sanitize (>= 5.0.0)
12
13
  selenium-webdriver (= 3.142.7)
@@ -14,8 +15,10 @@ PATH
14
15
  GEM
15
16
  remote: https://rubygems.org/
16
17
  specs:
18
+ Ascii85 (1.0.3)
17
19
  addressable (2.7.0)
18
20
  public_suffix (>= 2.0.2, < 5.0)
21
+ afm (0.2.2)
19
22
  byebug (11.1.3)
20
23
  childprocess (3.0.0)
21
24
  cld (0.8.0)
@@ -31,6 +34,7 @@ GEM
31
34
  ffi (1.13.1)
32
35
  guess_html_encoding (0.0.11)
33
36
  hashdiff (1.0.1)
37
+ hashery (2.1.2)
34
38
  http-cookie (1.0.3)
35
39
  domain_name (~> 0.5)
36
40
  mechanize (2.7.6)
@@ -57,6 +61,12 @@ GEM
57
61
  nokogumbo (2.0.2)
58
62
  nokogiri (~> 1.8, >= 1.8.4)
59
63
  ntlm-http (0.1.1)
64
+ pdf-reader (2.4.0)
65
+ Ascii85 (~> 1.0.0)
66
+ afm (~> 0.2.1)
67
+ hashery (~> 2.0)
68
+ ruby-rc4
69
+ ttfunk
60
70
  pry (0.13.1)
61
71
  coderay (~> 1.1)
62
72
  method_source (~> 1.0)
@@ -78,18 +88,20 @@ GEM
78
88
  diff-lcs (>= 1.2.0, < 2.0)
79
89
  rspec-support (~> 3.9.0)
80
90
  rspec-support (3.9.3)
91
+ ruby-rc4 (0.1.5)
81
92
  ruby-readability (0.7.0)
82
93
  guess_html_encoding (>= 0.0.4)
83
94
  nokogiri (>= 1.6.0)
84
95
  rubyzip (2.3.0)
85
96
  safe_yaml (1.0.5)
86
- sanitize (5.2.0)
97
+ sanitize (5.2.1)
87
98
  crass (~> 1.0.2)
88
99
  nokogiri (>= 1.8.0)
89
100
  nokogumbo (~> 2.0)
90
101
  selenium-webdriver (3.142.7)
91
102
  childprocess (>= 0.5, < 4.0)
92
103
  rubyzip (>= 1.2.2)
104
+ ttfunk (1.6.2.1)
93
105
  unf (0.1.4)
94
106
  unf_ext
95
107
  unf_ext (0.0.7.7)
@@ -108,7 +120,7 @@ DEPENDENCIES
108
120
  rake (>= 10.0)
109
121
  rspec (>= 3.0)
110
122
  web_stat!
111
- webmock (>= 3.6.0)
123
+ webmock (>= 3.8.3)
112
124
 
113
125
  BUNDLED WITH
114
126
  2.1.4
data/lib/web_stat.rb CHANGED
@@ -8,6 +8,7 @@ require 'sanitize'
8
8
  require 'nokogiri'
9
9
  require 'open-uri'
10
10
  require 'net/http'
11
+ require 'pdf/reader'
11
12
  require 'ruby-readability'
12
13
  require 'selenium-webdriver'
13
14
 
data/lib/web_stat/fetch/fetch_as_web.rb CHANGED
@@ -9,7 +9,31 @@ module WebStat
9
9
  raise WebStat::INVALID_URL, url
10
10
  end
11
11
  @url = original_url(url)
12
- @html = get_url(@url)
12
+ if @url.match?(/\.pdf$/)
13
+ title = nil
14
+ body = nil
15
+ URI.open(@url) do |io|
16
+ reader = PDF::Reader.new(io)
17
+ if reader.info.key?(:Title)
18
+ title = reader.info[:Title]
19
+ else
20
+ title = File.basename(@url, ".pdf")
21
+ end
22
+ body = reader.pages.first.text
23
+ end
24
+ @html = <<-"EOS"
25
+ <html>
26
+ <head>
27
+ <title>#{title}</title>
28
+ </head>
29
+ <body>
30
+ #{body}
31
+ </body>
32
+ </html>
33
+ EOS
34
+ else
35
+ @html = get_url(@url)
36
+ end
13
37
  @nokogiri = ::Nokogiri::HTML(@html)
14
38
  end
15
39
 
data/lib/web_stat/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module WebStat
2
- VERSION = "0.3.14"
2
+ VERSION = "0.3.15"
3
3
  end
data/spec/spec_helper.rb CHANGED
@@ -6,8 +6,9 @@ require "web_stat"
6
6
 
7
7
  require 'webmock'
8
8
  include WebMock::API
9
+ ENV['ENV'] = 'test'
9
10
  WebMock.enable!
10
-
11
+
11
12
  RSpec.configure do |config|
12
13
  # Enable flags like --only-failures and --next-failure
13
14
  config.example_status_persistence_file_path = ".rspec_status"
@@ -52,7 +53,14 @@ module WebStatTestHelper
52
53
  # Get htmls of fixture
53
54
  def scheme_and_files
54
55
  Dir.glob(File.join(File.dirname(__FILE__), "fixtures", "htmls", "*.html")).map do |file|
55
- "https://newsdict.blog/#{File.basename(file)}"
56
+ "https://newsdict.blog/#{File.basename(file)}"
57
+ end
58
+ end
59
+
60
+ # Get pdfs of fixture
61
+ def pdfs
62
+ Dir.glob(File.join(File.dirname(__FILE__), "fixtures", "pdfs", "*.pdf")).map do |file|
63
+ "https://newsdict.blog/#{File.basename(file)}"
56
64
  end
57
65
  end
58
66
  end
@@ -67,6 +75,13 @@ WebStatTestHelper.scheme_and_files.each do |url|
67
75
  body: File.new(File.join(File.dirname(__FILE__), "fixtures", "htmls", File.basename(url))),
68
76
  headers: {content_type: 'application/html; charset=utf-8'})
69
77
  end
78
+ WebStatTestHelper.pdfs.each do |url|
79
+ WebMock.stub_request(:get, url)
80
+ .to_return(
81
+ status: 200,
82
+ body: File.read(File.join(File.dirname(__FILE__), "fixtures", "pdfs", File.basename(url))),
83
+ headers: {content_type: 'application/pdf'})
84
+ end
70
85
 
71
86
  WebMock.stub_request(:get, "https://newsdict.blog/robots.txt")
72
87
  .to_return(
data/spec/web_stat/fetch_spec.rb CHANGED
@@ -66,6 +66,106 @@ RSpec.describe WebStat::Fetch do
66
66
  end
67
67
  end
68
68
 
69
+ [{fixture: "https://newsdict.blog/rfc2616.pdf", class: WebStat::FetchAsWeb}].each do |fetch|
70
+
71
+ it "Get title by #{fetch[:class].to_s}" do
72
+ web_stat = fetch[:class].new(fetch[:fixture])
73
+ expect(web_stat.title).to eq "Microsoft Word"
74
+ end
75
+
76
+ it "Get site name by #{fetch[:class].to_s}" do
77
+ web_stat = fetch[:class].new(fetch[:fixture])
78
+ expect(web_stat.site_name).to eq "RFC2616.doc"
79
+ end
80
+
81
+ it "Get Document's content by #{fetch[:class].to_s}" do
82
+ web_stat = fetch[:class].new(fetch[:fixture])
83
+ expect(web_stat.content).not_to eq nil
84
+ end
85
+
86
+ it "WebStat content do not include html by #{fetch[:class].to_s}" do
87
+ web_stat = fetch[:class].new(fetch[:fixture])
88
+ expect(Sanitize.clean(web_stat.content).length).to eq web_stat.content.length
89
+ end
90
+
91
+ it "Get eyecatch image blob by #{fetch[:class].to_s}" do
92
+ web_stat = fetch[:class].new(fetch[:fixture])
93
+ web_stat.url = "https://newsdict.blog"
94
+ unless web_stat.stat[:eyecatch_image_path].nil?
95
+ image = File.read(web_stat.stat[:eyecatch_image_path])
96
+ expect(image.encoding.to_s).to eq("UTF-8")
97
+ end
98
+ end
99
+
100
+ it "Get eyecatch image path by #{fetch[:class].to_s}" do
101
+ web_stat = fetch[:class].new(fetch[:fixture])
102
+ web_stat.url = "https://newsdict.blog"
103
+ expect(web_stat.eyecatch_image_path).to be_string_or_nil
104
+ end
105
+
106
+ it "Get language_iso by #{fetch[:class].to_s}" do
107
+ web_stat = fetch[:class].new(fetch[:fixture])
108
+ web_stat.url = "https://newsdict.blog"
109
+ expect(web_stat.stat[:language_code]).to eq("en")
110
+ end
111
+
112
+ it "Get local path of eyecatch image by #{fetch[:class].to_s}" do
113
+ web_stat = fetch[:class].new(fetch[:fixture])
114
+ web_stat.url = "https://newsdict.blog"
115
+ expect(web_stat.stat[:eyecatch_image_path]).to be_tmp_file_or_nil
116
+ end
117
+ end
118
+
119
+ [{fixture: "https://newsdict.blog/newsdict.blog.pdf", class: WebStat::FetchAsWeb}].each do |fetch|
120
+
121
+ it "Get title by #{fetch[:class].to_s}" do
122
+ web_stat = fetch[:class].new(fetch[:fixture])
123
+ expect(web_stat.title).to eq "newsdict.blog"
124
+ end
125
+
126
+ it "Get site name by #{fetch[:class].to_s}" do
127
+ web_stat = fetch[:class].new(fetch[:fixture])
128
+ expect(web_stat.site_name).to eq "newsdict.blog"
129
+ end
130
+
131
+ it "Get Document's content by #{fetch[:class].to_s}" do
132
+ web_stat = fetch[:class].new(fetch[:fixture])
133
+ expect(web_stat.content).not_to eq nil
134
+ end
135
+
136
+ it "WebStat content do not include html by #{fetch[:class].to_s}" do
137
+ web_stat = fetch[:class].new(fetch[:fixture])
138
+ expect(Sanitize.clean(web_stat.content).length).to eq web_stat.content.length
139
+ end
140
+
141
+ it "Get eyecatch image blob by #{fetch[:class].to_s}" do
142
+ web_stat = fetch[:class].new(fetch[:fixture])
143
+ web_stat.url = "https://newsdict.blog"
144
+ unless web_stat.stat[:eyecatch_image_path].nil?
145
+ image = File.read(web_stat.stat[:eyecatch_image_path])
146
+ expect(image.encoding.to_s).to eq("UTF-8")
147
+ end
148
+ end
149
+
150
+ it "Get eyecatch image path by #{fetch[:class].to_s}" do
151
+ web_stat = fetch[:class].new(fetch[:fixture])
152
+ web_stat.url = "https://newsdict.blog"
153
+ expect(web_stat.eyecatch_image_path).to be_string_or_nil
154
+ end
155
+
156
+ it "Get language_iso by #{fetch[:class].to_s}" do
157
+ web_stat = fetch[:class].new(fetch[:fixture])
158
+ web_stat.url = "https://newsdict.blog"
159
+ expect(web_stat.stat[:language_code]).to eq("ja")
160
+ end
161
+
162
+ it "Get local path of eyecatch image by #{fetch[:class].to_s}" do
163
+ web_stat = fetch[:class].new(fetch[:fixture])
164
+ web_stat.url = "https://newsdict.blog"
165
+ expect(web_stat.stat[:eyecatch_image_path]).to be_tmp_file_or_nil
166
+ end
167
+ end
168
+
69
169
  it "WebStat.stat_by_html" do
70
170
  WebStatTestHelper.htmls.each do |fixture|
71
171
  web_stat = WebStat.stat_by_html(fixture, "https://newsdict.blog")
@@ -104,4 +204,4 @@ RSpec.describe WebStat::Fetch do
104
204
  expect(web_stat_fetch_web_class.url_valid?("https://gxyt4.app.goo.gl/Mn64U")).to be true
105
205
  expect(web_stat_fetch_web_class.url_valid?("https://status.cloud.google.com/incident/cloud-functions/19010")).to be true
106
206
  end
107
- end
207
+ end
data/web_stat.gemspec CHANGED
@@ -28,10 +28,11 @@ Gem::Specification.new do |spec|
28
28
  spec.add_runtime_dependency "sanitize", ">= 5.0.0"
29
29
  spec.add_runtime_dependency "cld", ">= 0.8.0"
30
30
  spec.add_runtime_dependency "selenium-webdriver", "= 3.142.7"
31
+ spec.add_runtime_dependency "pdf-reader", "2.4.0"
31
32
 
32
33
  spec.add_development_dependency "rake", ">= 10.0"
33
34
  spec.add_development_dependency "rspec", ">= 3.0"
34
35
  spec.add_development_dependency "pry", ">= 0.13.1"
35
- spec.add_development_dependency "webmock", ">= 3.6.0"
36
+ spec.add_development_dependency "webmock", ">= 3.8.3"
36
37
  spec.add_development_dependency "pry-byebug", "3.9.0"
37
38
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: web_stat
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.14
4
+ version: 0.3.15
5
5
  platform: ruby
6
6
  authors:
7
7
  - yusuke abe
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-06-21 00:00:00.000000000 Z
11
+ date: 2020-07-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -122,6 +122,20 @@ dependencies:
122
122
  - - '='
123
123
  - !ruby/object:Gem::Version
124
124
  version: 3.142.7
125
+ - !ruby/object:Gem::Dependency
126
+ name: pdf-reader
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - '='
130
+ - !ruby/object:Gem::Version
131
+ version: 2.4.0
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - '='
137
+ - !ruby/object:Gem::Version
138
+ version: 2.4.0
125
139
  - !ruby/object:Gem::Dependency
126
140
  name: rake
127
141
  requirement: !ruby/object:Gem::Requirement
@@ -170,14 +184,14 @@ dependencies:
170
184
  requirements:
171
185
  - - ">="
172
186
  - !ruby/object:Gem::Version
173
- version: 3.6.0
187
+ version: 3.8.3
174
188
  type: :development
175
189
  prerelease: false
176
190
  version_requirements: !ruby/object:Gem::Requirement
177
191
  requirements:
178
192
  - - ">="
179
193
  - !ruby/object:Gem::Version
180
- version: 3.6.0
194
+ version: 3.8.3
181
195
  - !ruby/object:Gem::Dependency
182
196
  name: pry-byebug
183
197
  requirement: !ruby/object:Gem::Requirement
@@ -233,6 +247,8 @@ files:
233
247
  - spec/fixtures/htmls/image.html
234
248
  - spec/fixtures/images/facebook-3.jpg
235
249
  - spec/fixtures/images/newsdict-5d8601394c3f4eea2d7161ab92ab327ac7099e22214c853327011b3a71859b8e.png
250
+ - spec/fixtures/pdfs/newsdict.blog.pdf
251
+ - spec/fixtures/pdfs/rfc2616.pdf
236
252
  - spec/spec_helper.rb
237
253
  - spec/web_stat/configure_spec.rb
238
254
  - spec/web_stat/fetch_spec.rb
@@ -268,6 +284,8 @@ test_files:
268
284
  - spec/fixtures/htmls/image.html
269
285
  - spec/fixtures/images/facebook-3.jpg
270
286
  - spec/fixtures/images/newsdict-5d8601394c3f4eea2d7161ab92ab327ac7099e22214c853327011b3a71859b8e.png
287
+ - spec/fixtures/pdfs/newsdict.blog.pdf
288
+ - spec/fixtures/pdfs/rfc2616.pdf
271
289
  - spec/spec_helper.rb
272
290
  - spec/web_stat/configure_spec.rb
273
291
  - spec/web_stat/fetch_spec.rb