web_stat 0.1.12 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -1
  3. data/Gemfile.lock +34 -31
  4. data/README.md +2 -2
  5. data/lib/web_stat/fetch.rb +3 -8
  6. data/lib/web_stat/version.rb +1 -1
  7. data/lib/web_stat.rb +10 -0
  8. data/spec/web_stat/fetch_spec.rb +8 -0
  9. data/web_stat.gemspec +11 -10
  10. metadata +35 -21
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: daa2f034827920f7dbdfac48830325f67cf9e95a9b724b12e643342f9b2aa170
4
- data.tar.gz: 4a036530d8181f787522f884f5059eb321acc1ed142e6062ca3e442773d05c41
3
+ metadata.gz: 89fb8389fb61745427075c700e9ba094ed2fbfe78f22e6f9364d836472ba8334
4
+ data.tar.gz: 2f07517f36832489f2bd72f259d93d1ee60bab29247e4fd88990a29ee1d3f912
5
5
  SHA512:
6
- metadata.gz: 95cfc5fa7d23d6ed5bfa1f4375143acfd814a56623df73687b37cd18545fad1bbb0aa0112a231f3164459dd8dc64980c70680c15aee47a5053e3bdabb1e4c2aa
7
- data.tar.gz: 5024406dbe28de59e72b8d3cc0ccac411cc8ebcd5b74d8f40b2c861b802f3de0eaab6b4a29f3c67ed98498f159346570fc3c395cd7b8686c92c374e6d4eb4d67
6
+ metadata.gz: 56e2f7e8ed3ad8e44100f5ac767206cbdfda1e8cac431813a005998823acbca4060d441ddf723dd17b46fbfbbd37fd9700f3e459ed99fe3e73eaadb42ca2c6d0
7
+ data.tar.gz: 64923ad006daa2db6cd196d2c6f3256b011673d48fc74eb315556aebea950550c7c1dc6c123c97b4fd55ef4ae2a1d3714ce93b62d34b8f61d16346dc9a1f870e
data/.gitignore CHANGED
@@ -1,4 +1,4 @@
1
- /.bundle/
1
+ /vendor/
2
2
  /.yardoc
3
3
  /_yardoc/
4
4
  /coverage/
data/Gemfile.lock CHANGED
@@ -1,29 +1,32 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- web_stat (0.1.12)
4
+ web_stat (0.2.0)
5
5
  bundler (>= 2.0.2)
6
- final_redirect_url (~> 0.1.0)
7
- mechanize (~> 2.7)
8
- natto (~> 1.1.2)
6
+ cld (>= 0.8.0)
7
+ final_redirect_url (>= 0.1.0)
8
+ mechanize (>= 2.7)
9
+ natto (>= 1.1.2)
9
10
  nokogiri (>= 1.10.4)
10
- ruby-readability (~> 0.7)
11
- sanitize (~> 5.0.0)
11
+ ruby-readability (>= 0.7)
12
+ sanitize (>= 5.0.0)
12
13
 
13
14
  GEM
14
15
  remote: https://rubygems.org/
15
16
  specs:
16
17
  addressable (2.7.0)
17
18
  public_suffix (>= 2.0.2, < 5.0)
19
+ cld (0.8.0)
20
+ ffi
18
21
  coderay (1.1.2)
19
22
  connection_pool (2.2.2)
20
23
  crack (0.4.3)
21
24
  safe_yaml (~> 1.0.0)
22
- crass (1.0.4)
25
+ crass (1.0.5)
23
26
  diff-lcs (1.3)
24
27
  domain_name (0.5.20190701)
25
28
  unf (>= 0.0.5, < 1.0.0)
26
- ffi (1.11.1)
29
+ ffi (1.11.3)
27
30
  final_redirect_url (0.1.0)
28
31
  guess_html_encoding (0.0.11)
29
32
  hashdiff (1.0.0)
@@ -39,50 +42,50 @@ GEM
39
42
  ntlm-http (~> 0.1, >= 0.1.1)
40
43
  webrobots (>= 0.0.9, < 0.2)
41
44
  method_source (0.9.2)
42
- mime-types (3.3)
45
+ mime-types (3.3.1)
43
46
  mime-types-data (~> 3.2015)
44
- mime-types-data (3.2019.0904)
47
+ mime-types-data (3.2019.1009)
45
48
  mini_portile2 (2.4.0)
46
49
  natto (1.1.2)
47
50
  ffi (>= 1.9.0)
48
51
  net-http-digest_auth (1.4.1)
49
52
  net-http-persistent (3.1.0)
50
53
  connection_pool (~> 2.2)
51
- nokogiri (1.10.4)
54
+ nokogiri (1.10.7)
52
55
  mini_portile2 (~> 2.4.0)
53
- nokogumbo (2.0.1)
56
+ nokogumbo (2.0.2)
54
57
  nokogiri (~> 1.8, >= 1.8.4)
55
58
  ntlm-http (0.1.1)
56
59
  pry (0.12.2)
57
60
  coderay (~> 1.1.0)
58
61
  method_source (~> 0.9.0)
59
- public_suffix (4.0.1)
60
- rake (10.5.0)
61
- rspec (3.8.0)
62
- rspec-core (~> 3.8.0)
63
- rspec-expectations (~> 3.8.0)
64
- rspec-mocks (~> 3.8.0)
65
- rspec-core (3.8.2)
66
- rspec-support (~> 3.8.0)
67
- rspec-expectations (3.8.4)
62
+ public_suffix (4.0.2)
63
+ rake (13.0.1)
64
+ rspec (3.9.0)
65
+ rspec-core (~> 3.9.0)
66
+ rspec-expectations (~> 3.9.0)
67
+ rspec-mocks (~> 3.9.0)
68
+ rspec-core (3.9.1)
69
+ rspec-support (~> 3.9.1)
70
+ rspec-expectations (3.9.0)
68
71
  diff-lcs (>= 1.2.0, < 2.0)
69
- rspec-support (~> 3.8.0)
70
- rspec-mocks (3.8.1)
72
+ rspec-support (~> 3.9.0)
73
+ rspec-mocks (3.9.1)
71
74
  diff-lcs (>= 1.2.0, < 2.0)
72
- rspec-support (~> 3.8.0)
73
- rspec-support (3.8.2)
75
+ rspec-support (~> 3.9.0)
76
+ rspec-support (3.9.2)
74
77
  ruby-readability (0.7.0)
75
78
  guess_html_encoding (>= 0.0.4)
76
79
  nokogiri (>= 1.6.0)
77
80
  safe_yaml (1.0.5)
78
- sanitize (5.0.0)
81
+ sanitize (5.1.0)
79
82
  crass (~> 1.0.2)
80
83
  nokogiri (>= 1.8.0)
81
84
  nokogumbo (~> 2.0)
82
85
  unf (0.1.4)
83
86
  unf_ext
84
87
  unf_ext (0.0.7.6)
85
- webmock (3.6.2)
88
+ webmock (3.7.6)
86
89
  addressable (>= 2.3.6)
87
90
  crack (>= 0.3.2)
88
91
  hashdiff (>= 0.4.0, < 2.0.0)
@@ -92,11 +95,11 @@ PLATFORMS
92
95
  ruby
93
96
 
94
97
  DEPENDENCIES
95
- pry (~> 0.12.2)
96
- rake (~> 10.0)
97
- rspec (~> 3.0)
98
+ pry (>= 0.12.2)
99
+ rake (>= 10.0)
100
+ rspec (>= 3.0)
98
101
  web_stat!
99
- webmock (~> 3.6.0)
102
+ webmock (>= 3.6.0)
100
103
 
101
104
  BUNDLED WITH
102
105
  2.1.2
data/README.md CHANGED
@@ -44,8 +44,8 @@ And then execute:
44
44
 
45
45
  ### spec
46
46
 
47
- $ rake spec
47
+ $ bundle exec rake spec
48
48
 
49
49
  Test a file
50
50
 
51
- $ bundle exec rspec spec/web_stat/fetch_spec.rb
51
+ $ bundle exec rspec spec/web_stat/fetch_spec.rb
data/lib/web_stat/fetch.rb CHANGED
@@ -1,10 +1,3 @@
1
- require 'uri'
2
- require 'digest'
3
- require 'sanitize'
4
- require 'nokogiri'
5
- require 'open-uri'
6
- require 'ruby-readability'
7
- require 'final_redirect_url'
8
1
  module WebStat
9
2
  class Fetch
10
3
  attr_accessor :url, :html, :nokogiri
@@ -60,7 +53,7 @@ module WebStat
60
53
  def save_local_path(url)
61
54
  return nil if url.nil?
62
55
  tmp_file = "/tmp/#{Digest::SHA1.hexdigest(url)}"
63
- open(original_url(url)) do |remote_file|
56
+ URI.open(original_url(url)) do |remote_file|
64
57
  File.open(tmp_file, "w+b") do |_file|
65
58
  _file.puts(remote_file.read)
66
59
  end
@@ -79,11 +72,13 @@ module WebStat
79
72
 
80
73
  # Get the informations of @url
81
74
  def stat
75
+ content.gsub!(/(\n|\t|\s|　)/, "")
82
76
  tag = WebStat::Tag.new(content)
83
77
  {
84
78
  title: title,
85
79
  site_name: site_name,
86
80
  content: content,
81
+ language_code: CLD.detect_language(content)[:code],
87
82
  url: @url,
88
83
  eyecatch_image_path: save_local_path(eyecatch_image_path),
89
84
  tags: tag.nouns
data/lib/web_stat/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module WebStat
2
- VERSION = "0.1.12"
2
+ VERSION = "0.2.0"
3
3
  end
data/lib/web_stat.rb CHANGED
@@ -1,4 +1,14 @@
1
1
  require "bundler"
2
+
3
+ require 'uri'
4
+ require 'digest'
5
+ require 'sanitize'
6
+ require 'nokogiri'
7
+ require 'open-uri'
8
+ require 'ruby-readability'
9
+ require 'final_redirect_url'
10
+ require 'cld'
11
+
2
12
  require "web_stat/categorize"
3
13
  require "web_stat/configure"
4
14
  require "web_stat/errors"
data/spec/web_stat/fetch_spec.rb CHANGED
@@ -49,6 +49,14 @@ RSpec.describe WebStat::Fetch do
49
49
  end
50
50
  end
51
51
 
52
+ it "Get language_iso by #{fetch[:class].to_s}" do
53
+ fetch[:fixture].each do |fixture|
54
+ web_stat = fetch[:class].new(fixture)
55
+ web_stat.url = "https://newsdict.blog"
56
+ expect(web_stat.stat[:language_code]).to eq("ja")
57
+ end
58
+ end
59
+
52
60
  it "Get local path of eyecatch image by #{fetch[:class].to_s}" do
53
61
  fetch[:fixture].each do |fixture|
54
62
  web_stat = fetch[:class].new(fixture)
data/web_stat.gemspec CHANGED
@@ -22,14 +22,15 @@ Gem::Specification.new do |spec|
22
22
 
23
23
  spec.add_runtime_dependency "bundler", ">= 2.0.2"
24
24
  spec.add_runtime_dependency "nokogiri", ">= 1.10.4"
25
- spec.add_runtime_dependency "mechanize", "~> 2.7"
26
- spec.add_runtime_dependency "ruby-readability", "~> 0.7"
27
- spec.add_runtime_dependency "final_redirect_url", "~> 0.1.0"
28
- spec.add_runtime_dependency "natto", "~> 1.1.2"
29
- spec.add_runtime_dependency "sanitize", "~> 5.0.0"
30
-
31
- spec.add_development_dependency "rake", "~> 10.0"
32
- spec.add_development_dependency "rspec", "~> 3.0"
33
- spec.add_development_dependency "pry", "~> 0.12.2"
34
- spec.add_development_dependency "webmock", "~> 3.6.0"
25
+ spec.add_runtime_dependency "mechanize", ">= 2.7"
26
+ spec.add_runtime_dependency "ruby-readability", ">= 0.7"
27
+ spec.add_runtime_dependency "final_redirect_url", ">= 0.1.0"
28
+ spec.add_runtime_dependency "natto", ">= 1.1.2"
29
+ spec.add_runtime_dependency "sanitize", ">= 5.0.0"
30
+ spec.add_runtime_dependency "cld", ">= 0.8.0"
31
+
32
+ spec.add_development_dependency "rake", ">= 10.0"
33
+ spec.add_development_dependency "rspec", ">= 3.0"
34
+ spec.add_development_dependency "pry", ">= 0.12.2"
35
+ spec.add_development_dependency "webmock", ">= 3.6.0"
35
36
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: web_stat
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.12
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - yusuke abe
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-12-28 00:00:00.000000000 Z
11
+ date: 2020-01-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -42,126 +42,140 @@ dependencies:
42
42
  name: mechanize
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - "~>"
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: '2.7'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - "~>"
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '2.7'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: ruby-readability
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - "~>"
59
+ - - ">="
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0.7'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - "~>"
66
+ - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0.7'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: final_redirect_url
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - "~>"
73
+ - - ">="
74
74
  - !ruby/object:Gem::Version
75
75
  version: 0.1.0
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - "~>"
80
+ - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: 0.1.0
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: natto
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - "~>"
87
+ - - ">="
88
88
  - !ruby/object:Gem::Version
89
89
  version: 1.1.2
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - "~>"
94
+ - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: 1.1.2
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: sanitize
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - "~>"
101
+ - - ">="
102
102
  - !ruby/object:Gem::Version
103
103
  version: 5.0.0
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - "~>"
108
+ - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: 5.0.0
111
+ - !ruby/object:Gem::Dependency
112
+ name: cld
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: 0.8.0
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: 0.8.0
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: rake
113
127
  requirement: !ruby/object:Gem::Requirement
114
128
  requirements:
115
- - - "~>"
129
+ - - ">="
116
130
  - !ruby/object:Gem::Version
117
131
  version: '10.0'
118
132
  type: :development
119
133
  prerelease: false
120
134
  version_requirements: !ruby/object:Gem::Requirement
121
135
  requirements:
122
- - - "~>"
136
+ - - ">="
123
137
  - !ruby/object:Gem::Version
124
138
  version: '10.0'
125
139
  - !ruby/object:Gem::Dependency
126
140
  name: rspec
127
141
  requirement: !ruby/object:Gem::Requirement
128
142
  requirements:
129
- - - "~>"
143
+ - - ">="
130
144
  - !ruby/object:Gem::Version
131
145
  version: '3.0'
132
146
  type: :development
133
147
  prerelease: false
134
148
  version_requirements: !ruby/object:Gem::Requirement
135
149
  requirements:
136
- - - "~>"
150
+ - - ">="
137
151
  - !ruby/object:Gem::Version
138
152
  version: '3.0'
139
153
  - !ruby/object:Gem::Dependency
140
154
  name: pry
141
155
  requirement: !ruby/object:Gem::Requirement
142
156
  requirements:
143
- - - "~>"
157
+ - - ">="
144
158
  - !ruby/object:Gem::Version
145
159
  version: 0.12.2
146
160
  type: :development
147
161
  prerelease: false
148
162
  version_requirements: !ruby/object:Gem::Requirement
149
163
  requirements:
150
- - - "~>"
164
+ - - ">="
151
165
  - !ruby/object:Gem::Version
152
166
  version: 0.12.2
153
167
  - !ruby/object:Gem::Dependency
154
168
  name: webmock
155
169
  requirement: !ruby/object:Gem::Requirement
156
170
  requirements:
157
- - - "~>"
171
+ - - ">="
158
172
  - !ruby/object:Gem::Version
159
173
  version: 3.6.0
160
174
  type: :development
161
175
  prerelease: false
162
176
  version_requirements: !ruby/object:Gem::Requirement
163
177
  requirements:
164
- - - "~>"
178
+ - - ">="
165
179
  - !ruby/object:Gem::Version
166
180
  version: 3.6.0
167
181
  description: Fetch the web pages and stat.
@@ -221,7 +235,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
221
235
  - !ruby/object:Gem::Version
222
236
  version: '0'
223
237
  requirements: []
224
- rubygems_version: 3.0.6
238
+ rubygems_version: 3.1.2
225
239
  signing_key:
226
240
  specification_version: 4
227
241
  summary: Get the status of the web pages.