web_stat 0.1.12 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -1
  3. data/Gemfile.lock +34 -31
  4. data/README.md +2 -2
  5. data/lib/web_stat/fetch.rb +3 -8
  6. data/lib/web_stat/version.rb +1 -1
  7. data/lib/web_stat.rb +10 -0
  8. data/spec/web_stat/fetch_spec.rb +8 -0
  9. data/web_stat.gemspec +11 -10
  10. metadata +35 -21
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: daa2f034827920f7dbdfac48830325f67cf9e95a9b724b12e643342f9b2aa170
4
- data.tar.gz: 4a036530d8181f787522f884f5059eb321acc1ed142e6062ca3e442773d05c41
3
+ metadata.gz: 89fb8389fb61745427075c700e9ba094ed2fbfe78f22e6f9364d836472ba8334
4
+ data.tar.gz: 2f07517f36832489f2bd72f259d93d1ee60bab29247e4fd88990a29ee1d3f912
5
5
  SHA512:
6
- metadata.gz: 95cfc5fa7d23d6ed5bfa1f4375143acfd814a56623df73687b37cd18545fad1bbb0aa0112a231f3164459dd8dc64980c70680c15aee47a5053e3bdabb1e4c2aa
7
- data.tar.gz: 5024406dbe28de59e72b8d3cc0ccac411cc8ebcd5b74d8f40b2c861b802f3de0eaab6b4a29f3c67ed98498f159346570fc3c395cd7b8686c92c374e6d4eb4d67
6
+ metadata.gz: 56e2f7e8ed3ad8e44100f5ac767206cbdfda1e8cac431813a005998823acbca4060d441ddf723dd17b46fbfbbd37fd9700f3e459ed99fe3e73eaadb42ca2c6d0
7
+ data.tar.gz: 64923ad006daa2db6cd196d2c6f3256b011673d48fc74eb315556aebea950550c7c1dc6c123c97b4fd55ef4ae2a1d3714ce93b62d34b8f61d16346dc9a1f870e
data/.gitignore CHANGED
@@ -1,4 +1,4 @@
1
- /.bundle/
1
+ /vendor/
2
2
  /.yardoc
3
3
  /_yardoc/
4
4
  /coverage/
data/Gemfile.lock CHANGED
@@ -1,29 +1,32 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- web_stat (0.1.12)
4
+ web_stat (0.2.0)
5
5
  bundler (>= 2.0.2)
6
- final_redirect_url (~> 0.1.0)
7
- mechanize (~> 2.7)
8
- natto (~> 1.1.2)
6
+ cld (>= 0.8.0)
7
+ final_redirect_url (>= 0.1.0)
8
+ mechanize (>= 2.7)
9
+ natto (>= 1.1.2)
9
10
  nokogiri (>= 1.10.4)
10
- ruby-readability (~> 0.7)
11
- sanitize (~> 5.0.0)
11
+ ruby-readability (>= 0.7)
12
+ sanitize (>= 5.0.0)
12
13
 
13
14
  GEM
14
15
  remote: https://rubygems.org/
15
16
  specs:
16
17
  addressable (2.7.0)
17
18
  public_suffix (>= 2.0.2, < 5.0)
19
+ cld (0.8.0)
20
+ ffi
18
21
  coderay (1.1.2)
19
22
  connection_pool (2.2.2)
20
23
  crack (0.4.3)
21
24
  safe_yaml (~> 1.0.0)
22
- crass (1.0.4)
25
+ crass (1.0.5)
23
26
  diff-lcs (1.3)
24
27
  domain_name (0.5.20190701)
25
28
  unf (>= 0.0.5, < 1.0.0)
26
- ffi (1.11.1)
29
+ ffi (1.11.3)
27
30
  final_redirect_url (0.1.0)
28
31
  guess_html_encoding (0.0.11)
29
32
  hashdiff (1.0.0)
@@ -39,50 +42,50 @@ GEM
39
42
  ntlm-http (~> 0.1, >= 0.1.1)
40
43
  webrobots (>= 0.0.9, < 0.2)
41
44
  method_source (0.9.2)
42
- mime-types (3.3)
45
+ mime-types (3.3.1)
43
46
  mime-types-data (~> 3.2015)
44
- mime-types-data (3.2019.0904)
47
+ mime-types-data (3.2019.1009)
45
48
  mini_portile2 (2.4.0)
46
49
  natto (1.1.2)
47
50
  ffi (>= 1.9.0)
48
51
  net-http-digest_auth (1.4.1)
49
52
  net-http-persistent (3.1.0)
50
53
  connection_pool (~> 2.2)
51
- nokogiri (1.10.4)
54
+ nokogiri (1.10.7)
52
55
  mini_portile2 (~> 2.4.0)
53
- nokogumbo (2.0.1)
56
+ nokogumbo (2.0.2)
54
57
  nokogiri (~> 1.8, >= 1.8.4)
55
58
  ntlm-http (0.1.1)
56
59
  pry (0.12.2)
57
60
  coderay (~> 1.1.0)
58
61
  method_source (~> 0.9.0)
59
- public_suffix (4.0.1)
60
- rake (10.5.0)
61
- rspec (3.8.0)
62
- rspec-core (~> 3.8.0)
63
- rspec-expectations (~> 3.8.0)
64
- rspec-mocks (~> 3.8.0)
65
- rspec-core (3.8.2)
66
- rspec-support (~> 3.8.0)
67
- rspec-expectations (3.8.4)
62
+ public_suffix (4.0.2)
63
+ rake (13.0.1)
64
+ rspec (3.9.0)
65
+ rspec-core (~> 3.9.0)
66
+ rspec-expectations (~> 3.9.0)
67
+ rspec-mocks (~> 3.9.0)
68
+ rspec-core (3.9.1)
69
+ rspec-support (~> 3.9.1)
70
+ rspec-expectations (3.9.0)
68
71
  diff-lcs (>= 1.2.0, < 2.0)
69
- rspec-support (~> 3.8.0)
70
- rspec-mocks (3.8.1)
72
+ rspec-support (~> 3.9.0)
73
+ rspec-mocks (3.9.1)
71
74
  diff-lcs (>= 1.2.0, < 2.0)
72
- rspec-support (~> 3.8.0)
73
- rspec-support (3.8.2)
75
+ rspec-support (~> 3.9.0)
76
+ rspec-support (3.9.2)
74
77
  ruby-readability (0.7.0)
75
78
  guess_html_encoding (>= 0.0.4)
76
79
  nokogiri (>= 1.6.0)
77
80
  safe_yaml (1.0.5)
78
- sanitize (5.0.0)
81
+ sanitize (5.1.0)
79
82
  crass (~> 1.0.2)
80
83
  nokogiri (>= 1.8.0)
81
84
  nokogumbo (~> 2.0)
82
85
  unf (0.1.4)
83
86
  unf_ext
84
87
  unf_ext (0.0.7.6)
85
- webmock (3.6.2)
88
+ webmock (3.7.6)
86
89
  addressable (>= 2.3.6)
87
90
  crack (>= 0.3.2)
88
91
  hashdiff (>= 0.4.0, < 2.0.0)
@@ -92,11 +95,11 @@ PLATFORMS
92
95
  ruby
93
96
 
94
97
  DEPENDENCIES
95
- pry (~> 0.12.2)
96
- rake (~> 10.0)
97
- rspec (~> 3.0)
98
+ pry (>= 0.12.2)
99
+ rake (>= 10.0)
100
+ rspec (>= 3.0)
98
101
  web_stat!
99
- webmock (~> 3.6.0)
102
+ webmock (>= 3.6.0)
100
103
 
101
104
  BUNDLED WITH
102
105
  2.1.2
data/README.md CHANGED
@@ -44,8 +44,8 @@ And then execute:
44
44
 
45
45
  ### spec
46
46
 
47
- $ rake spec
47
+ $ bundle exec rake spec
48
48
 
49
49
  Test a file
50
50
 
51
- $ bundle exec rspec spec/web_stat/fetch_spec.rb
51
+ $ bundle exec rspec spec/web_stat/fetch_spec.rb
data/lib/web_stat/fetch.rb CHANGED
@@ -1,10 +1,3 @@
1
- require 'uri'
2
- require 'digest'
3
- require 'sanitize'
4
- require 'nokogiri'
5
- require 'open-uri'
6
- require 'ruby-readability'
7
- require 'final_redirect_url'
8
1
  module WebStat
9
2
  class Fetch
10
3
  attr_accessor :url, :html, :nokogiri
@@ -60,7 +53,7 @@ module WebStat
60
53
  def save_local_path(url)
61
54
  return nil if url.nil?
62
55
  tmp_file = "/tmp/#{Digest::SHA1.hexdigest(url)}"
63
- open(original_url(url)) do |remote_file|
56
+ URI.open(original_url(url)) do |remote_file|
64
57
  File.open(tmp_file, "w+b") do |_file|
65
58
  _file.puts(remote_file.read)
66
59
  end
@@ -79,11 +72,13 @@ module WebStat
79
72
 
80
73
  # Get the informations of @url
81
74
  def stat
75
+ content.gsub!(/(\n|\t|\s|　)/, "")
82
76
  tag = WebStat::Tag.new(content)
83
77
  {
84
78
  title: title,
85
79
  site_name: site_name,
86
80
  content: content,
81
+ language_code: CLD.detect_language(content)[:code],
87
82
  url: @url,
88
83
  eyecatch_image_path: save_local_path(eyecatch_image_path),
89
84
  tags: tag.nouns
data/lib/web_stat/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module WebStat
2
- VERSION = "0.1.12"
2
+ VERSION = "0.2.0"
3
3
  end
data/lib/web_stat.rb CHANGED
@@ -1,4 +1,14 @@
1
1
  require "bundler"
2
+
3
+ require 'uri'
4
+ require 'digest'
5
+ require 'sanitize'
6
+ require 'nokogiri'
7
+ require 'open-uri'
8
+ require 'ruby-readability'
9
+ require 'final_redirect_url'
10
+ require 'cld'
11
+
2
12
  require "web_stat/categorize"
3
13
  require "web_stat/configure"
4
14
  require "web_stat/errors"
data/spec/web_stat/fetch_spec.rb CHANGED
@@ -49,6 +49,14 @@ RSpec.describe WebStat::Fetch do
49
49
  end
50
50
  end
51
51
 
52
+ it "Get language_iso by #{fetch[:class].to_s}" do
53
+ fetch[:fixture].each do |fixture|
54
+ web_stat = fetch[:class].new(fixture)
55
+ web_stat.url = "https://newsdict.blog"
56
+ expect(web_stat.stat[:language_code]).to eq("ja")
57
+ end
58
+ end
59
+
52
60
  it "Get local path of eyecatch image by #{fetch[:class].to_s}" do
53
61
  fetch[:fixture].each do |fixture|
54
62
  web_stat = fetch[:class].new(fixture)
data/web_stat.gemspec CHANGED
@@ -22,14 +22,15 @@ Gem::Specification.new do |spec|
22
22
 
23
23
  spec.add_runtime_dependency "bundler", ">= 2.0.2"
24
24
  spec.add_runtime_dependency "nokogiri", ">= 1.10.4"
25
- spec.add_runtime_dependency "mechanize", "~> 2.7"
26
- spec.add_runtime_dependency "ruby-readability", "~> 0.7"
27
- spec.add_runtime_dependency "final_redirect_url", "~> 0.1.0"
28
- spec.add_runtime_dependency "natto", "~> 1.1.2"
29
- spec.add_runtime_dependency "sanitize", "~> 5.0.0"
30
-
31
- spec.add_development_dependency "rake", "~> 10.0"
32
- spec.add_development_dependency "rspec", "~> 3.0"
33
- spec.add_development_dependency "pry", "~> 0.12.2"
34
- spec.add_development_dependency "webmock", "~> 3.6.0"
25
+ spec.add_runtime_dependency "mechanize", ">= 2.7"
26
+ spec.add_runtime_dependency "ruby-readability", ">= 0.7"
27
+ spec.add_runtime_dependency "final_redirect_url", ">= 0.1.0"
28
+ spec.add_runtime_dependency "natto", ">= 1.1.2"
29
+ spec.add_runtime_dependency "sanitize", ">= 5.0.0"
30
+ spec.add_runtime_dependency "cld", ">= 0.8.0"
31
+
32
+ spec.add_development_dependency "rake", ">= 10.0"
33
+ spec.add_development_dependency "rspec", ">= 3.0"
34
+ spec.add_development_dependency "pry", ">= 0.12.2"
35
+ spec.add_development_dependency "webmock", ">= 3.6.0"
35
36
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: web_stat
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.12
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - yusuke abe
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-12-28 00:00:00.000000000 Z
11
+ date: 2020-01-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -42,126 +42,140 @@ dependencies:
42
42
  name: mechanize
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - "~>"
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: '2.7'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - "~>"
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '2.7'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: ruby-readability
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - "~>"
59
+ - - ">="
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0.7'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - "~>"
66
+ - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0.7'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: final_redirect_url
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - "~>"
73
+ - - ">="
74
74
  - !ruby/object:Gem::Version
75
75
  version: 0.1.0
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - "~>"
80
+ - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: 0.1.0
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: natto
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - "~>"
87
+ - - ">="
88
88
  - !ruby/object:Gem::Version
89
89
  version: 1.1.2
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - "~>"
94
+ - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: 1.1.2
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: sanitize
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - "~>"
101
+ - - ">="
102
102
  - !ruby/object:Gem::Version
103
103
  version: 5.0.0
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - "~>"
108
+ - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: 5.0.0
111
+ - !ruby/object:Gem::Dependency
112
+ name: cld
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: 0.8.0
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: 0.8.0
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: rake
113
127
  requirement: !ruby/object:Gem::Requirement
114
128
  requirements:
115
- - - "~>"
129
+ - - ">="
116
130
  - !ruby/object:Gem::Version
117
131
  version: '10.0'
118
132
  type: :development
119
133
  prerelease: false
120
134
  version_requirements: !ruby/object:Gem::Requirement
121
135
  requirements:
122
- - - "~>"
136
+ - - ">="
123
137
  - !ruby/object:Gem::Version
124
138
  version: '10.0'
125
139
  - !ruby/object:Gem::Dependency
126
140
  name: rspec
127
141
  requirement: !ruby/object:Gem::Requirement
128
142
  requirements:
129
- - - "~>"
143
+ - - ">="
130
144
  - !ruby/object:Gem::Version
131
145
  version: '3.0'
132
146
  type: :development
133
147
  prerelease: false
134
148
  version_requirements: !ruby/object:Gem::Requirement
135
149
  requirements:
136
- - - "~>"
150
+ - - ">="
137
151
  - !ruby/object:Gem::Version
138
152
  version: '3.0'
139
153
  - !ruby/object:Gem::Dependency
140
154
  name: pry
141
155
  requirement: !ruby/object:Gem::Requirement
142
156
  requirements:
143
- - - "~>"
157
+ - - ">="
144
158
  - !ruby/object:Gem::Version
145
159
  version: 0.12.2
146
160
  type: :development
147
161
  prerelease: false
148
162
  version_requirements: !ruby/object:Gem::Requirement
149
163
  requirements:
150
- - - "~>"
164
+ - - ">="
151
165
  - !ruby/object:Gem::Version
152
166
  version: 0.12.2
153
167
  - !ruby/object:Gem::Dependency
154
168
  name: webmock
155
169
  requirement: !ruby/object:Gem::Requirement
156
170
  requirements:
157
- - - "~>"
171
+ - - ">="
158
172
  - !ruby/object:Gem::Version
159
173
  version: 3.6.0
160
174
  type: :development
161
175
  prerelease: false
162
176
  version_requirements: !ruby/object:Gem::Requirement
163
177
  requirements:
164
- - - "~>"
178
+ - - ">="
165
179
  - !ruby/object:Gem::Version
166
180
  version: 3.6.0
167
181
  description: Fetch the web pages and stat.
@@ -221,7 +235,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
221
235
  - !ruby/object:Gem::Version
222
236
  version: '0'
223
237
  requirements: []
224
- rubygems_version: 3.0.6
238
+ rubygems_version: 3.1.2
225
239
  signing_key:
226
240
  specification_version: 4
227
241
  summary: Get the status of the web pages.