web_stat 0.3.3 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d27f9de72e744c0cff9c1876952b23d7dd86a3d8a8331ee95293ff06d0ad16a9
4
- data.tar.gz: a9985bd6c7167e70bcffb0f215ce13afbdef85d34790a0a5486b72b7380b01fb
3
+ metadata.gz: bbf065f5482ee7bf68e14e41af1e48b855f4219b9f44a7e16440834c92f404d5
4
+ data.tar.gz: a16fc8b8a66cb534bb8ce61c6da28212faff44f2df6324772261f9b6847cfc7f
5
5
  SHA512:
6
- metadata.gz: 2f2a7d199c6cd737bb6b6facd97fd94324898ccb14554e2497d44d567f19c06deb127da3db5be968737d8dffde25b0b498845ebb60661408810891a3c4eea142
7
- data.tar.gz: 0c1b425aa39397ac13e68009b20c0015d7939ce4af1faabbb2716b22e2cb2c4776aaecd59ab6328113a270c3a4d545f9d2d2d1472373e3f53d083cbcd3833186
6
+ metadata.gz: 83b8b7d71a1047ec2614a9bfbc3c6305806cf7a616ba6b2568382b72ef91f5b0947e7aefb3d926b33c4970d291d2a34c99a86ba0f0e320a3e7fde9dd2b5bc154
7
+ data.tar.gz: aa4af6063d7dd81c7c78c0100ff5354d7c54f28d7b2fac7393656cb5bde932b7260c95d1fe08fc560784e85ead7bbf49134c4d56cb108cef522cef753e477032
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- web_stat (0.3.3)
4
+ web_stat (0.3.7)
5
5
  bundler (>= 2.0.2)
6
6
  cld (>= 0.8.0)
7
7
  mechanize (>= 2.7)
@@ -83,7 +83,7 @@ GEM
83
83
  nokogiri (>= 1.6.0)
84
84
  rubyzip (2.3.0)
85
85
  safe_yaml (1.0.5)
86
- sanitize (5.2.0)
86
+ sanitize (5.2.1)
87
87
  crass (~> 1.0.2)
88
88
  nokogiri (>= 1.8.0)
89
89
  nokogumbo (~> 2.0)
@@ -47,6 +47,9 @@ module WebStat
47
47
  break
48
48
  end
49
49
  end
50
+ if path.nil? || path.empty? || @nokogiri.at('body').xpath('//img').first
51
+ path = @nokogiri.at('body').xpath('//img').first.attr('src')
52
+ end
50
53
  if ! path.nil? && path.match(/^\//)
51
54
  "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}#{path}"
52
55
  else
@@ -57,7 +60,7 @@ module WebStat
57
60
  # Get local path to save url
58
61
  # @param [String] url
59
62
  def save_local_path(url)
60
- return nil if url.nil?
63
+ return nil if url.nil? || ! url.match(%{^http})
61
64
  tmp_file = "/tmp/#{Digest::SHA1.hexdigest(url)}"
62
65
  agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
63
66
  image = agent.get(url)
@@ -75,17 +78,25 @@ module WebStat
75
78
  # @param [String] url
76
79
  # @param [String] body
77
80
  def get_url(url)
78
- agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
81
+ mech = Mechanize.new { |_mech| _mech.user_agent = WebStat::Configure.get["user_agent"] }
79
82
  # Enable to read Robots.txt
80
- agent.robots = true
83
+ mech.robots = true
81
84
  begin
82
- document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
83
- if document.class == Mechanize::File
84
- body = document.body
85
+ if mech.agent.robots_disallowed?(url)
86
+ raise Mechanize::RobotsDisallowedError.new(url)
87
+ end
88
+ if WebStat::Configure.get["use_chromedirver"]
89
+ document = WebStat::WebDriverHelper.get_last_url(url)
90
+ @status = 200
85
91
  else
86
- body = document.body.encode('UTF-8', document.encoding)
92
+ document = mech.get(url, [], nil, { 'Accept-Language' => 'ja'})
93
+ if document.class == Mechanize::File
94
+ body = document.body
95
+ else
96
+ body = document.body.encode('UTF-8', document.encoding)
97
+ end
98
+ @status = document.code
87
99
  end
88
- @status = document.code
89
100
  rescue Mechanize::ResponseCodeError => e
90
101
  body = e.page.body
91
102
  @status = e.page.code
@@ -96,7 +107,7 @@ module WebStat
96
107
  # Get the informations of @url
97
108
  # @param [Hash] Specify a dictionary for each language code. example ) {"ja": /***/**.dic, "other": /***/***.dic}
98
109
  def stat(userdics: nil)
99
- clean_content = content.scrub('').gsub(/[\n\t\r ]/, "").gsub(/\s{2,}/, "\s")
110
+ clean_content = content.scrub('').gsub(/[\n\t\r ]/, "").gsub(/\s{2,}/, "\s").gsub(URI.regexp, "")
100
111
  language_code = CLD.detect_language(clean_content)[:code]
101
112
  if userdics && userdics.has_key?(language_code) && File.exists?(userdics[language_code])
102
113
  tag = WebStat::Tag.new("#{title} #{content}", userdic: userdics[language_code])
@@ -1,3 +1,3 @@
1
1
  module WebStat
2
- VERSION = "0.3.3"
2
+ VERSION = "0.3.8"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: web_stat
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.3
4
+ version: 0.3.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - yusuke abe
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-06-13 00:00:00.000000000 Z
11
+ date: 2020-06-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler