web_stat 0.2.9 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6851adcf7a19adb66dbfc7f241cd2b659878ef954836e5656d42d3540a117fd8
-  data.tar.gz: '013188fbf26bd0ccce741afc7500060e40a33650e89fd7c6a1dc4d2ad216fcf3'
+  metadata.gz: eff1d2b80ff6bf4b61f82a100319a6b37ee13cb8a2ad96a11fd336c8a7164398
+  data.tar.gz: f4bf36491dbe6ae32b1cbf718ceb1fe7e91adce60848f4296d6c1319ea5c4eea
 SHA512:
-  metadata.gz: a1bf10c3439fbe6e93b1368970725838f6b649854b5eac8e03bfd3e3b1e16f2e99b5a5bfe324980f51073e75e74a890b6a9e8d49735f28ca638d48eb36c0696f
-  data.tar.gz: 5fc04389537b37d597200c50de8691010e46308063804a29bbbfd1ce2406c592f9d11678752e748606c45061b3d55e2b694ad590af1f486cfc166f646b6ec6c9
+  metadata.gz: f38fe9d4f0ad495107c5ac42d2d240b42e810ae4f7ad0db5738b705f7e03b24846875cbc2139a334ebfc7d33ff1df953dec768f69f13a0932cd01a2c6f221753
+  data.tar.gz: 5f30259af5a84e2eb43645766aac49d151367a82c9e5643a0ddf3a5f15dbcbba59884def134cfaa19eaca1a932e77c28d65bfd389cecdd0262812cad808ff5e3

data/.gitignore

@@ -6,6 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/.bundle
 
 # rspec failure tracking
 .rspec_status

data/.ruby-version

@@ -1 +1 @@
-2.7.0
+2.7.1

data/Dockerfile

@@ -0,0 +1,25 @@
+# Define base image, you can use --build-arg
+ARG base_image="newsdict/rails:ubuntu20.10_nvmv0.35.2_nodev14.3.0_rubyv2.7.1_sasscv2.3.0_ffiv1.13.1_chromedriver"
+FROM $base_image
+
+# Set locale
+ENV LANG "C.UTF-8"
+ENV NOKOGIRI_USE_SYSTEM_LIBRARIES "YES"
+
+# Set correct environment variables.
+RUN mkdir -p /var/www/docker
+WORKDIR /var/www/docker
+
+# Set up application
+COPY . .
+
+# Init gems
+RUN echo "gem: --no-rdoc --no-ri" > ~/.gemrc
+RUN . /etc/profile.d/rvm.sh && \
+  bundle config --global with 'development test' && \
+  bundle config --global system true && \
+  bundle config --global jobs 10 && \
+  bundle config --global build.nokogiri --use-system-libraries && \
+  bundle install
+
+CMD ["bash"]

data/Gemfile

@@ -1,6 +1,4 @@
 source "https://rubygems.org"
 
-gem "final_redirect_url", :git => "git@github.com:yubele/final_redirect_url"
-
 # Specify your gem's dependencies in web_stat.gemspec
 gemspec

data/Gemfile.lock

@@ -1,13 +1,7 @@
-GIT
-  remote: git@github.com:yubele/final_redirect_url
-  revision: 45df878ec9495ebbfa06dc0a60cc5043c2519e16
-  specs:
-    final_redirect_url (0.1.1)
-
 PATH
   remote: .
   specs:
-    web_stat (0.2.9)
+    web_stat (0.3.2)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)
@@ -15,25 +9,28 @@ PATH
       nokogiri (>= 1.10.4)
       ruby-readability (>= 0.7)
       sanitize (>= 5.0.0)
+      selenium-webdriver (= 3.142.7)
 
 GEM
   remote: https://rubygems.org/
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
+    byebug (11.1.3)
+    childprocess (3.0.0)
     cld (0.8.0)
       ffi
-    coderay (1.1.2)
-    connection_pool (2.2.2)
+    coderay (1.1.3)
+    connection_pool (2.2.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    crass (1.0.5)
+    crass (1.0.6)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.11.3)
+    ffi (1.13.1)
     guess_html_encoding (0.0.11)
-    hashdiff (1.0.0)
+    hashdiff (1.0.1)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     mechanize (2.7.6)
@@ -45,51 +42,58 @@ GEM
       nokogiri (~> 1.6)
       ntlm-http (~> 0.1, >= 0.1.1)
       webrobots (>= 0.0.9, < 0.2)
-    method_source (0.9.2)
+    method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2020.0512)
     mini_portile2 (2.4.0)
-    natto (1.1.2)
+    natto (1.2.0)
       ffi (>= 1.9.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (3.1.0)
+    net-http-persistent (4.0.0)
       connection_pool (~> 2.2)
-    nokogiri (1.10.7)
+    nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
     nokogumbo (2.0.2)
       nokogiri (~> 1.8, >= 1.8.4)
     ntlm-http (0.1.1)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    public_suffix (4.0.2)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    public_suffix (4.0.5)
     rake (13.0.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.1)
-      rspec-support (~> 3.9.1)
-    rspec-expectations (3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-support (3.9.2)
+    rspec-support (3.9.3)
     ruby-readability (0.7.0)
       guess_html_encoding (>= 0.0.4)
       nokogiri (>= 1.6.0)
+    rubyzip (2.3.0)
     safe_yaml (1.0.5)
-    sanitize (5.1.0)
+    sanitize (5.2.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)
+    selenium-webdriver (3.142.7)
+      childprocess (>= 0.5, < 4.0)
+      rubyzip (>= 1.2.2)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.6)
-    webmock (3.7.6)
+    unf_ext (0.0.7.7)
+    webmock (3.8.3)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
@@ -99,12 +103,12 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  final_redirect_url!
-  pry (>= 0.12.2)
+  pry (>= 0.13.1)
+  pry-byebug (= 3.9.0)
   rake (>= 10.0)
   rspec (>= 3.0)
   web_stat!
   webmock (>= 3.6.0)
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/README.md

@@ -25,6 +25,10 @@ Fetch the web pages and stat.
 - "rake", "~> 10.0"
 - "rspec", "~> 3.0"
 
+### Install mecab
+
+    $ sudo apt install mecab-ipadic-utf8 libmecab
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3.8"
+networks:
+  app-tier:
+    driver: bridge
+services:
+  web_stat:
+    tty: true
+    stdin_open: true
+    container_name: web_stat
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ./:/var/www/docker:cached
+    working_dir: /var/www/docker
+    command: bash
+    networks:
+      - app-tier

data/docker/exec

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Check to support tty.
+if [ "$(tty>/dev/null;echo $?)" != "0" ];then
+    DOCKERCOMPOSE_EXEC="docker-compose exec -T"
+else
+    DOCKERCOMPOSE_EXEC="docker-compose exec"
+fi
+$DOCKERCOMPOSE_EXEC web_stat /bin/bash -c ". /etc/profile.d/rvm.sh && . /root/.nvm/nvm.sh && $*"

data/docker/start

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+if [ "$1" = "rm" ];then
+  shift
+  docker-compose rm -f
+fi
+docker-compose stop
+if [ "$1" = "attach" ]; then
+  docker-compose up -d
+  docker attach $2
+else
+  docker-compose up $@
+fi

data/lib/helpers/web_drive_helper.rb

@@ -0,0 +1,30 @@
+module WebStat
+  class WebDriverHelper
+    class << self
+      # Get last url
+      # @param [String] url
+      # @param [Integer] delay
+      def get_last_url(url, delay=nil)
+        Selenium::WebDriver.logger.output = File.join("/tmp", "selenium.log")
+        Selenium::WebDriver.logger.level = :info
+        options = Selenium::WebDriver::Chrome::Options.new(args: [
+          'headless',
+          'no-sandbox',
+          'disable-gpu',
+          'start-maximized',
+          'window-size=1920,1080'
+          ])
+        driver = Selenium::WebDriver.for(:chrome, options: options)
+        driver.manage.timeouts.implicit_wait = 10
+        Selenium::WebDriver::Wait.new(timeout: 10)
+        driver.get(url)
+        if delay.is_a?(Integer)
+          sleep delay
+        end
+        last_url = driver.current_url
+        driver.quit
+        last_url
+      end
+    end
+  end
+end

data/lib/web_stat.rb

@@ -1,14 +1,18 @@
 require "bundler"
 
+require 'cld'
 require 'uri'
 require 'digest'
+require 'logger'
 require 'sanitize'
 require 'nokogiri'
 require 'open-uri'
+require 'net/http'
 require 'ruby-readability'
-require 'final_redirect_url'
-require 'cld'
+require 'selenium-webdriver'
 
+require "helpers/web_drive_helper"
+require "web_stat/final_redirect_url"
 require "web_stat/categorize"
 require "web_stat/configure"
 require "web_stat/errors"

data/lib/web_stat/config/web_stat.yml

@@ -11,4 +11,5 @@ eyecatch_image_xpaths:
   - '//img[@class="attachment-post-thumbnail"]/@src'
   - '//div[@id="content"]//img/@src'
   - '//img/@src'
-userdic: ""
+userdic: ""
+use_chromedirver: true

data/lib/web_stat/fetch.rb

@@ -1,6 +1,6 @@
 module WebStat
   class Fetch
-    attr_accessor :url, :html, :nokogiri, :userdic
+    attr_accessor :url, :html, :nokogiri, :userdic, :status
 
     # Get title
     # @return [String] title
@@ -13,9 +13,12 @@ module WebStat
       rescue
         title = @nokogiri.title
       end
-      title.strip
+      if title.nil?
+        "No Title"
+      else
+        title.strip
+      end
     end
-    
     # Get name of domain 
     def site_name
       begin
@@ -23,9 +26,12 @@ module WebStat
       rescue
         site_name = @nokogiri.title
       end
-      site_name.strip
+      if site_name.nil?
+        "No Sitename"
+      else
+        site_name.strip
+      end
     end
-      []
     # Get main section
     def content
       Sanitize.clean(Readability::Document.new(@nokogiri.at('body')).content)
@@ -67,16 +73,24 @@ module WebStat
     
     # Get url
     # @param [String] url
+    # @param [String] body
     def get_url(url)
       agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
       # Enable to read Robots.txt
       agent.robots = true
-      document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
-      if document.class == Mechanize::File
-        document.body
-      else
-        document.body.encode('UTF-8', document.encoding)
+      begin
+        document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
+        if document.class == Mechanize::File
+          body = document.body
+        else
+          body = document.body.encode('UTF-8', document.encoding)
+        end
+        @status = document.code
+      rescue Mechanize::ResponseCodeError => e
+        body = e.page.body
+        @status = e.page.code
       end
+      body
     end
     
     # Get the informations of @url
@@ -96,6 +110,7 @@ module WebStat
         site_name: site_name,
         content: clean_content,
         language_code: language_code,
+        status: @status,
         url: @url,
         eyecatch_image_path: save_local_path(eyecatch_image_path),
         tags: tag.nouns
@@ -107,8 +122,9 @@ module WebStat
     # Get original url
     # @param [String] url
     def original_url(url)
-      if url.match(/^http/)
-        FinalRedirectUrl.final_redirect_url(url)
+      last_url = WebStat::FinalRedirectUrl.final_redirect_url(url)
+      unless last_url.nil? || last_url.scrub('').empty?
+        last_url
       else
         url
       end

data/lib/web_stat/final_redirect_url.rb

@@ -0,0 +1,54 @@
+# ref) https://github.com/indyarocks/final_redirect_url
+# customize
+#  Changed 
+module WebStat
+  class FinalRedirectUrl
+    class << self
+      def final_redirect_url(url, options={})
+        final_url = ''
+        if is_valid_url?(url)
+          begin
+            redirect_lookup_depth = options[:depth].to_i > 0 ? options[:depth].to_i : 10
+            response_uri = get_final_redirect_url(url, redirect_lookup_depth)
+            final_url =  url_string_from_uri(response_uri)
+          rescue Exception => ex
+            # nothing
+          end
+        end
+        final_url
+      end
+    
+      private
+      def is_valid_url?(url)
+        url.to_s.match? URI::regexp(['http', 'https'])
+      end
+      def get_final_redirect_url(url, limit = 10)
+        return url if limit <= 0
+        uri = URI.parse(url)
+        response = ::Net::HTTP.get_response(uri)
+        if response.class == Net::HTTPOK
+          if WebStat::Configure.get["use_chromedirver"]
+            return URI.parse(WebStat::WebDriverHelper.get_last_url(uri))
+          else
+            return URI.parse(uri)
+          end
+        else
+          redirect_location = response['location']
+          location_uri = URI.parse(redirect_location)
+          if location_uri.host.nil?
+            redirect_location = uri.scheme + '://' + uri.host + redirect_location
+          end
+          warn "redirected to #{redirect_location}"
+          get_final_redirect_url(redirect_location, limit - 1)
+        end
+      end
+      def url_string_from_uri(uri)
+        url_str = "#{uri.scheme}://#{uri.host}#{uri.request_uri}"
+        if uri.fragment
+          url_str = url_str + "##{uri.fragment}"
+        end
+        url_str
+      end
+    end
+  end
+end

data/lib/web_stat/tag.rb

@@ -4,7 +4,7 @@ module WebStat
     attr_accessor :natto_mecab, :article
     
     def initialize(article, userdic: nil)
-        @natto_mecab = Natto::MeCab.new(userdic: userdic)
+      @natto_mecab = Natto::MeCab.new(userdic: userdic)
       @article = article
     end
     

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.2.9"
-end
+  VERSION = "0.3.2"
+end

data/spec/spec_helper.rb

@@ -1,12 +1,18 @@
 require 'rspec/expectations'
 require "bundler/setup"
 require 'pry'
+require 'pry-byebug'
 require "web_stat"
 
 require 'webmock'
 include WebMock::API
 WebMock.enable!
 
+WebMock.disable_net_connect!({
+  allow_localhost: true,
+  allow: 'chromedriver.storage.googleapis.com'
+})
+
 RSpec.configure do |config|
   # Enable flags like --only-failures and --next-failure
   config.example_status_persistence_file_path = ".rspec_status"
@@ -59,9 +65,10 @@ end
 
 # Set webmock
 WebStatTestHelper.scheme_and_files.each do |url|
+  status = [200, 404, 503].sample
   WebMock.stub_request(:get, url)
     .to_return(
-      status: 200,
+      status: status,
       body: File.new(File.join(File.dirname(__FILE__), "fixtures", "htmls", File.basename(url))),
       headers: {content_type: 'application/html; charset=utf-8'})
 end

data/spec/web_stat/fetch_spec.rb

@@ -83,6 +83,7 @@ RSpec.describe WebStat::Fetch do
       expect(web_stat[:title]).to eq "gem作成でついまずいたところ"
       expect(web_stat[:site_name]).to eq "newsdict.blog"
       expect(web_stat[:content]).not_to eq nil
+      expect(web_stat[:status]).to eq("200").or eq("404").or eq("503")
       expect(Sanitize.clean(web_stat[:content]).length).to eq web_stat[:content].length
       expect(web_stat[:eyecatch_image_path]).to be_tmp_file_or_nil
     end

data/web_stat.gemspec

@@ -27,9 +27,11 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "natto", ">= 1.1.2"
   spec.add_runtime_dependency "sanitize", ">= 5.0.0"
   spec.add_runtime_dependency "cld", ">= 0.8.0"
+  spec.add_runtime_dependency "selenium-webdriver", "= 3.142.7"
 
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", ">= 3.0"
-  spec.add_development_dependency "pry", ">= 0.12.2"
+  spec.add_development_dependency "pry", ">= 0.13.1"
   spec.add_development_dependency "webmock", ">= 3.6.0"
+  spec.add_development_dependency "pry-byebug", "3.9.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.2.9
+  version: 0.3.2
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-08 00:00:00.000000000 Z
+date: 2020-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.142.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.142.7
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +156,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.2
+        version: 0.13.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.2
+        version: 0.13.1
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +178,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.6.0
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.9.0
 description: Fetch the web pages and stat.
 email:
 - yube@newsdict.jp
@@ -177,12 +205,17 @@ files:
 - ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
+- Dockerfile
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/fetch_as_html
+- docker-compose.yml
+- docker/exec
+- docker/start
+- lib/helpers/web_drive_helper.rb
 - lib/web_stat.rb
 - lib/web_stat/categorize.rb
 - lib/web_stat/config/web_stat.yml
@@ -191,6 +224,7 @@ files:
 - lib/web_stat/fetch.rb
 - lib/web_stat/fetch/fetch_as_html.rb
 - lib/web_stat/fetch/fetch_as_web.rb
+- lib/web_stat/final_redirect_url.rb
 - lib/web_stat/tag.rb
 - lib/web_stat/tasks/install.rake
 - lib/web_stat/version.rb