web_stat 0.2.11 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bcfaeb202076ea30cae6205877d0b0ad9060eb84116b84ef7bd3580cc4349aac
-  data.tar.gz: 98e586440c8f3aed29e38a003f6d1afc96b26b2ae4c21e43d47e1dfaf6aaa16a
+  metadata.gz: 746fe3666bc5a47c315e0499fd23dc84319142a8d850b872e5118553b1e711b6
+  data.tar.gz: d2335c2d0324fc81d5b891e7cf383646f8573ef0147ac4b34672add61ecdb6c6
 SHA512:
-  metadata.gz: dfbe6264256f08550ebb42244d92bd81c976dd5dd72a6a4dabffdf6a1366a8f010e8a20527d0bbe7a8334cdb028e86ecae0c63d6cc4368741abe815f1fcb3092
-  data.tar.gz: 9098d904f26dfdfe14c87352cb47ca3f0333f5424ef26d2cd8232c088c2cf8d7dcbeb07012a9294a5c4e4949b51fe71e94d0995465400d87cf7c037fd09ba978
+  metadata.gz: d23e332cb9114aec4c7302aafb1ed4ca6fc081b3b71a21586779ec5d67de74a196508c38dd70bb8c1e5af3828ced4bdeda7f7c86ea84336fbd6fc476ba87de31
+  data.tar.gz: 9c4e7955432b59b8f66edf71d814412e6eb7720c81f1c647e9f9420f7592febbcfe349c7ceb7a2b6d371c1b7d1de7d98148f7c015d5cc1f4ef52cf08a84afcbe

data/.gitignore

@@ -6,6 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/.bundle
 
 # rspec failure tracking
 .rspec_status

data/.ruby-version

@@ -1 +1 @@
-2.7.0
+2.7.1

data/Dockerfile

@@ -0,0 +1,25 @@
+# Define base image, you can use --build-arg
+ARG base_image="newsdict/rails:ubuntu20.10_nvmv0.35.2_nodev14.3.0_rubyv2.7.1_sasscv2.3.0_ffiv1.13.1_chromedriver"
+FROM $base_image
+
+# Set locale
+ENV LANG "C.UTF-8"
+ENV NOKOGIRI_USE_SYSTEM_LIBRARIES "YES"
+
+# Set correct environment variables.
+RUN mkdir -p /var/www/docker
+WORKDIR /var/www/docker
+
+# Set up application
+COPY . .
+
+# Init gems
+RUN echo "gem: --no-rdoc --no-ri" > ~/.gemrc
+RUN . /etc/profile.d/rvm.sh && \
+  bundle config --global with 'development test' && \
+  bundle config --global system true && \
+  bundle config --global jobs 10 && \
+  bundle config --global build.nokogiri --use-system-libraries && \
+  bundle install
+
+CMD ["bash"]

data/Gemfile

@@ -1,6 +1,4 @@
 source "https://rubygems.org"
 
-gem "final_redirect_url", :git => "git@github.com:yubele/final_redirect_url"
-
 # Specify your gem's dependencies in web_stat.gemspec
 gemspec

data/Gemfile.lock

@@ -1,13 +1,7 @@
-GIT
-  remote: git@github.com:yubele/final_redirect_url
-  revision: 45df878ec9495ebbfa06dc0a60cc5043c2519e16
-  specs:
-    final_redirect_url (0.1.1)
-
 PATH
   remote: .
   specs:
-    web_stat (0.2.11)
+    web_stat (0.3.4)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)
@@ -15,23 +9,26 @@ PATH
       nokogiri (>= 1.10.4)
       ruby-readability (>= 0.7)
       sanitize (>= 5.0.0)
+      selenium-webdriver (= 3.142.7)
 
 GEM
   remote: https://rubygems.org/
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
+    byebug (11.1.3)
+    childprocess (3.0.0)
     cld (0.8.0)
       ffi
-    coderay (1.1.2)
-    connection_pool (2.2.2)
+    coderay (1.1.3)
+    connection_pool (2.2.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     crass (1.0.6)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.12.2)
+    ffi (1.13.1)
     guess_html_encoding (0.0.11)
     hashdiff (1.0.1)
     http-cookie (1.0.3)
@@ -48,7 +45,7 @@ GEM
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0425)
+    mime-types-data (3.2020.0512)
     mini_portile2 (2.4.0)
     natto (1.2.0)
       ffi (>= 1.9.0)
@@ -63,7 +60,10 @@ GEM
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (4.0.4)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    public_suffix (4.0.5)
     rake (13.0.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -71,7 +71,7 @@ GEM
       rspec-mocks (~> 3.9.0)
     rspec-core (3.9.2)
       rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.1)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-mocks (3.9.1)
@@ -81,11 +81,15 @@ GEM
     ruby-readability (0.7.0)
       guess_html_encoding (>= 0.0.4)
       nokogiri (>= 1.6.0)
+    rubyzip (2.3.0)
     safe_yaml (1.0.5)
-    sanitize (5.1.0)
+    sanitize (5.2.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)
+    selenium-webdriver (3.142.7)
+      childprocess (>= 0.5, < 4.0)
+      rubyzip (>= 1.2.2)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
@@ -99,8 +103,8 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  final_redirect_url!
-  pry (>= 0.12.2)
+  pry (>= 0.13.1)
+  pry-byebug (= 3.9.0)
   rake (>= 10.0)
   rspec (>= 3.0)
   web_stat!

data/docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3.8"
+networks:
+  app-tier:
+    driver: bridge
+services:
+  web_stat:
+    tty: true
+    stdin_open: true
+    container_name: web_stat
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ./:/var/www/docker:cached
+    working_dir: /var/www/docker
+    command: bash
+    networks:
+      - app-tier

data/docker/exec

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Check to support tty.
+if [ "$(tty>/dev/null;echo $?)" != "0" ];then
+    DOCKERCOMPOSE_EXEC="docker-compose exec -T"
+else
+    DOCKERCOMPOSE_EXEC="docker-compose exec"
+fi
+$DOCKERCOMPOSE_EXEC web_stat /bin/bash -c ". /etc/profile.d/rvm.sh && . /root/.nvm/nvm.sh && $*"

data/docker/start

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+if [ "$1" = "rm" ];then
+  shift
+  docker-compose rm -f
+fi
+docker-compose stop
+if [ "$1" = "attach" ]; then
+  docker-compose up -d
+  docker attach $2
+else
+  docker-compose up $@
+fi

data/lib/helpers/web_drive_helper.rb

@@ -0,0 +1,30 @@
+module WebStat
+  class WebDriverHelper
+    class << self
+      # Get last url
+      # @param [String] url
+      # @param [Integer] delay
+      def get_last_url(url, delay=nil)
+        Selenium::WebDriver.logger.output = File.join("/tmp", "selenium.log")
+        Selenium::WebDriver.logger.level = :info
+        options = Selenium::WebDriver::Chrome::Options.new(args: [
+          'headless',
+          'no-sandbox',
+          'disable-gpu',
+          'start-maximized',
+          'window-size=1920,1080'
+          ])
+        driver = Selenium::WebDriver.for(:chrome, options: options)
+        driver.manage.timeouts.implicit_wait = 10
+        Selenium::WebDriver::Wait.new(timeout: 10)
+        driver.get(url)
+        if delay.is_a?(Integer)
+          sleep delay
+        end
+        last_url = driver.current_url
+        driver.quit
+        last_url
+      end
+    end
+  end
+end

data/lib/web_stat.rb

@@ -1,14 +1,18 @@
 require "bundler"
 
+require 'cld'
 require 'uri'
 require 'digest'
+require 'logger'
 require 'sanitize'
 require 'nokogiri'
 require 'open-uri'
+require 'net/http'
 require 'ruby-readability'
-require 'final_redirect_url'
-require 'cld'
+require 'selenium-webdriver'
 
+require "helpers/web_drive_helper"
+require "web_stat/final_redirect_url"
 require "web_stat/categorize"
 require "web_stat/configure"
 require "web_stat/errors"

data/lib/web_stat/config/web_stat.yml

@@ -1,14 +1,21 @@
-# Minimum number of characters to detect meta title
-min_length_of_meta_title: 10
-# Split regular expression for titles 
-regex_to_sprit_title: '\||-|:|｜|：|〜|\~| – '
-# User Agent
-user_agent: "web_stat gem agent"
-# Eyecatch image xpaths
-eyecatch_image_xpaths:
-  - '/html/head/meta[@property="twitter:image"]/@content'
-  - '/html/head/meta[@property="og:image"]/@content'
-  - '//img[@class="attachment-post-thumbnail"]/@src'
-  - '//div[@id="content"]//img/@src'
-  - '//img/@src'
-userdic: ""
+development: &development
+  # Minimum number of characters to detect meta title
+  min_length_of_meta_title: 10
+  # Split regular expression for titles 
+  regex_to_sprit_title: '\||-|:|｜|：|〜|\~| – '
+  # User Agent
+  user_agent: "web_stat gem agent"
+  # Eyecatch image xpaths
+  eyecatch_image_xpaths:
+    - '/html/head/meta[@property="twitter:image"]/@content'
+    - '/html/head/meta[@property="og:image"]/@content'
+    - '//img[@class="attachment-post-thumbnail"]/@src'
+    - '//div[@id="content"]//img/@src'
+    - '//img/@src'
+  userdic: ""
+  use_chromedirver: false
+test:
+  <<: *development
+production:
+  <<: *development
+  use_chromedirver: true

data/lib/web_stat/configure.rb

@@ -3,31 +3,37 @@ module WebStat
   class Configure
     DEFAULT_CONFIG_FILE_PATH = 'config/web_stat.yml'
     
-    # Get yaml
-    def self.get
-      YAML.load_file(self.get_configure_path)
-    end
-    
-    # Get configure path
-    def self.get_configure_path
-      if File.exists?(self.get_custom_configure_path)
-        self.get_custom_configure_path
-      else
-        self.get_default_configure_path
+    class << self
+      # Get yaml
+      def get
+        if defined? Rails
+          YAML.load_file(get_configure_path)[Rails.env]
+        else
+          YAML.load_file(get_configure_path)["production"]
+        end
       end
-    end
-    
-    # Get default configure path
-    def self.get_default_configure_path
-      File.join(File.expand_path("../", __FILE__), DEFAULT_CONFIG_FILE_PATH)
-    end
-    
-    # Get custom configure path
-    def self.get_custom_configure_path
-      if defined? Rails
-	File.join(Rails.root, DEFAULT_CONFIG_FILE_PATH)
-      else
-        File.join(Bundler.root, DEFAULT_CONFIG_FILE_PATH)
+      
+      # Get configure path
+      def get_configure_path
+        if File.exists?(get_custom_configure_path)
+          get_custom_configure_path
+        else
+          get_default_configure_path
+        end
+      end
+      
+      # Get default configure path
+      def get_default_configure_path
+        File.join(File.expand_path("../", __FILE__), DEFAULT_CONFIG_FILE_PATH)
+      end
+      
+      # Get custom configure path
+      def get_custom_configure_path
+        if defined? Rails
+  	      File.join(Rails.root, DEFAULT_CONFIG_FILE_PATH)
+        else
+          File.join(Bundler.root, DEFAULT_CONFIG_FILE_PATH)
+        end
       end
     end
   end

data/lib/web_stat/fetch.rb

@@ -19,7 +19,6 @@ module WebStat
         title.strip
       end
     end
-    
     # Get name of domain 
     def site_name
       begin
@@ -33,7 +32,6 @@ module WebStat
         site_name.strip
       end
     end
-      []
     # Get main section
     def content
       Sanitize.clean(Readability::Document.new(@nokogiri.at('body')).content)
@@ -59,7 +57,7 @@ module WebStat
     # Get local path to save url
     # @param [String] url
     def save_local_path(url)
-      return nil if url.nil?
+      return nil if url.nil? || url.empty?
       tmp_file = "/tmp/#{Digest::SHA1.hexdigest(url)}"
       agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
       image = agent.get(url)
@@ -124,7 +122,7 @@ module WebStat
     # Get original url
     # @param [String] url
     def original_url(url)
-      last_url = FinalRedirectUrl.final_redirect_url(url)
+      last_url = WebStat::FinalRedirectUrl.final_redirect_url(url)
       unless last_url.nil? || last_url.scrub('').empty?
         last_url
       else

data/lib/web_stat/final_redirect_url.rb

@@ -0,0 +1,54 @@
+# ref) https://github.com/indyarocks/final_redirect_url
+# customize
+#  Changed 
+module WebStat
+  class FinalRedirectUrl
+    class << self
+      def final_redirect_url(url, options={})
+        final_url = ''
+        if is_valid_url?(url)
+          begin
+            redirect_lookup_depth = options[:depth].to_i > 0 ? options[:depth].to_i : 10
+            response_uri = get_final_redirect_url(url, redirect_lookup_depth)
+            final_url =  url_string_from_uri(response_uri)
+          rescue Exception => ex
+            # nothing
+          end
+        end
+        final_url
+      end
+    
+      private
+      def is_valid_url?(url)
+        url.to_s.match? URI::regexp(['http', 'https'])
+      end
+      def get_final_redirect_url(url, limit = 10)
+        return url if limit <= 0
+        uri = URI.parse(url)
+        response = ::Net::HTTP.get_response(uri)
+        if response.class == Net::HTTPOK
+          if WebStat::Configure.get["use_chromedirver"]
+            return URI.parse(WebStat::WebDriverHelper.get_last_url(uri))
+          else
+            return URI.parse(uri)
+          end
+        else
+          redirect_location = response['location']
+          location_uri = URI.parse(redirect_location)
+          if location_uri.host.nil?
+            redirect_location = uri.scheme + '://' + uri.host + redirect_location
+          end
+          warn "redirected to #{redirect_location}"
+          get_final_redirect_url(redirect_location, limit - 1)
+        end
+      end
+      def url_string_from_uri(uri)
+        url_str = "#{uri.scheme}://#{uri.host}#{uri.request_uri}"
+        if uri.fragment
+          url_str = url_str + "##{uri.fragment}"
+        end
+        url_str
+      end
+    end
+  end
+end

data/lib/web_stat/tag.rb

@@ -4,7 +4,7 @@ module WebStat
     attr_accessor :natto_mecab, :article
     
     def initialize(article, userdic: nil)
-        @natto_mecab = Natto::MeCab.new(userdic: userdic)
+      @natto_mecab = Natto::MeCab.new(userdic: userdic)
       @article = article
     end
     

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.2.11"
+  VERSION = "0.3.4"
 end

data/spec/spec_helper.rb

@@ -1,12 +1,18 @@
 require 'rspec/expectations'
 require "bundler/setup"
 require 'pry'
+require 'pry-byebug'
 require "web_stat"
 
 require 'webmock'
 include WebMock::API
 WebMock.enable!
 
+WebMock.disable_net_connect!({
+  allow_localhost: true,
+  allow: 'chromedriver.storage.googleapis.com'
+})
+
 RSpec.configure do |config|
   # Enable flags like --only-failures and --next-failure
   config.example_status_persistence_file_path = ".rspec_status"

data/web_stat.gemspec

@@ -27,9 +27,11 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "natto", ">= 1.1.2"
   spec.add_runtime_dependency "sanitize", ">= 5.0.0"
   spec.add_runtime_dependency "cld", ">= 0.8.0"
+  spec.add_runtime_dependency "selenium-webdriver", "= 3.142.7"
 
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", ">= 3.0"
-  spec.add_development_dependency "pry", ">= 0.12.2"
+  spec.add_development_dependency "pry", ">= 0.13.1"
   spec.add_development_dependency "webmock", ">= 3.6.0"
+  spec.add_development_dependency "pry-byebug", "3.9.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.3.4
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-09 00:00:00.000000000 Z
+date: 2020-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.142.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.142.7
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +156,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.2
+        version: 0.13.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.2
+        version: 0.13.1
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +178,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.6.0
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.9.0
 description: Fetch the web pages and stat.
 email:
 - yube@newsdict.jp
@@ -177,12 +205,17 @@ files:
 - ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
+- Dockerfile
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/fetch_as_html
+- docker-compose.yml
+- docker/exec
+- docker/start
+- lib/helpers/web_drive_helper.rb
 - lib/web_stat.rb
 - lib/web_stat/categorize.rb
 - lib/web_stat/config/web_stat.yml
@@ -191,6 +224,7 @@ files:
 - lib/web_stat/fetch.rb
 - lib/web_stat/fetch/fetch_as_html.rb
 - lib/web_stat/fetch/fetch_as_web.rb
+- lib/web_stat/final_redirect_url.rb
 - lib/web_stat/tag.rb
 - lib/web_stat/tasks/install.rake
 - lib/web_stat/version.rb
@@ -224,7 +258,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Get the status of  the web pages.