web_stat 0.2.11 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bcfaeb202076ea30cae6205877d0b0ad9060eb84116b84ef7bd3580cc4349aac
-  data.tar.gz: 98e586440c8f3aed29e38a003f6d1afc96b26b2ae4c21e43d47e1dfaf6aaa16a
+  metadata.gz: 0f62b7ce2720dcd1851a3c0620af8da6e00313e1b5de918e8b1e23ceeeacee89
+  data.tar.gz: 1a52579415bf2a4f96efd39931ef41849751f693658f5cfbcd88202c680a446c
 SHA512:
-  metadata.gz: dfbe6264256f08550ebb42244d92bd81c976dd5dd72a6a4dabffdf6a1366a8f010e8a20527d0bbe7a8334cdb028e86ecae0c63d6cc4368741abe815f1fcb3092
-  data.tar.gz: 9098d904f26dfdfe14c87352cb47ca3f0333f5424ef26d2cd8232c088c2cf8d7dcbeb07012a9294a5c4e4949b51fe71e94d0995465400d87cf7c037fd09ba978
+  metadata.gz: 51b5d4fbc2e0969913cec3c1ae4b745cca8a790b4f9638d9a920d844840500424c69f18e7d3f58d2d870b8391e0817f06781699b302f36b2a68d81467608a867
+  data.tar.gz: 2d4ba4bfac9c5d5c76fcc48ba0202b7dbf223b4f42b7d6a649cce22c0742d6b7605c3386c93be2cb29487308714fc94fd2bf1063faf78bdcd5b7b6c5ab060d8b

data/.gitignore

@@ -6,6 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/.bundle
 
 # rspec failure tracking
 .rspec_status

data/.ruby-version

@@ -1 +1 @@
-2.7.0
+2.7.1

data/Dockerfile

@@ -0,0 +1,25 @@
+# Define base image, you can use --build-arg
+ARG base_image="newsdict/rails:ubuntu20.10_nvmv0.35.2_nodev14.3.0_rubyv2.7.1_sasscv2.3.0_ffiv1.13.1_chromedriver"
+FROM $base_image
+
+# Set locale
+ENV LANG "C.UTF-8"
+ENV NOKOGIRI_USE_SYSTEM_LIBRARIES "YES"
+
+# Set correct environment variables.
+RUN mkdir -p /var/www/docker
+WORKDIR /var/www/docker
+
+# Set up application
+COPY . .
+
+# Init gems
+RUN echo "gem: --no-rdoc --no-ri" > ~/.gemrc
+RUN . /etc/profile.d/rvm.sh && \
+  bundle config --global with 'development test' && \
+  bundle config --global system true && \
+  bundle config --global jobs 10 && \
+  bundle config --global build.nokogiri --use-system-libraries && \
+  bundle install
+
+CMD ["bash"]

data/Gemfile

@@ -1,6 +1,4 @@
 source "https://rubygems.org"
 
-gem "final_redirect_url", :git => "git@github.com:yubele/final_redirect_url"
-
 # Specify your gem's dependencies in web_stat.gemspec
 gemspec

data/Gemfile.lock

@@ -1,13 +1,7 @@
-GIT
-  remote: git@github.com:yubele/final_redirect_url
-  revision: 45df878ec9495ebbfa06dc0a60cc5043c2519e16
-  specs:
-    final_redirect_url (0.1.1)
-
 PATH
   remote: .
   specs:
-    web_stat (0.2.11)
+    web_stat (0.3.0)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)
@@ -15,23 +9,26 @@ PATH
       nokogiri (>= 1.10.4)
       ruby-readability (>= 0.7)
       sanitize (>= 5.0.0)
+      selenium-webdriver (= 3.142.7)
 
 GEM
   remote: https://rubygems.org/
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
+    byebug (11.1.3)
+    childprocess (3.0.0)
     cld (0.8.0)
       ffi
-    coderay (1.1.2)
-    connection_pool (2.2.2)
+    coderay (1.1.3)
+    connection_pool (2.2.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     crass (1.0.6)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.12.2)
+    ffi (1.13.1)
     guess_html_encoding (0.0.11)
     hashdiff (1.0.1)
     http-cookie (1.0.3)
@@ -48,7 +45,7 @@ GEM
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0425)
+    mime-types-data (3.2020.0512)
     mini_portile2 (2.4.0)
     natto (1.2.0)
       ffi (>= 1.9.0)
@@ -63,7 +60,10 @@ GEM
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (4.0.4)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    public_suffix (4.0.5)
     rake (13.0.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -71,7 +71,7 @@ GEM
       rspec-mocks (~> 3.9.0)
     rspec-core (3.9.2)
       rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.1)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-mocks (3.9.1)
@@ -81,11 +81,15 @@ GEM
     ruby-readability (0.7.0)
       guess_html_encoding (>= 0.0.4)
       nokogiri (>= 1.6.0)
+    rubyzip (2.3.0)
     safe_yaml (1.0.5)
-    sanitize (5.1.0)
+    sanitize (5.2.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)
+    selenium-webdriver (3.142.7)
+      childprocess (>= 0.5, < 4.0)
+      rubyzip (>= 1.2.2)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
@@ -99,8 +103,8 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  final_redirect_url!
-  pry (>= 0.12.2)
+  pry (>= 0.13.1)
+  pry-byebug (= 3.9.0)
   rake (>= 10.0)
   rspec (>= 3.0)
   web_stat!

data/docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3.8"
+networks:
+  app-tier:
+    driver: bridge
+services:
+  web_stat:
+    tty: true
+    stdin_open: true
+    container_name: web_stat
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ./:/var/www/docker:cached
+    working_dir: /var/www/docker
+    command: bash
+    networks:
+      - app-tier

data/docker/exec

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Check to support tty.
+if [ "$(tty>/dev/null;echo $?)" != "0" ];then
+    DOCKERCOMPOSE_EXEC="docker-compose exec -T"
+else
+    DOCKERCOMPOSE_EXEC="docker-compose exec"
+fi
+$DOCKERCOMPOSE_EXEC web_stat /bin/bash -c ". /etc/profile.d/rvm.sh && . /root/.nvm/nvm.sh && $*"

data/docker/start

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+if [ "$1" = "rm" ];then
+  shift
+  docker-compose rm -f
+fi
+docker-compose stop
+if [ "$1" = "attach" ]; then
+  docker-compose up -d
+  docker attach $2
+else
+  docker-compose up $@
+fi

data/lib/helpers/web_drive_helper.rb

@@ -0,0 +1,28 @@
+class WebDriverHelper
+  class << self
+    # Get last url
+    # @param [String] url
+    # @param [Integer] delay
+    def get_last_url(url, delay=nil)
+      Selenium::WebDriver.logger.output = File.join("/tmp", "selenium.log")
+      Selenium::WebDriver.logger.level = :info
+      options = Selenium::WebDriver::Chrome::Options.new(args: [
+        'headless',
+        'no-sandbox',
+        'disable-gpu',
+        'start-maximized',
+        'window-size=1920,1080'
+        ])
+      driver = Selenium::WebDriver.for(:chrome, options: options)
+      driver.manage.timeouts.implicit_wait = 10
+      Selenium::WebDriver::Wait.new(timeout: 10)
+      driver.get(url)
+      if delay.is_a?(Integer)
+        sleep delay
+      end
+      last_url = driver.current_url
+      driver.quit
+      last_url
+    end
+  end
+end

data/lib/web_stat.rb

@@ -1,14 +1,18 @@
 require "bundler"
 
+require 'cld'
 require 'uri'
 require 'digest'
+require 'logger'
 require 'sanitize'
 require 'nokogiri'
 require 'open-uri'
+require 'net/http'
 require 'ruby-readability'
-require 'final_redirect_url'
-require 'cld'
+require 'selenium-webdriver'
 
+require "helpers/web_drive_helper"
+require "web_stat/final_redirect_url"
 require "web_stat/categorize"
 require "web_stat/configure"
 require "web_stat/errors"

data/lib/web_stat/fetch.rb

@@ -19,7 +19,6 @@ module WebStat
         title.strip
       end
     end
-    
     # Get name of domain 
     def site_name
       begin
@@ -33,7 +32,6 @@ module WebStat
         site_name.strip
       end
     end
-      []
     # Get main section
     def content
       Sanitize.clean(Readability::Document.new(@nokogiri.at('body')).content)

data/lib/web_stat/final_redirect_url.rb

@@ -0,0 +1,50 @@
+# ref) https://github.com/indyarocks/final_redirect_url
+# customize
+#  Changed 
+
+module FinalRedirectUrl
+
+  def self.final_redirect_url(url, options={})
+    final_url = ''
+    if is_valid_url?(url)
+      begin
+        redirect_lookup_depth = options[:depth].to_i > 0 ? options[:depth].to_i : 10
+        response_uri = get_final_redirect_url(url, redirect_lookup_depth)
+        final_url =  url_string_from_uri(response_uri)
+      rescue Exception => ex
+        # nothing
+      end
+    end
+    final_url
+  end
+
+  private
+  def self.is_valid_url?(url)
+    url.to_s.match? URI::regexp(['http', 'https'])
+  end
+
+  def self.get_final_redirect_url(url, limit = 10)
+    return url if limit <= 0
+    uri = URI.parse(url)
+    response = ::Net::HTTP.get_response(uri)
+    if response.class == Net::HTTPOK
+      return URI.parse(WebDriverHelper.get_last_url(uri))
+    else
+      redirect_location = response['location']
+      location_uri = URI.parse(redirect_location)
+      if location_uri.host.nil?
+        redirect_location = uri.scheme + '://' + uri.host + redirect_location
+      end
+      warn "redirected to #{redirect_location}"
+      get_final_redirect_url(redirect_location, limit - 1)
+    end
+  end
+
+  def self.url_string_from_uri(uri)
+    url_str = "#{uri.scheme}://#{uri.host}#{uri.request_uri}"
+    if uri.fragment
+      url_str = url_str + "##{uri.fragment}"
+    end
+    url_str
+  end
+end

data/lib/web_stat/tag.rb

@@ -4,7 +4,7 @@ module WebStat
     attr_accessor :natto_mecab, :article
     
     def initialize(article, userdic: nil)
-        @natto_mecab = Natto::MeCab.new(userdic: userdic)
+      @natto_mecab = Natto::MeCab.new(userdic: userdic)
       @article = article
     end
     

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.2.11"
+  VERSION = "0.3.0"
 end

data/spec/spec_helper.rb

@@ -1,12 +1,18 @@
 require 'rspec/expectations'
 require "bundler/setup"
 require 'pry'
+require 'pry-byebug'
 require "web_stat"
 
 require 'webmock'
 include WebMock::API
 WebMock.enable!
 
+WebMock.disable_net_connect!({
+  allow_localhost: true,
+  allow: 'chromedriver.storage.googleapis.com'
+})
+
 RSpec.configure do |config|
   # Enable flags like --only-failures and --next-failure
   config.example_status_persistence_file_path = ".rspec_status"

data/web_stat.gemspec

@@ -27,9 +27,11 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "natto", ">= 1.1.2"
   spec.add_runtime_dependency "sanitize", ">= 5.0.0"
   spec.add_runtime_dependency "cld", ">= 0.8.0"
+  spec.add_runtime_dependency "selenium-webdriver", "= 3.142.7"
 
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", ">= 3.0"
-  spec.add_development_dependency "pry", ">= 0.12.2"
+  spec.add_development_dependency "pry", ">= 0.13.1"
   spec.add_development_dependency "webmock", ">= 3.6.0"
+  spec.add_development_dependency "pry-byebug", "3.9.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.3.0
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-09 00:00:00.000000000 Z
+date: 2020-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.142.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.142.7
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +156,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.2
+        version: 0.13.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.2
+        version: 0.13.1
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +178,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.6.0
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.9.0
 description: Fetch the web pages and stat.
 email:
 - yube@newsdict.jp
@@ -177,12 +205,17 @@ files:
 - ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
+- Dockerfile
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/fetch_as_html
+- docker-compose.yml
+- docker/exec
+- docker/start
+- lib/helpers/web_drive_helper.rb
 - lib/web_stat.rb
 - lib/web_stat/categorize.rb
 - lib/web_stat/config/web_stat.yml
@@ -191,6 +224,7 @@ files:
 - lib/web_stat/fetch.rb
 - lib/web_stat/fetch/fetch_as_html.rb
 - lib/web_stat/fetch/fetch_as_web.rb
+- lib/web_stat/final_redirect_url.rb
 - lib/web_stat/tag.rb
 - lib/web_stat/tasks/install.rake
 - lib/web_stat/version.rb
@@ -224,7 +258,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Get the status of  the web pages.