web_stat 0.2.7 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32af8e84a325614bf546a76fdf603e592abbf4201a559857bc0964fe49226cc1
-  data.tar.gz: 8b63075dbd78d2e4b7368cb6a0a4aa2f8ae655cf7f85b526bd413a5fa210fbc0
+  metadata.gz: 0f62b7ce2720dcd1851a3c0620af8da6e00313e1b5de918e8b1e23ceeeacee89
+  data.tar.gz: 1a52579415bf2a4f96efd39931ef41849751f693658f5cfbcd88202c680a446c
 SHA512:
-  metadata.gz: 45a0fe16c58679cbea01c56710324a461da3bd5c48caa1ca44cf981b2f5443b6ed06ce1db84577edac968e7651cd47ec998ef331dbf47c7306b243a1190fa76d
-  data.tar.gz: c391ce77bf97673c708821fa9067ba64e5b89fbf67d3490376341931f3b9ee6ecebf4a51e0abe69dcc827f7f35d7d6986af353ea2ab4f1d0945f4de2a7d34b2f
+  metadata.gz: 51b5d4fbc2e0969913cec3c1ae4b745cca8a790b4f9638d9a920d844840500424c69f18e7d3f58d2d870b8391e0817f06781699b302f36b2a68d81467608a867
+  data.tar.gz: 2d4ba4bfac9c5d5c76fcc48ba0202b7dbf223b4f42b7d6a649cce22c0742d6b7605c3386c93be2cb29487308714fc94fd2bf1063faf78bdcd5b7b6c5ab060d8b

data/.gitignore

@@ -6,6 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/.bundle
 
 # rspec failure tracking
 .rspec_status

data/.ruby-version

@@ -1 +1 @@
-2.7.0
+2.7.1

data/Dockerfile

@@ -0,0 +1,25 @@
+# Define base image, you can use --build-arg
+ARG base_image="newsdict/rails:ubuntu20.10_nvmv0.35.2_nodev14.3.0_rubyv2.7.1_sasscv2.3.0_ffiv1.13.1_chromedriver"
+FROM $base_image
+
+# Set locale
+ENV LANG "C.UTF-8"
+ENV NOKOGIRI_USE_SYSTEM_LIBRARIES "YES"
+
+# Set correct environment variables.
+RUN mkdir -p /var/www/docker
+WORKDIR /var/www/docker
+
+# Set up application
+COPY . .
+
+# Init gems
+RUN echo "gem: --no-rdoc --no-ri" > ~/.gemrc
+RUN . /etc/profile.d/rvm.sh && \
+  bundle config --global with 'development test' && \
+  bundle config --global system true && \
+  bundle config --global jobs 10 && \
+  bundle config --global build.nokogiri --use-system-libraries && \
+  bundle install
+
+CMD ["bash"]

data/Gemfile

@@ -1,6 +1,4 @@
 source "https://rubygems.org"
 
-gem "final_redirect_url", :git => "git@github.com:yubele/final_redirect_url.git"
-
 # Specify your gem's dependencies in web_stat.gemspec
 gemspec

data/Gemfile.lock

@@ -1,13 +1,7 @@
-GIT
-  remote: git@github.com:yubele/final_redirect_url.git
-  revision: a8e8fb256f044606313d9726278516a0f24449d4
-  specs:
-    final_redirect_url (0.1.0)
-
 PATH
   remote: .
   specs:
-    web_stat (0.2.7)
+    web_stat (0.3.0)
       bundler (>= 2.0.2)
       cld (>= 0.8.0)
       mechanize (>= 2.7)
@@ -15,25 +9,28 @@ PATH
       nokogiri (>= 1.10.4)
       ruby-readability (>= 0.7)
       sanitize (>= 5.0.0)
+      selenium-webdriver (= 3.142.7)
 
 GEM
   remote: https://rubygems.org/
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
+    byebug (11.1.3)
+    childprocess (3.0.0)
     cld (0.8.0)
       ffi
-    coderay (1.1.2)
-    connection_pool (2.2.2)
+    coderay (1.1.3)
+    connection_pool (2.2.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    crass (1.0.5)
+    crass (1.0.6)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    ffi (1.11.3)
+    ffi (1.13.1)
     guess_html_encoding (0.0.11)
-    hashdiff (1.0.0)
+    hashdiff (1.0.1)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     mechanize (2.7.6)
@@ -45,51 +42,58 @@ GEM
       nokogiri (~> 1.6)
       ntlm-http (~> 0.1, >= 0.1.1)
       webrobots (>= 0.0.9, < 0.2)
-    method_source (0.9.2)
+    method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2020.0512)
     mini_portile2 (2.4.0)
-    natto (1.1.2)
+    natto (1.2.0)
       ffi (>= 1.9.0)
     net-http-digest_auth (1.4.1)
-    net-http-persistent (3.1.0)
+    net-http-persistent (4.0.0)
       connection_pool (~> 2.2)
-    nokogiri (1.10.7)
+    nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
     nokogumbo (2.0.2)
       nokogiri (~> 1.8, >= 1.8.4)
     ntlm-http (0.1.1)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    public_suffix (4.0.2)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    public_suffix (4.0.5)
     rake (13.0.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.1)
-      rspec-support (~> 3.9.1)
-    rspec-expectations (3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-support (3.9.2)
+    rspec-support (3.9.3)
     ruby-readability (0.7.0)
       guess_html_encoding (>= 0.0.4)
       nokogiri (>= 1.6.0)
+    rubyzip (2.3.0)
     safe_yaml (1.0.5)
-    sanitize (5.1.0)
+    sanitize (5.2.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.8.0)
       nokogumbo (~> 2.0)
+    selenium-webdriver (3.142.7)
+      childprocess (>= 0.5, < 4.0)
+      rubyzip (>= 1.2.2)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.6)
-    webmock (3.7.6)
+    unf_ext (0.0.7.7)
+    webmock (3.8.3)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
@@ -99,12 +103,12 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  final_redirect_url!
-  pry (>= 0.12.2)
+  pry (>= 0.13.1)
+  pry-byebug (= 3.9.0)
   rake (>= 10.0)
   rspec (>= 3.0)
   web_stat!
   webmock (>= 3.6.0)
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/README.md

@@ -1,3 +1,10 @@
+# !!!!! Precautions when using with Rails !!!!!
+
+Write this line your Gemfile.
+```
+gem "final_redirect_url", :git => "git@github.com:yubele/final_redirect_url"
+```
+
 # WebStat
 
 Fetch the web pages and stat.
@@ -18,6 +25,10 @@ Fetch the web pages and stat.
 - "rake", "~> 10.0"
 - "rspec", "~> 3.0"
 
+### Install mecab
+
+    $ sudo apt install mecab-ipadic-utf8 libmecab
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3.8"
+networks:
+  app-tier:
+    driver: bridge
+services:
+  web_stat:
+    tty: true
+    stdin_open: true
+    container_name: web_stat
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ./:/var/www/docker:cached
+    working_dir: /var/www/docker
+    command: bash
+    networks:
+      - app-tier

data/docker/exec

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Check to support tty.
+if [ "$(tty>/dev/null;echo $?)" != "0" ];then
+    DOCKERCOMPOSE_EXEC="docker-compose exec -T"
+else
+    DOCKERCOMPOSE_EXEC="docker-compose exec"
+fi
+$DOCKERCOMPOSE_EXEC web_stat /bin/bash -c ". /etc/profile.d/rvm.sh && . /root/.nvm/nvm.sh && $*"

data/docker/start

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+if [ "$1" = "rm" ];then
+  shift
+  docker-compose rm -f
+fi
+docker-compose stop
+if [ "$1" = "attach" ]; then
+  docker-compose up -d
+  docker attach $2
+else
+  docker-compose up $@
+fi

data/lib/helpers/web_drive_helper.rb

@@ -0,0 +1,28 @@
+class WebDriverHelper
+  class << self
+    # Get last url
+    # @param [String] url
+    # @param [Integer] delay
+    def get_last_url(url, delay=nil)
+      Selenium::WebDriver.logger.output = File.join("/tmp", "selenium.log")
+      Selenium::WebDriver.logger.level = :info
+      options = Selenium::WebDriver::Chrome::Options.new(args: [
+        'headless',
+        'no-sandbox',
+        'disable-gpu',
+        'start-maximized',
+        'window-size=1920,1080'
+        ])
+      driver = Selenium::WebDriver.for(:chrome, options: options)
+      driver.manage.timeouts.implicit_wait = 10
+      Selenium::WebDriver::Wait.new(timeout: 10)
+      driver.get(url)
+      if delay.is_a?(Integer)
+        sleep delay
+      end
+      last_url = driver.current_url
+      driver.quit
+      last_url
+    end
+  end
+end

data/lib/web_stat.rb

@@ -1,14 +1,18 @@
 require "bundler"
 
+require 'cld'
 require 'uri'
 require 'digest'
+require 'logger'
 require 'sanitize'
 require 'nokogiri'
 require 'open-uri'
+require 'net/http'
 require 'ruby-readability'
-require 'final_redirect_url'
-require 'cld'
+require 'selenium-webdriver'
 
+require "helpers/web_drive_helper"
+require "web_stat/final_redirect_url"
 require "web_stat/categorize"
 require "web_stat/configure"
 require "web_stat/errors"

data/lib/web_stat/fetch.rb

@@ -1,6 +1,6 @@
 module WebStat
   class Fetch
-    attr_accessor :url, :html, :nokogiri, :userdic
+    attr_accessor :url, :html, :nokogiri, :userdic, :status
 
     # Get title
     # @return [String] title
@@ -13,9 +13,12 @@ module WebStat
       rescue
         title = @nokogiri.title
       end
-      title.strip
+      if title.nil?
+        "No Title"
+      else
+        title.strip
+      end
     end
-    
     # Get name of domain 
     def site_name
       begin
@@ -23,9 +26,12 @@ module WebStat
       rescue
         site_name = @nokogiri.title
       end
-      site_name.strip
+      if site_name.nil?
+        "No Sitename"
+      else
+        site_name.strip
+      end
     end
-      []
     # Get main section
     def content
       Sanitize.clean(Readability::Document.new(@nokogiri.at('body')).content)
@@ -41,7 +47,7 @@ module WebStat
           break
         end
       end
-      if path.match(/^\//)
+      if ! path.nil? && path.match(/^\//)
         "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}#{path}"
       else
         path
@@ -67,26 +73,34 @@ module WebStat
     
     # Get url
     # @param [String] url
+    # @param [String] body
     def get_url(url)
       agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
       # Enable to read Robots.txt
       agent.robots = true
-      document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
-      if document.class == Mechanize::File
-        document.body
-      else
-        document.body.encode('UTF-8', document.encoding)
+      begin
+        document = agent.get(url, [], nil, { 'Accept-Language' => 'ja'})
+        if document.class == Mechanize::File
+          body = document.body
+        else
+          body = document.body.encode('UTF-8', document.encoding)
+        end
+        @status = document.code
+      rescue Mechanize::ResponseCodeError => e
+        body = e.page.body
+        @status = e.page.code
       end
+      body
     end
     
     # Get the informations of @url
     # @param [Hash] Specify a dictionary for each language code. example ) {"ja": /***/**.dic, "other": /***/***.dic}
     def stat(userdics: nil)
-      clean_content = content.scrub('').gsub(/[\s　]/, "")
+      clean_content = content.scrub('').gsub(/[\n\t\r　]/, "").gsub(/\s{2,}/, "\s")
       language_code = CLD.detect_language(clean_content)[:code]
-      if userdics && userdics.has_key?(language_code)
+      if userdics && userdics.has_key?(language_code) && File.exists?(userdics[language_code])
         tag = WebStat::Tag.new("#{title} #{content}", userdic: userdics[language_code])
-      elsif userdics && userdics.has_key?("other")
+      elsif userdics && userdics.has_key?("other") && File.exists?(userdics["other"])
         tag = WebStat::Tag.new("#{title} #{content}", userdic: userdics["other"])
       else
         tag = WebStat::Tag.new("#{title} #{content}", userdic: WebStat::Configure.get["userdic"])
@@ -96,6 +110,7 @@ module WebStat
         site_name: site_name,
         content: clean_content,
         language_code: language_code,
+        status: @status,
         url: @url,
         eyecatch_image_path: save_local_path(eyecatch_image_path),
         tags: tag.nouns
@@ -107,8 +122,9 @@ module WebStat
     # Get original url
     # @param [String] url
     def original_url(url)
-      if url.match(/^http/)
-        FinalRedirectUrl.final_redirect_url(url)
+      last_url = FinalRedirectUrl.final_redirect_url(url)
+      unless last_url.nil? || last_url.scrub('').empty?
+        last_url
       else
         url
       end

data/lib/web_stat/final_redirect_url.rb

@@ -0,0 +1,50 @@
+# ref) https://github.com/indyarocks/final_redirect_url
+# customize
+#  Changed 
+
+module FinalRedirectUrl
+
+  def self.final_redirect_url(url, options={})
+    final_url = ''
+    if is_valid_url?(url)
+      begin
+        redirect_lookup_depth = options[:depth].to_i > 0 ? options[:depth].to_i : 10
+        response_uri = get_final_redirect_url(url, redirect_lookup_depth)
+        final_url =  url_string_from_uri(response_uri)
+      rescue Exception => ex
+        # nothing
+      end
+    end
+    final_url
+  end
+
+  private
+  def self.is_valid_url?(url)
+    url.to_s.match? URI::regexp(['http', 'https'])
+  end
+
+  def self.get_final_redirect_url(url, limit = 10)
+    return url if limit <= 0
+    uri = URI.parse(url)
+    response = ::Net::HTTP.get_response(uri)
+    if response.class == Net::HTTPOK
+      return URI.parse(WebDriverHelper.get_last_url(uri))
+    else
+      redirect_location = response['location']
+      location_uri = URI.parse(redirect_location)
+      if location_uri.host.nil?
+        redirect_location = uri.scheme + '://' + uri.host + redirect_location
+      end
+      warn "redirected to #{redirect_location}"
+      get_final_redirect_url(redirect_location, limit - 1)
+    end
+  end
+
+  def self.url_string_from_uri(uri)
+    url_str = "#{uri.scheme}://#{uri.host}#{uri.request_uri}"
+    if uri.fragment
+      url_str = url_str + "##{uri.fragment}"
+    end
+    url_str
+  end
+end

data/lib/web_stat/tag.rb

@@ -4,7 +4,7 @@ module WebStat
     attr_accessor :natto_mecab, :article
     
     def initialize(article, userdic: nil)
-        @natto_mecab = Natto::MeCab.new(userdic: userdic)
+      @natto_mecab = Natto::MeCab.new(userdic: userdic)
       @article = article
     end
     

data/lib/web_stat/version.rb

@@ -1,3 +1,3 @@
 module WebStat
-  VERSION = "0.2.7"
-end
+  VERSION = "0.3.0"
+end

data/spec/spec_helper.rb

@@ -1,12 +1,18 @@
 require 'rspec/expectations'
 require "bundler/setup"
 require 'pry'
+require 'pry-byebug'
 require "web_stat"
 
 require 'webmock'
 include WebMock::API
 WebMock.enable!
 
+WebMock.disable_net_connect!({
+  allow_localhost: true,
+  allow: 'chromedriver.storage.googleapis.com'
+})
+
 RSpec.configure do |config|
   # Enable flags like --only-failures and --next-failure
   config.example_status_persistence_file_path = ".rspec_status"
@@ -59,9 +65,10 @@ end
 
 # Set webmock
 WebStatTestHelper.scheme_and_files.each do |url|
+  status = [200, 404, 503].sample
   WebMock.stub_request(:get, url)
     .to_return(
-      status: 200,
+      status: status,
       body: File.new(File.join(File.dirname(__FILE__), "fixtures", "htmls", File.basename(url))),
       headers: {content_type: 'application/html; charset=utf-8'})
 end

data/spec/web_stat/fetch_spec.rb

@@ -83,6 +83,7 @@ RSpec.describe WebStat::Fetch do
       expect(web_stat[:title]).to eq "gem作成でついまずいたところ"
       expect(web_stat[:site_name]).to eq "newsdict.blog"
       expect(web_stat[:content]).not_to eq nil
+      expect(web_stat[:status]).to eq("200").or eq("404").or eq("503")
       expect(Sanitize.clean(web_stat[:content]).length).to eq web_stat[:content].length
       expect(web_stat[:eyecatch_image_path]).to be_tmp_file_or_nil
     end

data/web_stat.gemspec

@@ -27,9 +27,11 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "natto", ">= 1.1.2"
   spec.add_runtime_dependency "sanitize", ">= 5.0.0"
   spec.add_runtime_dependency "cld", ">= 0.8.0"
+  spec.add_runtime_dependency "selenium-webdriver", "= 3.142.7"
 
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", ">= 3.0"
-  spec.add_development_dependency "pry", ">= 0.12.2"
+  spec.add_development_dependency "pry", ">= 0.13.1"
   spec.add_development_dependency "webmock", ">= 3.6.0"
+  spec.add_development_dependency "pry-byebug", "3.9.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_stat
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.3.0
 platform: ruby
 authors:
 - yusuke abe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-05 00:00:00.000000000 Z
+date: 2020-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.142.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.142.7
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +156,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.2
+        version: 0.13.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.2
+        version: 0.13.1
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +178,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.6.0
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.9.0
 description: Fetch the web pages and stat.
 email:
 - yube@newsdict.jp
@@ -177,12 +205,17 @@ files:
 - ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
+- Dockerfile
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/fetch_as_html
+- docker-compose.yml
+- docker/exec
+- docker/start
+- lib/helpers/web_drive_helper.rb
 - lib/web_stat.rb
 - lib/web_stat/categorize.rb
 - lib/web_stat/config/web_stat.yml
@@ -191,6 +224,7 @@ files:
 - lib/web_stat/fetch.rb
 - lib/web_stat/fetch/fetch_as_html.rb
 - lib/web_stat/fetch/fetch_as_web.rb
+- lib/web_stat/final_redirect_url.rb
 - lib/web_stat/tag.rb
 - lib/web_stat/tasks/install.rake
 - lib/web_stat/version.rb