web_stat 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '09e8b3e9d7829de7767ecfa8136109ec26fd8b8d1534a65a756189b51b7a1c1d'
+  data.tar.gz: 004a67e393d63f543cfc5751ec410f308fd074a1e3867e04dfce9cee0002b2af
+SHA512:
+  metadata.gz: f24ef29fddc5ac0da80683c1574312797c8cd60004fe9c1b954add1b222cd802577d3b9abd6314b5d29def3dbd5b1db27f568032a9af5e61964b076d2020dc14
+  data.tar.gz: b29b88a3984adeedb11c8eeca7aed18d9ef8c7417ef431881caaeffa47a2896ba64b53ad00b9bed5af4bc3cca48e6beeb8a10a6202dbf4fc298287c8427a289c

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+# builded gems
+*.gem
+/config

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.5.5
+before_install: gem install bundler -v 2.0.1

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at yube@newsdict.jp. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in web_stat.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,84 @@
+PATH
+  remote: .
+  specs:
+    web_stat (0.1.0)
+      bundler (~> 2.0)
+      final_redirect_url (~> 0.1.0)
+      mechanize (~> 2.7)
+      natto (~> 1.1.2)
+      nokogiri (~> 1.10)
+      ruby-readability (~> 0.7)
+      sanitize (~> 5.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    connection_pool (2.2.2)
+    crass (1.0.4)
+    diff-lcs (1.3)
+    domain_name (0.5.20180417)
+      unf (>= 0.0.5, < 1.0.0)
+    ffi (1.10.0)
+    final_redirect_url (0.1.0)
+    guess_html_encoding (0.0.11)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    mechanize (2.7.6)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0)
+      mime-types (>= 1.17.2)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (>= 2.5.2)
+      nokogiri (~> 1.6)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
+    mime-types (3.2.2)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2018.0812)
+    mini_portile2 (2.4.0)
+    natto (1.1.2)
+      ffi (>= 1.9.0)
+    net-http-digest_auth (1.4.1)
+    net-http-persistent (3.0.0)
+      connection_pool (~> 2.2)
+    nokogiri (1.10.1)
+      mini_portile2 (~> 2.4.0)
+    nokogumbo (2.0.1)
+      nokogiri (~> 1.8, >= 1.8.4)
+    ntlm-http (0.1.1)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    ruby-readability (0.7.0)
+      guess_html_encoding (>= 0.0.4)
+      nokogiri (>= 1.6.0)
+    sanitize (5.0.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.8.0)
+      nokogumbo (~> 2.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.5)
+    webrobots (0.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  web_stat!
+
+BUNDLED WITH
+   2.0.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 yusuke abe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# WebStat
+
+Fetch the web pages and stat.
+
+## Requirements
+
+-  [MeCab _0.996_](http://taku910.github.io/mecab/#download)
+- add runtime dependency
+ - "bundler", "~> 2.0"
+ - "nokogiri", "~> 1.10"
+ - "mechanize", "~> 2.7"
+ - "ruby-readability", "~> 0.7"
+ - "final_redirect_url", "~> 0.1.0"
+ - "natto", "~> 1.1.2"
+- add development dependency
+ - "rake", "~> 10.0"
+ - "rspec", "~> 3.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'web_stat'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install web_stat
+
+## Usage
+
+you can customize web_stat config.
+
+And then execute:
+
+    $ rake web_stat:install

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+load 'lib/web_stat.rb'
+load "lib/web_stat/tasks/install.rake"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/web_stat/categorize.rb

@@ -0,0 +1,5 @@
+module WebStat
+  class Categorize
+    # Unimplemented
+  end
+end

data/lib/web_stat/config/web_stat.yml

@@ -0,0 +1,13 @@
+# Minimum number of characters to detect meta title
+min_length_of_meta_title: 10
+# Split regular expression for titles 
+regex_to_sprit_title: '\||-|:|｜|：|〜|\~| – '
+# User Agent
+user_agent: "web_stat gem agent"
+# Eyecatch image xpaths
+eyecatch_image_xpaths:
+  - '/html/head/meta[@property="twitter:image"]/@content'
+  - '/html/head/meta[@property="og:image"]/@content'
+  - '//img[@class="attachment-post-thumbnail"]/@src'
+  - '//div[@id="content"]//img/@src'
+  - '//img/@src'

data/lib/web_stat/configure.rb

@@ -0,0 +1,30 @@
+require 'yaml'
+module WebStat
+  class Configure
+    DEFAULT_CONFIG_FILE_PATH = 'config/web_stat.yml'
+    
+    # Get yaml
+    def self.get
+      YAML.load_file(self.get_configure_path)
+    end
+    
+    # Get configure path
+    def self.get_configure_path
+      if File.exists?(self.get_custom_configure_path)
+        self.get_custom_configure_path
+      else
+        self.get_default_configure_path
+      end
+    end
+    
+    # Get default configure path
+    def self.get_default_configure_path
+      File.join(File.expand_path("../", __FILE__), DEFAULT_CONFIG_FILE_PATH)
+    end
+    
+    # Get custom configure path
+    def self.get_custom_configure_path
+      File.join(Bundler.root, DEFAULT_CONFIG_FILE_PATH)
+    end
+  end
+end

data/lib/web_stat/errors.rb

@@ -0,0 +1,3 @@
+module WebStat
+  # Unimplemented
+end

data/lib/web_stat/fetch/fetch_as_html.rb

@@ -0,0 +1,11 @@
+module WebStat
+  class FetchAsHtml < Fetch
+    
+    # initialize class
+    # @param [String] html
+    def initialize(html)
+      @html = html
+      @nokogiri = ::Nokogiri::HTML(@html)
+    end
+  end
+end

data/lib/web_stat/fetch/fetch_as_web.rb

@@ -0,0 +1,23 @@
+require 'mechanize'
+module WebStat
+  class FetchAsWeb < Fetch
+    attr_accessor :url
+    
+    # initialize class
+    # @param [String] url
+    def initialize(url)
+      @url = url
+      @html = get_url(url).force_encoding("utf-8")
+      @nokogiri = ::Nokogiri::HTML(@html)
+    end
+    
+    # Get original url
+    def original_url
+      if @url.match(/^http/)
+        FinalRedirectUrl.final_redirect_url(@url)
+      else
+        @url
+      end
+    end
+  end
+end

data/lib/web_stat/fetch.rb

@@ -0,0 +1,92 @@
+require 'uri'
+require 'digest'
+require 'sanitize'
+require 'nokogiri'
+require 'ruby-readability'
+require 'final_redirect_url'
+module WebStat
+  class Fetch
+    attr_accessor :html, :nokogiri, :original_url
+    
+    # Get title
+    # @return [String] title
+    def title
+      begin
+        title = @nokogiri.title.split(/#{WebStat::Configure.get["regex_to_sprit_title"]}/, 2).first
+        if title.length < WebStat::Configure.get["min_length_of_meta_title"]
+          title = @nokogiri.css("h1").first.content
+        end
+      rescue
+        title = @nokogiri.title
+      end
+      title.strip
+    end
+    
+    # Get name of domain 
+    def site_name
+      begin
+        site_name = @nokogiri.title.split(/#{WebStat::Configure.get["regex_to_sprit_title"]}/, 2).last
+      rescue
+        site_name = @nokogiri.title
+      end
+      site_name.strip
+    end
+      
+    # Get main section
+    def content
+      Sanitize.clean(Readability::Document.new(@html).content)
+    end
+      
+    # Get temporary path of image
+    def eyecatch_image_path
+      path = nil
+      WebStat::Configure.get["eyecatch_image_xpaths"].each do |xpath|
+        if @nokogiri.xpath(xpath).first.respond_to?(:value)
+          path = @nokogiri.xpath(xpath).first.value
+          break
+        end
+      end
+      if @url && path.is_a?(String) && !path.match(/^http/) 
+        if path.match(/^\//)
+          path = "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}#{path}"
+        else
+          path = "#{URI.parse(@url).scheme}://#{URI.parse(@url).host}/#{URI.parse(@url).path}/#{path}"
+        end
+      end
+      path
+    end
+    
+    # Get local path to save url
+    # @param [String] url
+    def save_local_path(url)
+      return nil if url.nil? || !url.match(/^http/)
+      tmp_file = "/tmp/#{Digest::SHA1.hexdigest(url)}"
+      File.open(tmp_file, "w") do |_file|
+        _file.puts(get_url(url))
+      end
+      tmp_file
+    end
+    
+    # Get url
+    # @param [String] url
+    def get_url(url)
+      agent = Mechanize.new { |_agent| _agent.user_agent = WebStat::Configure.get["user_agent"] }
+      # Enable to read Robots.txt
+      agent.robots = true
+      agent.get(url, [], nil, { 'Accept-Language' => 'ja'}).body
+    end
+    
+    # Get the informations of @url
+    def stat
+      tag = WebStat::Tag.new(content)
+      {
+        title: title,
+        site_name: site_name,
+        content: content,
+        url: original_url,
+        eyecatch_image_path: save_local_path(eyecatch_image_path),
+        tags: tag.nouns
+      }
+    end
+  end
+end

data/lib/web_stat/tag.rb

@@ -0,0 +1,22 @@
+require 'natto'
+module WebStat
+  class Tag
+    attr_accessor :natto_mecab, :article
+    
+    def initialize(article)
+      @natto_mecab = Natto::MeCab.new
+      @article = article
+    end
+    
+    def nouns
+      words = Hash.new
+      @natto_mecab.parse(@article) do |n|
+        if n.feature =~ /固有名詞/ && !n.surface.empty?
+          words[n.surface] = 1 unless words[n.surface]
+          words[n.surface] += 1
+        end
+      end
+      words.keys
+    end
+  end
+end

data/lib/web_stat/tasks/install.rake

@@ -0,0 +1,13 @@
+require 'rbconfig'
+namespace :web_stat do
+  desc "install web_stat"
+  task :install do
+    p RbConfig::CONFIG
+    # install custom configure
+    #dirname = File.dirname(WebStat::Configure.get_custom_configure_path)
+    #Dir.mkdir(dirname) unless File.directory?(dirname)
+    #FileUtils.cp(WebStat::Configure.get_default_configure_path, dirname)
+    # install mecab
+    
+  end
+end

data/lib/web_stat/version.rb

@@ -0,0 +1,3 @@
+module WebStat
+  VERSION = "0.1.1"
+end

data/lib/web_stat.rb

@@ -0,0 +1,20 @@
+require "bundler"
+Dir.glob("**/*.rb", base: 'lib').each do |file|
+  require(file)
+end
+
+module WebStat
+  class << self
+    # Get web page's stat by url
+    def stat_by_url(url)
+      web_stat = WebStat::FetchAsWeb.new(url)
+      web_stat.stat
+    end
+    
+    # Get web page's stat by html
+    def stat_by_html(html)
+      web_stat = WebStat::FetchAsHtml.new(html)
+      web_stat.stat
+    end
+  end
+end