url_reader 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 90d8ea16e1c937474887f35170886a5e3e283c5e
+  data.tar.gz: eec8a64973d2c4ecba39f266cf6a168a6e21145b
+SHA512:
+  metadata.gz: 97e812e7b7fa2ef8f004b86149ef9e28de861daa8190063de2ef0e2aec6e60b55a4fbcd9bd2688ae7fb25c62fe2b69037e7ac521c2042205180d773ce2a1be50
+  data.tar.gz: b7b3e385fdb44e1b58cbf71348556accc336d5ca3e0eb2ea1ef254b1d85cde697e2b1e31431ad3c47588334b6a846c6dfb779f032551892b8f2899117c11417a

data/.gitignore

@@ -0,0 +1,2 @@
+/*.gem
+/tmp

data/.ruby-version

@@ -0,0 +1 @@
+2.1.2

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,55 @@
+PATH
+  remote: .
+  specs:
+    url_reader (0.1.0)
+      activesupport (>= 4)
+      rest-client
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.1.8)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    addressable (2.3.6)
+    crack (0.4.2)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.2.5)
+    i18n (0.6.11)
+    json (1.8.1)
+    mime-types (2.4.3)
+    minitest (5.4.3)
+    netrc (0.8.0)
+    rest-client (1.7.2)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.7)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-mocks (3.1.3)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.2)
+    safe_yaml (1.0.4)
+    thread_safe (0.3.4)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    webmock (1.20.3)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rspec
+  url_reader!
+  webmock

data/lib/initialize.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'active_support/all'
+require 'rest_client'
+
+ActiveSupport::Dependencies.autoload_paths << File.expand_path('../../lib', __FILE__)

data/lib/url_reader/base_error.rb

@@ -0,0 +1,18 @@
+class UrlReader::BaseError < StandardError
+  def self.msg(error, additional_msg = nil)
+    msg = "#{error.class.name}: #{error.message}"
+    if additional_msg
+      msg += ", #{additional_msg}"
+    end
+    msg
+  end
+
+  def initialize(inner_or_msg = nil, additional_msg = nil)
+    if inner_or_msg.is_a?(String)
+      super(inner_or_msg)
+    else
+      super(self.class.msg(inner_or_msg, additional_msg))
+      set_backtrace(inner_or_msg.backtrace)
+    end
+  end
+end

data/lib/url_reader/cannot_resolve_encoding_error.rb

@@ -0,0 +1,2 @@
+class UrlReader::CannotResolveEncodingError < UrlReader::BaseError
+end

data/lib/url_reader/file_cache.rb

@@ -0,0 +1,48 @@
+class Util::UrlReader::FileCache
+  def initialize(cache_dir_path)
+    @cache_dir_path = cache_dir_path
+    @cache = {}
+  end
+
+  def read_entry(key)
+    unless @cache.has_key?(key)
+      ekey = encoded_key(key)
+      hash = hash(ekey)
+      file_path = File.join(@cache_dir_path, hash)
+      value = nil
+      if File.exist?(file_path)
+        value = (decoded_value(File.open(file_path).read.strip.split("\n")
+                                 .select { |x| x.start_with?("#{ekey}\t") }[0].split("\t", 2)[1]) rescue nil)
+      end
+      @cache[key] = value
+    end
+    @cache[key]
+  end
+
+  def write_entry(key, value)
+    @cache[key] = value
+    ekey = encoded_key(key)
+    hash = hash(ekey)
+    file_path = File.join(@cache_dir_path, hash)
+    File.open(file_path, 'a') { |f| f.puts("#{ekey}\t#{encoded_value(value)}") }
+    true
+  end
+
+  private
+
+  def encoded_key(key)
+    URI.encode_www_form_component(key)
+  end
+
+  def encoded_value(value)
+    CGI.escape(value)
+  end
+
+  def decoded_value(value)
+    CGI.unescape(value)
+  end
+
+  def hash(key)
+    Digest::SHA256.hexdigest(key)
+  end
+end

data/lib/url_reader/read_error.rb

@@ -0,0 +1,20 @@
+class UrlReader::ReadError < UrlReader::BaseError
+  PageNotFound = 0
+  RequestTimeout = 1
+  UnidentifiedError = 2
+
+  attr_reader :type
+
+  def initialize(*args)
+    super(*args)
+    inner = args[0]
+    @type =
+      if inner.is_a?(RestClient::ResourceNotFound)
+        PageNotFound
+      elsif inner.is_a?(RestClient::RequestTimeout)
+        RequestTimeout
+      else
+        UnidentifiedError
+      end
+  end
+end

data/lib/url_reader/url_fixer.rb

@@ -0,0 +1,7 @@
+module UrlReader::UrlFixer
+  extend ActiveSupport::Concern
+
+  def fixed_url(url)
+    url.gsub(/[^[:ascii:]]| /) { |c| URI.encode(c) }.gsub('[', '%5B').gsub(']', '%5D')
+  end
+end

data/lib/url_reader.rb

@@ -0,0 +1,108 @@
+require_relative 'initialize'
+
+require 'kconv'
+
+module UrlReader
+  extend ActiveSupport::Concern
+
+  include UrlFixer
+
+  REQUEST_TIMEOUT = 10
+  REQUEST_OPEN_TIMEOUT = 10
+
+  def read_url(url, options = {})
+    if defined?(Rails) && Rails.env.development?
+      if ENV['READ_URL_CACHE_NOT_USE'] != 'true'
+        ckey = cache_key(url, options)
+        res = cache.read_entry(ckey)
+        res || read_url_core_with_cache_write(url, options, ckey)
+      else
+        read_url_core_with_cache_write(url, options)
+      end
+    else
+      read_url_core(url, options)
+    end
+  end
+
+  private
+
+  def cache
+    @cache ||= begin
+                 %x(mkdir -p #{Rails.root}/tmp/cache/url_reader)
+                 FileCache.new(File.join(Rails.root, 'tmp/cache/url_reader'))
+               end
+  end
+
+  def cache_key(url, options)
+    "#{url}?#{options.to_s}"
+  end
+
+  def read_url_core_with_cache_write(url, options, ckey = nil)
+    ckey ||= cache_key(url, options)
+    res = read_url_core(url, options)
+    return nil if res.nil?
+    cache.write_entry(ckey, res)
+    res
+  end
+
+  def read_url_core(url, options)
+    valid_url = fixed_url(url)
+    headers = {}
+    headers[:user_agent] = options[:user_agent] if options[:user_agent]
+    hash = {
+      url: valid_url,
+      timeout: options[:request_timeout] || REQUEST_TIMEOUT,
+      open_timeout: options[:request_open_timeout] || REQUEST_OPEN_TIMEOUT,
+      headers: headers
+    }
+    response =
+      begin
+        if options[:method] == :post
+          RestClient::Request.execute(hash.merge(method: :post, payload: options[:params]))
+        else
+          RestClient::Request.execute(hash.merge(method: :get))
+        end
+      rescue RestClient::ResourceNotFound,
+             RestClient::InternalServerError,
+             RestClient::RequestTimeout,
+             RestClient::ServerBrokeConnection,
+             Errno::ECONNREFUSED,
+             Errno::ECONNRESET => e
+        ne = ReadError.new(e, "Read #{hash[:url]} failed")
+        if options[:ignore_not_found]
+          options[:ignore_read_errors] ||= []
+          options[:ignore_read_errors] << 'PageNotFound'
+        end
+        if ignore_errors = options[:ignore_read_errors]
+          return nil if ignore_errors.map { |x| x.is_a?(Integer) ? x : ReadError.const_get(x) }.include?(ne.type)
+        end
+        raise ne
+      end
+    return nil unless response
+    return resolve_encoding(response) if response.headers[:content_type] !~ /^image\//
+    response.to_str
+  end
+
+  def resolve_encoding(response)
+    response_str = response.to_str
+    encoding = response_encoding(response.headers, response_str)
+    begin
+      return response_str.encode(Encoding::UTF_8, encoding)
+    rescue Encoding::UndefinedConversionError => e
+      return response_str.encode(Encoding::UTF_8, Encoding::CP932) if encoding == Encoding::Shift_JIS
+      return response_str.encode(Encoding::UTF_8, Encoding::CP51932) if encoding == Encoding::EUC_JP
+      raise CannotResolveEncodingError, e
+    end
+  end
+
+  def response_encoding(response_headers, response_str)
+    response_str_utf8 = response_str.toutf8
+    [response_headers[:content_type].try(:match, /charset=(?<charset>[^;]+)($|;)/),
+     response_str_utf8.match(/<meta .*?content="[^"]*?charset=(?<charset>[^;"]+)/),
+     response_str_utf8.match(/<meta .*?charset="(?<charset>[^"]+)"/)]
+      .map { |x| x.try(:[], 'charset') }.compact
+      .map { |x| Encoding.find(x) rescue nil }.compact
+      .push(Encoding::UTF_8)
+      .first
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+require 'bundler/setup'
+Bundler.require
+
+require 'webmock/rspec'
+
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }

data/spec/test.html

@@ -0,0 +1 @@
+<html>Test</html>

data/spec/url_reader_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+class DummUrlReader
+  include UrlReader
+end
+
+describe UrlReader do
+  let(:content) { File.read(File.expand_path('../test.html', __FILE__)) }
+  let(:url) { 'http://www.example.com/test.html' }
+  let(:read) { -> { DummUrlReader.new.read_url(url) } }
+
+  describe '#read_url' do
+    it 'should read url' do
+      stub_request(:any, url).to_return(body: content)
+      expect(read.call).to eq content
+    end
+  end
+end

data/url_reader.gemspec

@@ -0,0 +1,15 @@
+Gem::Specification.new do |s|
+  s.name = 'url_reader'
+  s.version = '0.1.0'
+  s.authors = ['Tetsuri Moriya']
+  s.email = ['tetsuri.moriya@gmail.com']
+  s.summary = 'Url reader'
+  s.description = 'Web retrieval module with cache'
+  s.homepage = 'https://github.com/pandora2000/url_reader'
+  s.license = 'MIT'
+  s.files = `git ls-files`.split("\n")
+  s.add_development_dependency 'rspec', '>= 0'
+  s.add_development_dependency 'webmock', '>= 0'
+  s.add_runtime_dependency 'activesupport', '>= 4'
+  s.add_runtime_dependency 'rest-client', '>= 0'
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: url_reader
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Tetsuri Moriya
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Web retrieval module with cache
+email:
+- tetsuri.moriya@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
+- README.md
+- lib/initialize.rb
+- lib/url_reader.rb
+- lib/url_reader/base_error.rb
+- lib/url_reader/cannot_resolve_encoding_error.rb
+- lib/url_reader/file_cache.rb
+- lib/url_reader/read_error.rb
+- lib/url_reader/url_fixer.rb
+- spec/spec_helper.rb
+- spec/test.html
+- spec/url_reader_spec.rb
+- url_reader.gemspec
+homepage: https://github.com/pandora2000/url_reader
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Url reader
+test_files: []