web_scraping_helper 0.5.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 02ab48dc934b462c435c3cc75f1223f10624ce22
+  data.tar.gz: 1a1e020edcecc0ba2af3e2b2a67a7c0cd224e4a1
+SHA512:
+  metadata.gz: bb4312d25cddd201cfec98a8d24972d8ea96395d99451d0ca28224db0aee1b2be888fb269f7b3bc5dc014fdfbd005de25f19d1fe4d95b8340a73751ddb33f845
+  data.tar.gz: b8895c65fe159fc1c8e0f152807f292e9eaabb53e945c479f5cb74b294ff0d2ac859982f2b0f959900e2f435864916ddb356ecfb7870c0ceee0988684f6e6779

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/vendor

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in web_scraping_helper.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,6 @@
+guard :minitest do
+  # with Minitest::Unit
+  watch(%r{^test/(.*)\/?(.*)_test\.rb$})
+  watch(%r{^lib/(.*/)?([^/]+)\.rb$})     { |m| "test/#{m[1]}#{m[2]}_test.rb" }
+  watch(%r{^test/test_helper\.rb$})      { 'test' }
+end

data/README.md

@@ -0,0 +1,36 @@
+# WebScrapingHelper
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/web_scraping_helper`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'web_scraping_helper'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install web_scraping_helper
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/web_scraping_helper.
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "web_scraping_helper"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/web_scraping_helper/version.rb

@@ -0,0 +1,3 @@
+class WebScrapingHelper
+  VERSION = "0.5.1"
+end

data/lib/web_scraping_helper.rb

@@ -0,0 +1,146 @@
+#-*- encoding: utf-8 -*-
+require "web_scraping_helper/version"
+require 'rest-client'
+require 'fileutils'
+
+class WebScrapingHelper
+  DEFAULT_USER_AGENT = 'Mozilla/5.0'
+  DEFAULT_WAIT_TIME = 1
+  DEFAULT_ENCODING = Encoding::UTF_8
+
+  def self.reset!
+    @@global_cache_dir = nil
+    @@blocks = []
+  end
+  reset!
+
+  def self.cache_dir
+    @@global_cache_dir
+  end
+
+  def self.cache_dir=(v)
+    @@global_cache_dir = v
+  end
+
+  def self.before(&block)
+    @@blocks << {timing: :before, proc: block}
+  end
+
+  def self.after(&block)
+    @@blocks << {timing: :after, proc: block}
+  end
+
+  def initialize(cookie_filename = nil)
+    @jar = HTTP::CookieJar.new
+    if cookie_filename
+      @jar.load(cookie_filename) if File.exist?(cookie_filename)
+      @cookie_filename = cookie_filename
+    end
+  end
+
+  attr_accessor :user_agent, :wait_time, :encoding
+  attr_accessor :cache_dir
+
+  def post(url, opts = {})
+    request_http(:post, url, opts)
+  end
+  alias post_http post # support old method
+
+  def get(url, opts = {})
+    request_http(:get, url, opts)
+  end
+  alias get_http get # support old method
+
+  def exist_cookie?(url)
+    cookie = HTTP::Cookie.cookie_value(@jar.cookies(url))
+    not cookie.empty?
+  end
+
+  private
+
+  def request_http(request_method, url, opts)
+    @@blocks
+      .select{|block| block[:timing] == :before}
+      .each{|block| block[:proc].call(url) }
+
+    if request_method == :get && (res = find_cache(url))
+      return res
+    end
+    wait
+
+    headers = {}
+    opts.each{|k, v| headers[k.downcase] = v if String === k}
+    unless headers.key?("user-agent")
+      headers["user-agent"] = @user_agent || DEFAULT_USER_AGENT
+    end
+    unless headers.key?("cookie")
+      cookie = HTTP::Cookie.cookie_value(@jar.cookies(url))
+      headers["cookie"] = cookie unless cookie.empty?
+    end
+
+    params = {
+      method: request_method,
+      url: url,
+      headers: headers
+    }
+    params[:payload] = opts[:body] if opts[:body]
+    res = RestClient::Request.execute(params)
+
+    encoding = opts[:encoding] || get_encoding(res)
+    if encoding
+      res.force_encoding(encoding)
+      res.encode!(@encoding || DEFAULT_ENCODING)
+    end
+
+    cookies = res.headers[:set_cookie]
+    if cookies
+      cookies.each{|cookie| @jar.parse(cookie, url)}
+      @jar.save(@cookie_filename) if @cookie_filename
+    end
+
+    set_wait_base_time
+    save_cache(url, res)
+
+    @@blocks
+      .select{|block| block[:timing] == :after}
+      .each{|block| block[:proc].call(url, res) }
+
+    res
+  end
+
+  def get_encoding(res)
+    content_type = res.headers[:content_type]
+    return content_type[/;\s*charset=([^;]+)/, 1] if content_type
+  end
+
+  def find_cache(url)
+    cache_file = url_to_cache_path(url)
+    return nil if cache_file.nil? or not File.exist?(cache_file)
+    File.open(cache_file, "r:utf-8"){|f| f.read}
+  end
+
+  def save_cache(url, html)
+    cache_file = url_to_cache_path(url)
+    return if cache_file.nil?
+    FileUtils.mkdir_p(File.dirname(cache_file))
+    File.open(cache_file, "w+:utf-8"){|f| f.print html}
+  end
+
+  def url_to_cache_path(url)
+    cache_dir = @cache_dir || @@global_cache_dir
+    return unless cache_dir
+    File.expand_path(url.gsub(%r{[\\/:\?"<>\|]}, "_"), cache_dir)
+  end
+
+  def wait
+    if not @prev_time.nil?
+      wait_time = @wait_time || DEFAULT_WAIT_TIME
+      wait_time -= (Time.now - @prev_time)
+      sleep wait_time if wait_time > 0
+    end
+  end
+
+  def set_wait_base_time
+    @prev_time = Time.now
+  end
+end

data/web_scraping_helper.gemspec

@@ -0,0 +1,39 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'web_scraping_helper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "web_scraping_helper"
+  spec.version       = WebScrapingHelper::VERSION
+  spec.authors       = ["momo-lab"]
+  spec.email         = ["momotaro.n@gmail.com"]
+
+  spec.summary       = "WebScrapingHelper"
+  spec.description   = "WebScrapingHelper"
+  spec.homepage      = "https://github.com/momo-lab/web_scraping_helper"
+
+#  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+#  # delete this section to allow pushing this gem to any host.
+#  if spec.respond_to?(:metadata)
+#    spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+#  else
+#    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+#  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rest-client"
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "minitest-reporters"
+  spec.add_development_dependency "minitest-power_assert"
+  spec.add_development_dependency "webmock", "~> 2.0"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency "guard-minitest"
+end

metadata

@@ -0,0 +1,179 @@
+--- !ruby/object:Gem::Specification
+name: web_scraping_helper
+version: !ruby/object:Gem::Version
+  version: 0.5.1
+platform: ruby
+authors:
+- momo-lab
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-06-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-power_assert
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: WebScrapingHelper
+email:
+- momotaro.n@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Guardfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/web_scraping_helper.rb
+- lib/web_scraping_helper/version.rb
+- web_scraping_helper.gemspec
+homepage: https://github.com/momo-lab/web_scraping_helper
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: WebScrapingHelper
+test_files: []