crawler_sample 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZTEyODc4NDZlMmE3MWYyYzM5YzUyYmQ5ZjlkMmEwZTAxNGQ3NWMwMQ==
+  data.tar.gz: !binary |-
+    ZGE0ZTA0NzdhNTkwNGM5OWY4ZmE5NDU5ZWY5MjJmZjQzNWFjNjBjNw==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MGZmYmZlZTRkYmUxYWU5NjNlMjcyMGNmNTZhOWI4YjcxZmVkNzBlOThlNTFm
+    OTU1ZWUyYzY3MWE5MGU1ZmQyYTA3YWNlMTAyMTc0ODIzZTJjZTZmNjNlNWNj
+    MDJjNDY5ODFjZGFhN2NiZjc5YzQxOTNkMmU4OGY4NGUxNGMzYzk=
+  data.tar.gz: !binary |-
+    NTI3ZWMxMTJkOWU1MzNhNDUzYTA2NzFmOWFlNTNlY2JjNjBkMjA0OWY5YjM1
+    NzMwMDE4YTI4MjVhOTgyN2ZkYTdjZjQ5YTg2Y2ZmNDAzMWM5ZTBlM2YzNzY3
+    YWVkMDc5YWJlZDRmMTc2MjVhZmFhN2ExYjViNTRjZmQ1MDJmN2I=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in crawler_sample.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Shigeki Doumae
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+# CrawlerSample
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'crawler_sample'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install crawler_sample
+
+## Usage
+
+This gem is a web crawler sample code.
+So I do not reccmmend that you use.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/crawler_sample.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'crawler_sample/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "crawler_sample"
+  gem.version       = CrawlerSample::VERSION
+  gem.authors       = ["Shigeki Doumae"]
+  gem.email         = ["shigeki.doumae@gmail.com"]
+  gem.description   = %q{This gem is a web crawler sample code.So I don't reccmmend that you use.}
+  gem.summary       = %q{This gem is a web crawler sample code.So I don't reccmmend that you use.}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency "nokogiri", "~>1.5.6"
+  #gem.add_dependency "open-uri"
+  #gem.add_dependency "kconv"
+
+end

data/lib/crawler_sample.rb

@@ -0,0 +1,80 @@
+require "crawler_sample/version"
+require 'crawler_sample/url'
+require 'crawler_sample/string'
+require "nokogiri"
+require "open-uri"
+require 'kconv'
+module CrawlerSample
+  # Your code goes here...
+  attr_accessor :top_url, :target_urls, :exclude_urls, :deep_flg, :delay, :stop_flg, :crawl_stop_count
+
+  def crawl(url=nil)
+    raise "URL is Blank" if url.nil?
+    self.target_urls=[url]
+    self.exclude_urls=[]
+    target_scheme=URI.parse(url).scheme
+    target_host=URI.parse(url).host
+    error_cnt=0
+    crawl_page_cnt=0
+    self.crawl_stop_count = 1000 if self.crawl_stop_count.to_i <= 0
+    loop do
+      begin
+        break if self.target_urls.empty? || self.stop_flg==true
+        url =  self.target_urls.pop
+        self.exclude_urls << url
+        begin
+          p "SuccessURL #{url}"
+          site_contents = self.scrape(url)
+          crawl_page_cnt += 1
+        rescue
+          p "ErrorURL #{url}"
+        end
+        site_contents.search("a").each do|anc|
+          #ホスト名がtargetのホストと違う場合はクロール対象外にする
+          begin; URI.parse(anc["href"]).host; rescue; next; end
+          next unless anc["href"].scan(/\.(jpg|jpeg|png|gif|bmp|zip|exe|pdf|lzh)/i).empty?
+          next if URI.parse(anc["href"]).host && target_host!=URI.parse(anc["href"]).host
+          anc["href"] = URI.parse(anc["href"]).path if URI.parse(anc["href"]).host
+          anc["href"] = anc["href"].gsub(/\/\.{1,2}/,"")
+          anc["href"] = "/#{anc["href"]}" if anc["href"][0] != "/"
+          self.target_urls << "#{target_scheme}://#{target_host}#{anc["href"]}".gsub(/\/\.{1,2}/,"")
+          self.target_urls = (self.target_urls - self.exclude_urls).uniq
+        end
+        yield site_contents
+      rescue => e
+        error_cnt +=1
+        p "error #{error_cnt} #{e}"
+        next
+      end
+      crawl_is_force_stop if error_cnt > 200 || crawl_page_cnt > self.crawl_stop_count
+    end
+  end
+
+  def crawl_is_force_stop
+    self.stop_flg=true
+  end
+
+  def crawl_from_url(url=nil)
+    raise "URL is Blank" if url.nil?
+    site_contents = self.scrape(url)
+    return if site_contents.nil?
+    yield site_contents
+  end
+
+  def crawl_delay=(delay=nil)
+    self.delay= delay.nil? ? 1 : delay
+  end
+
+  def crawl_delay
+    return self.delay.nil? ? 1 : self.delay
+  end
+
+  def scrape(url, option={})
+    #self.delay = option[:delay].present? ? option[:delay] : 1
+    sleep self.crawl_delay #delay
+    html=open(url,"r:binary","User-Agent"=>"Blue Field 0.5.0.1" ).read
+    return  Nokogiri::HTML(html.toutf8, nil, 'utf-8')
+  rescue
+    raise FaildScrape
+  end
+end

data/lib/crawler_sample/string.rb

@@ -0,0 +1,14 @@
+require "open-uri"
+class String
+  include CrawlerSample
+
+  def crawler(&block)
+    url=Url.new(:value=>self)
+    raise "URL is invalid" unless url.valid?
+    crawl(url.value) do|contents|
+      block.call(contents) if block_given?
+    end
+  end
+end
+
+

data/lib/crawler_sample/url.rb

@@ -0,0 +1,19 @@
+class Url
+  attr_accessor :scheme, :host, :value
+  def initialize(values = {})
+    values.each do |k, v|
+      self.send("#{k}=", v)
+    end
+  end
+
+  def valid?
+    begin
+      uri = URI.parse(self.value)
+      resp = uri.kind_of?(URI::HTTP)
+      return resp
+    rescue URI::InvalidURIError
+      return false
+    end
+  end
+
+end

data/lib/crawler_sample/version.rb

@@ -0,0 +1,3 @@
+module CrawlerSample
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: crawler_sample
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Shigeki Doumae
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.6
+description: This gem is a web crawler sample code.So I don't reccmmend that you use.
+email:
+- shigeki.doumae@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- crawler_sample.gemspec
+- lib/crawler_sample.rb
+- lib/crawler_sample/string.rb
+- lib/crawler_sample/url.rb
+- lib/crawler_sample/version.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: This gem is a web crawler sample code.So I don't reccmmend that you use.
+test_files: []