cosmicrawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1c51e03fd72a7db8bad8d0addc1195c5fd96a021
+  data.tar.gz: 2aff46f71f44585ef2bfb00a8c88dd2f6be0009d
+SHA512:
+  metadata.gz: 84f47ea22ed2c920e3f480485d6fbe36377d579131d7706f168c2f888155aab02be99bd0bb38d981dcb1d15934fbed0bb621c024619904c108602e77906ac1f9
+  data.tar.gz: 5f522e9dc55e3f335e5c84e26f9760f199cdb0329fc56f264abf732a191eb92a2e0ccd22e13057aa5bbabf0ffd51d4dada32f9426fdc8b10ebb53d7116f10f2f

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+/nbproject

data/.travis.yml

@@ -0,0 +1,23 @@
+rvm:
+  - 1.9.3
+  - jruby-18mode
+  - jruby-19mode
+  - rbx-18mode
+  - rbx-19mode
+  - ruby-head
+  - jruby-head
+  - 1.8.7
+  - ree
+jdk:
+  - openjdk7
+  - oraclejdk7
+  - openjdk6
+env:
+  - ISOLATED=true
+  - ISOLATED=false
+matrix:
+  exclude:
+- rvm: 1.9.3
+  jdk: openjdk7
+- rvm: 1.9.3
+  jdk: oraclejdk7

data/Gemfile

@@ -0,0 +1,16 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in cosmicrawler.gemspec
+gemspec
+
+group :development do
+  gem "bundler"
+  gem "rspec"
+  gem 'webmock'
+  gem 'pry'
+end
+
+gem 'eventmachine'
+gem 'em-http-request'
+gem 'em-synchrony'
+

data/README.md

@@ -0,0 +1,57 @@
+
+# Cosmicrawler
+
+Cosmicrawler is crawler library for Ruby. It provides scalable asynchronous crawling by http, file, etc using [EventMachine](https://github.com/eventmachine/eventmachine).
+
+[![Build Status](https://travis-ci.org/(https://travis-ci.org/bash0C7/cosmicrawler.png)](https://travis-ci.org/bash0C7/cosmicrawler)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'cosmicrawler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install cosmicrawler
+
+## Usage
+
+### http
+
+````ruby
+require 'cosmicrawler'
+
+Cosmicrawler.http_crawl(%w(http://example.com/1 http://example.com/2)) {|request|
+  get = request.get
+  puts get.response if get.response_header.status == 200
+}
+```` 
+
+````ruby
+require 'cosmicrawler'
+require 'em-http-request'
+
+Cosmicrawler.each(%w(http://example.com/1 http://example.com/2)) {|item|
+  request = EM::HttpRequest.new(item)
+  get = request.get
+  puts get.response if get.response_header.status == 200
+}      
+
+```` 
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## License
+
+Ruby's License

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/cosmicrawler.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cosmicrawler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "cosmicrawler"
+  spec.version       = Cosmicrawler::VERSION
+  spec.authors       = ["Toshiaki Koshiba"]
+  spec.email         = ["koshiba+github@4038nullpointer.com"]
+  spec.description   = 'Cosmicrawler is crawler library for Ruby. It provides scalable asynchronous crawling by (http|file|etc) using EventMachine.'
+  spec.summary       = 'Cosmicrawler is crawler library for Ruby. It provides scalable asynchronous crawling by (http|file|etc) using EventMachine.'
+  spec.homepage      = "https://github.com/bash0C7/cosmicrawler"
+  spec.license       = "Ruby"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  
+  spec.add_runtime_dependency("eventmachine")
+  spec.add_runtime_dependency("em-http-request")
+  spec.add_runtime_dependency("em-synchrony")
+end

data/lib/cosmicrawler.rb

@@ -0,0 +1,19 @@
+module Cosmicrawler
+  %w(version em).each do |lib|
+    require_relative 'cosmicrawler/' + lib
+  end
+
+  DEFAULT_CONCURRENCY = 8
+  
+  class << self
+    def each collection, concurrency = DEFAULT_CONCURRENCY, &block
+      c = Em::Crawler.new concurrency
+      c.each collection, &block
+    end
+
+    def http_crawl urls, concurrency = DEFAULT_CONCURRENCY, &block
+      c = Em::HttpCrawler.new concurrency
+      c.crawl urls, &block
+    end
+  end
+end

data/lib/cosmicrawler/em.rb

@@ -0,0 +1,27 @@
+require'eventmachine'
+require'em-synchrony'
+require "em-synchrony/fiber_iterator"
+
+require 'ostruct'
+
+require_relative 'em/crawler.rb'
+require_relative 'em/http_crawler.rb'
+
+module Cosmicrawler
+  module Em
+    def initialize concurrency
+      @concurrency = concurrency
+    end
+
+    private
+    def iterate collection, &block
+      EventMachine.synchrony do
+        EM::Synchrony::FiberIterator.new(collection, @concurrency).each do |item|
+          block.call item
+        end
+        EM.stop_event_loop
+      end #EventMachine.synchrony
+      self
+    end #iterate
+  end
+end

data/lib/cosmicrawler/em/crawler.rb

@@ -0,0 +1,17 @@
+module Cosmicrawler
+  module Em
+    class Crawler
+      include Em
+
+      def each collection, &block
+        raise unless block_given?
+      
+        iterate(collection) do |item|
+          block.call item
+        end
+      
+        self
+      end
+    end
+  end
+end

data/lib/cosmicrawler/em/http_crawler.rb

@@ -0,0 +1,20 @@
+require'em-http-request'
+require'em-synchrony/em-http'
+
+module Cosmicrawler
+  module Em
+    class HttpCrawler
+      include Em
+    
+      def crawl urls, &block
+        raise unless block_given?
+
+        iterate(urls) do |url|
+          request = ::EM::HttpRequest.new(url)
+          block.call request
+        end
+
+      end
+    end
+  end
+end

data/lib/cosmicrawler/version.rb

@@ -0,0 +1,3 @@
+module Cosmicrawler
+  VERSION = "0.0.1"
+end

data/spec/cosmicrawler_spec.rb

@@ -0,0 +1,94 @@
+require 'spec_helper'
+require 'cosmicrawler'
+
+describe Cosmicrawler do
+  before :each do
+    WebMock.reset!
+      stub_request(:get, /.+/).to_return(:status => 200, :body => "", :headers => {})
+  end
+
+  describe :each do
+    it 'block is required' do
+      lambda {Cosmicrawler.each(%w())}.should raise_error
+    end
+    
+    context %w() do
+      it 'block is not called' do
+        Cosmicrawler.each(%w()) {|item|
+          request = EM::HttpRequest.new('http://example.com/')
+          request.get
+        }
+        WebMock.should_not have_requested(:get, "http://example.com/")
+      end
+    end    
+    
+    context %w(http://example.com/1) do
+      it 'block is called once' do
+        Cosmicrawler.each(%w(http://example.com/1)) {|item|
+          request = EM::HttpRequest.new(item)
+          request.get
+        
+        }
+        WebMock.should have_requested(:get, "http://example.com/1")
+      end
+    end
+    
+    context %w(http://example.com/1 http://example.com/2) do
+      it 'block is called twice' do
+        Cosmicrawler.each(%w(http://example.com/1 http://example.com/2)) {|item|
+          request = EM::HttpRequest.new(item)
+          request.get
+        }      
+        WebMock.should have_requested(:get, "http://example.com/1")
+        WebMock.should have_requested(:get, "http://example.com/2")
+      end
+    end
+  end
+  
+  describe :http_crawl do
+    it 'block is required' do
+      lambda {Cosmicrawler.http_crawl(%w())}.should raise_error
+    end
+
+    context %w() do
+      it 'block is not called' do
+        Cosmicrawler.http_crawl(%w()) {|request|
+          EM::HttpRequest.new('http://example.com/').get
+        }
+        WebMock.should_not have_requested(:get, "http://example.com/")
+      end
+    end    
+    
+    context %w(http://example.com/1) do
+      it 'block is called once' do
+        Cosmicrawler.http_crawl(%w(http://example.com/1)) {|request|
+          request.get
+        
+        }
+        WebMock.should have_requested(:get, "http://example.com/1")
+      end
+    end
+    
+    context %w(http://example.com/1 http://example.com/2) do
+      it 'block is called twice' do
+        Cosmicrawler.http_crawl(%w(http://example.com/1 http://example.com/2)) {|request|
+          request.get
+        }      
+        WebMock.should have_requested(:get, "http://example.com/1")
+        WebMock.should have_requested(:get, "http://example.com/2")
+      end
+    end
+
+    context 'URI' do
+      it do
+        urls = %w(http://example.com/1 http://example.com/2).map {|str| URI.parse str}
+        
+        Cosmicrawler.http_crawl(urls) {|request|
+          request.get
+        }      
+        WebMock.should have_requested(:get, "http://example.com/1")
+        WebMock.should have_requested(:get, "http://example.com/2")
+      end
+    end
+  end  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+require 'rubygems'
+require 'pry'
+require 'webmock/rspec'

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: cosmicrawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Toshiaki Koshiba
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-synchrony
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Cosmicrawler is crawler library for Ruby. It provides scalable asynchronous
+  crawling by (http|file|etc) using EventMachine.
+email:
+- koshiba+github@4038nullpointer.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- README.md
+- Rakefile
+- cosmicrawler.gemspec
+- lib/cosmicrawler.rb
+- lib/cosmicrawler/em.rb
+- lib/cosmicrawler/em/crawler.rb
+- lib/cosmicrawler/em/http_crawler.rb
+- lib/cosmicrawler/version.rb
+- spec/cosmicrawler_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/bash0C7/cosmicrawler
+licenses:
+- Ruby
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0
+signing_key: 
+specification_version: 4
+summary: Cosmicrawler is crawler library for Ruby. It provides scalable asynchronous
+  crawling by (http|file|etc) using EventMachine.
+test_files:
+- spec/cosmicrawler_spec.rb
+- spec/spec_helper.rb