ruby-crawler 0.0.1

data/test/test_helper.rb

@@ -0,0 +1,50 @@
+require 'rubygems'
+require 'bundler'
+Bundler.require :test
+
+require 'simplecov'
+SimpleCov.start do
+  add_filter "/test/*_test.rb"
+end
+
+require 'redis'
+require 'minitest/autorun'
+require 'webmock/minitest'
+require 'crawler'
+
+# Support files
+Dir["#{File.expand_path(File.dirname(__FILE__))}/support/*.rb"].each do |file|
+  require file
+end
+
+class MiniTest::Test
+  include DomainHtml
+
+  def setup
+    stub_request(:get, "https://www.digitalocean.com/").
+      with(:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}).
+      to_return(:status => 200, :body => root_html , :headers => {})
+
+    stub_request(:get, "https://www.digitalocean.com/redirect").
+      with(:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}).
+      to_return(:status => 301, :body => root_html , :headers => {'Location'=>'https://www.digitalocean.com'})
+
+    stub_request(:get, "https://www.digitalocean.com/bad_request").
+      with(:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}).
+      to_return(:status => 404, :body => root_html , :headers => {})
+
+    stub_request(:get, "https://www.digitalocean.com/pricing/").
+      with(:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}).
+      to_return(:status => 200, :body => pricing_html , :headers => {})
+
+    stub_request(:get, "https://www.digitalocean.com/features/").
+      with(:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}).
+      to_return(:status => 200, :body => features_html , :headers => {})
+
+    stub_request(:get, "https://www.digitalocean.com/company/careers/").
+      with(:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}).
+      to_return(:status => 200, :body => careers_html , :headers => {})
+
+    Redis.new.flushdb
+  end
+end

metadata

@@ -0,0 +1,191 @@
+--- !ruby/object:Gem::Specification
+name: ruby-crawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Adam Ryan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple ruby web crawler
+email:
+- adam.g.ryan@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".DS_Store"
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- crawler.gemspec
+- lib/crawler.rb
+- lib/crawler/document.rb
+- lib/crawler/document_parser.rb
+- lib/crawler/error.rb
+- lib/crawler/formatting.rb
+- lib/crawler/http.rb
+- lib/crawler/index.rb
+- lib/crawler/storage.rb
+- lib/crawler/validations.rb
+- lib/crawler/version.rb
+- test/crawler/crawler_test.rb
+- test/crawler/document_test.rb
+- test/crawler/formatting_test.rb
+- test/crawler/http_test.rb
+- test/crawler/index_test.rb
+- test/crawler/storage_test.rb
+- test/crawler/validations_test.rb
+- test/support/domain_html.rb
+- test/test_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: Simple ruby web crawler
+test_files:
+- test/crawler/crawler_test.rb
+- test/crawler/document_test.rb
+- test/crawler/formatting_test.rb
+- test/crawler/http_test.rb
+- test/crawler/index_test.rb
+- test/crawler/storage_test.rb
+- test/crawler/validations_test.rb
+- test/support/domain_html.rb
+- test/test_helper.rb