spidr_epg 1.0.0

data/spec/spidr_spec.rb

@@ -0,0 +1,39 @@
+require 'spidr'
+
+require 'spec_helper'
+
+describe Spidr do
+  it "should have a VERSION constant" do
+    subject.const_defined?('VERSION').should == true
+  end
+
+  describe "proxy" do
+    after(:all) do
+      subject.disable_proxy!
+    end
+
+    it "should not have proxy settings by default" do
+      subject.proxy[:host].should be_nil
+    end
+
+    it "should allow setting new proxy settings" do
+      subject.proxy = {:host => 'example.com', :port => 8010}
+
+      subject.proxy[:host].should == 'example.com'
+      subject.proxy[:port].should == 8010
+    end
+
+    it "should default the :port option of new proxy settings" do
+      subject.proxy = {:host => 'example.com'}
+
+      subject.proxy[:host].should == 'example.com'
+      subject.proxy[:port].should == Spidr::COMMON_PROXY_PORT
+    end
+
+    it "should allow disabling the proxy" do
+      subject.disable_proxy!
+
+      subject.proxy[:host].should be_nil
+    end
+  end
+end

data/spidr.gemspec

@@ -0,0 +1,133 @@
+# encoding: utf-8
+
+require 'yaml'
+
+Gem::Specification.new do |gemspec|
+  root = File.dirname(__FILE__)
+  lib_dir = File.join(root,'lib')
+  files = if File.directory?('.git')
+            `git ls-files`.split($/)
+          elsif File.directory?('.hg')
+            `hg manifest`.split($/)
+          elsif File.directory?('.svn')
+            `svn ls -R`.split($/).select { |path| File.file?(path) }
+          else
+            Dir['{**/}{.*,*}'].select { |path| File.file?(path) }
+          end
+
+  filter_files = lambda { |paths|
+    case paths
+    when Array
+      (files & paths)
+    when String
+      (files & Dir[paths])
+    end
+  }
+
+  version = {
+    :file => 'spidr/version',
+    :constant => 'Spidr::VERSION'
+  }
+
+  defaults = {
+   # 'name' => File.basename(root),
+    'name' => 'spidr_epg',
+    'files' => files,
+    'executables' => filter_files['bin/*'].map { |path| File.basename(path) },
+    'test_files' => filter_files['{test/{**/}*_test.rb,spec/{**/}*_spec.rb}'],
+    'extra_doc_files' => filter_files['*.{txt,rdoc,md,markdown,tt,textile}'],
+  }
+
+  metadata = defaults.merge(YAML.load_file('gemspec.yml'))
+
+  #gemspec.name = metadata.fetch('name',defaults[:name])
+  gemspec.name = 'spidr_epg'
+  gemspec.version = if metadata['version']
+                      metadata['version']
+                    else
+                      $LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir)
+
+                      require version[:file]
+                      eval(version[:constant])
+                    end
+
+  gemspec.summary = metadata.fetch('summary',metadata['description'])
+  gemspec.description = metadata.fetch('description',metadata['summary'])
+
+  case metadata['license']
+  when Array
+    gemspec.licenses = metadata['license']
+  when String
+    gemspec.license = metadata['license']
+  end
+
+  case metadata['authors']
+  when Array
+    gemspec.authors = metadata['authors']
+  when String
+    gemspec.author = metadata['authors']
+  end
+
+  gemspec.email = metadata['email']
+  gemspec.homepage = metadata['homepage']
+
+  case metadata['require_paths']
+  when Array
+    gemspec.require_paths = metadata['require_paths']
+  when String
+    gemspec.require_path = metadata['require_paths']
+  end
+
+  gemspec.files = filter_files[metadata['files']]
+
+  gemspec.executables = metadata['executables']
+  gemspec.extensions = metadata['extensions']
+
+  if Gem::VERSION < '1.7.'
+    gemspec.default_executable = gemspec.executables.first
+  end
+
+  gemspec.test_files = filter_files[metadata['test_files']]
+
+  unless gemspec.files.include?('.document')
+    gemspec.extra_rdoc_files = metadata['extra_doc_files']
+  end
+
+  gemspec.post_install_message = metadata['post_install_message']
+  gemspec.requirements = metadata['requirements']
+
+  if gemspec.respond_to?(:required_ruby_version=)
+    gemspec.required_ruby_version = metadata['required_ruby_version']
+  end
+
+  if gemspec.respond_to?(:required_rubygems_version=)
+    gemspec.required_rubygems_version = metadata['required_rubygems_version']
+  end
+
+  parse_versions = lambda { |versions|
+    case versions
+    when Array
+      versions.map { |v| v.to_s }
+    when String
+      versions.split(/,\s*/)
+    end
+  }
+
+  if metadata['dependencies']
+    metadata['dependencies'].each do |name,versions|
+      gemspec.add_dependency(name,parse_versions[versions])  
+    end
+  end
+
+  if metadata['runtime_dependencies']
+    metadata['runtime_dependencies'].each do |name,versions|
+      gemspec.add_runtime_dependency(name,parse_versions[versions])  
+    end
+  end
+
+  if metadata['development_dependencies']
+    metadata['development_dependencies'].each do |name,versions|
+      gemspec.add_development_dependency(name,parse_versions[versions])  
+    end
+  end
+end

data/spidr.gemspec~

@@ -0,0 +1,131 @@
+# encoding: utf-8
+
+require 'yaml'
+
+Gem::Specification.new do |gemspec|
+  root = File.dirname(__FILE__)
+  lib_dir = File.join(root,'lib')
+  files = if File.directory?('.git')
+            `git ls-files`.split($/)
+          elsif File.directory?('.hg')
+            `hg manifest`.split($/)
+          elsif File.directory?('.svn')
+            `svn ls -R`.split($/).select { |path| File.file?(path) }
+          else
+            Dir['{**/}{.*,*}'].select { |path| File.file?(path) }
+          end
+
+  filter_files = lambda { |paths|
+    case paths
+    when Array
+      (files & paths)
+    when String
+      (files & Dir[paths])
+    end
+  }
+
+  version = {
+    :file => 'spidr/version',
+    :constant => 'Spidr::VERSION'
+  }
+
+  defaults = {
+    'name' => File.basename(root),
+    'files' => files,
+    'executables' => filter_files['bin/*'].map { |path| File.basename(path) },
+    'test_files' => filter_files['{test/{**/}*_test.rb,spec/{**/}*_spec.rb}'],
+    'extra_doc_files' => filter_files['*.{txt,rdoc,md,markdown,tt,textile}'],
+  }
+
+  metadata = defaults.merge(YAML.load_file('gemspec.yml'))
+
+  gemspec.name = metadata.fetch('name',defaults[:name])
+  gemspec.version = if metadata['version']
+                      metadata['version']
+                    else
+                      $LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir)
+
+                      require version[:file]
+                      eval(version[:constant])
+                    end
+
+  gemspec.summary = metadata.fetch('summary',metadata['description'])
+  gemspec.description = metadata.fetch('description',metadata['summary'])
+
+  case metadata['license']
+  when Array
+    gemspec.licenses = metadata['license']
+  when String
+    gemspec.license = metadata['license']
+  end
+
+  case metadata['authors']
+  when Array
+    gemspec.authors = metadata['authors']
+  when String
+    gemspec.author = metadata['authors']
+  end
+
+  gemspec.email = metadata['email']
+  gemspec.homepage = metadata['homepage']
+
+  case metadata['require_paths']
+  when Array
+    gemspec.require_paths = metadata['require_paths']
+  when String
+    gemspec.require_path = metadata['require_paths']
+  end
+
+  gemspec.files = filter_files[metadata['files']]
+
+  gemspec.executables = metadata['executables']
+  gemspec.extensions = metadata['extensions']
+
+  if Gem::VERSION < '1.7.'
+    gemspec.default_executable = gemspec.executables.first
+  end
+
+  gemspec.test_files = filter_files[metadata['test_files']]
+
+  unless gemspec.files.include?('.document')
+    gemspec.extra_rdoc_files = metadata['extra_doc_files']
+  end
+
+  gemspec.post_install_message = metadata['post_install_message']
+  gemspec.requirements = metadata['requirements']
+
+  if gemspec.respond_to?(:required_ruby_version=)
+    gemspec.required_ruby_version = metadata['required_ruby_version']
+  end
+
+  if gemspec.respond_to?(:required_rubygems_version=)
+    gemspec.required_rubygems_version = metadata['required_rubygems_version']
+  end
+
+  parse_versions = lambda { |versions|
+    case versions
+    when Array
+      versions.map { |v| v.to_s }
+    when String
+      versions.split(/,\s*/)
+    end
+  }
+
+  if metadata['dependencies']
+    metadata['dependencies'].each do |name,versions|
+      gemspec.add_dependency(name,parse_versions[versions])  
+    end
+  end
+
+  if metadata['runtime_dependencies']
+    metadata['runtime_dependencies'].each do |name,versions|
+      gemspec.add_runtime_dependency(name,parse_versions[versions])  
+    end
+  end
+
+  if metadata['development_dependencies']
+    metadata['development_dependencies'].each do |name,versions|
+      gemspec.add_development_dependency(name,parse_versions[versions])  
+    end
+  end
+end

metadata

@@ -0,0 +1,158 @@
+--- !ruby/object:Gem::Specification
+name: spidr_epg
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Postmodern
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+description: Spidr is a versatile Ruby web spidering library that can spider a site,
+  multiple domains, certain links or infinitely. Spidr is designed to be fast and
+  easy to use.
+email: postmodern.mod3@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+- ChangeLog.md
+files:
+- .rspec
+- .gitignore
+- .yardopts
+- LICENSE.txt
+- Rakefile
+- Gemfile.lock
+- ChangeLog.md~
+- lib/spidr.rb
+- lib/spidr/filters.rb
+- lib/spidr/actions.rb
+- lib/spidr/rules.rb
+- lib/spidr/links.rb
+- lib/spidr/body.rb
+- lib/spidr/spidr.rb
+- lib/spidr/session_cache.rb
+- lib/spidr/extensions.rb
+- lib/spidr/agent.rb
+- lib/spidr/auth_store.rb
+- lib/spidr/auth_credential.rb
+- lib/spidr/page.rb
+- lib/spidr/version.rb~
+- lib/spidr/actions/actions.rb
+- lib/spidr/actions/exceptions.rb
+- lib/spidr/actions/exceptions/skip_link.rb
+- lib/spidr/actions/exceptions/paused.rb
+- lib/spidr/actions/exceptions/action.rb
+- lib/spidr/actions/exceptions/skip_page.rb
+- lib/spidr/headers.rb
+- lib/spidr/version.rb
+- lib/spidr/sanitizers.rb
+- lib/spidr/cookie_jar.rb
+- lib/spidr/extensions/uri.rb
+- lib/spidr/events.rb
+- spidr.gemspec~
+- gemspec.yml
+- Gemfile~
+- spidr.gemspec
+- Gemfile
+- README.md~
+- README.md
+- pkg/spidr-1.0.0.gem
+- ChangeLog.md
+- spec/filters_spec.rb
+- spec/agent_spec.rb
+- spec/spec_helper.rb
+- spec/cookie_jar_spec.rb
+- spec/auth_store_spec.rb
+- spec/spidr_spec.rb
+- spec/session_cache.rb
+- spec/page_spec.rb
+- spec/page_examples.rb
+- spec/actions_spec.rb
+- spec/helpers/history.rb
+- spec/helpers/wsoc.rb
+- spec/helpers/page.rb
+- spec/rules_spec.rb
+- spec/sanitizers_spec.rb
+- spec/extensions/uri_spec.rb
+homepage: http://github.com/postmodern/spidr
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0
+signing_key: 
+specification_version: 4
+summary: A versatile Ruby web spidering library
+test_files:
+- spec/filters_spec.rb
+- spec/agent_spec.rb
+- spec/cookie_jar_spec.rb
+- spec/auth_store_spec.rb
+- spec/spidr_spec.rb
+- spec/page_spec.rb
+- spec/actions_spec.rb
+- spec/rules_spec.rb
+- spec/sanitizers_spec.rb
+- spec/extensions/uri_spec.rb