ronin-web 0.3.0.pre2 → 0.3.0.rc1

data/ronin-web.gemspec

@@ -1,15 +1,131 @@
-# -*- encoding: utf-8 -*-
-
-begin
-  Ore::Specification.new do |gemspec|
-    # custom logic here
-  end
-rescue NameError
-  begin
-    require 'ore/specification'
-    retry
-  rescue LoadError
-    STDERR.puts "The '#{__FILE__}' file requires Ore."
-    STDERR.puts "Run `gem install ore-core` to install Ore."
+# encoding: utf-8
+
+require 'yaml'
+
+Gem::Specification.new do |gemspec|
+  root = File.dirname(__FILE__)
+  lib_dir = File.join(root,'lib')
+  files = if File.directory?('.git')
+            `git ls-files`.split($/)
+          elsif File.directory?('.hg')
+            `hg manifest`.split($/)
+          elsif File.directory?('.svn')
+            `svn ls -R`.split($/).select { |path| File.file?(path) }
+          else
+            Dir['{**/}{.*,*}'].select { |path| File.file?(path) }
+          end
+
+  filter_files = lambda { |paths|
+    case paths
+    when Array
+      (files & paths)
+    when String
+      (files & Dir[paths])
+    end
+  }
+
+  version = {
+    :file => 'ronin/web/version',
+    :constant => 'Ronin::Web::VERSION'
+  }
+
+  defaults = {
+    'name' => File.basename(root),
+    'files' => files,
+    'executables' => filter_files['bin/*'].map { |path| File.basename(path) },
+    'test_files' => filter_files['{test/{**/}*_test.rb,spec/{**/}*_spec.rb}'],
+    'extra_doc_files' => filter_files['*.{txt,rdoc,md,markdown,tt,textile}'],
+  }
+
+  metadata = defaults.merge(YAML.load_file('gemspec.yml'))
+
+  gemspec.name = metadata.fetch('name',defaults[:name])
+  gemspec.version = if metadata['version']
+                      metadata['version']
+                    else
+                      $LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir)
+
+                      require version[:file]
+                      eval(version[:constant])
+                    end
+
+  gemspec.summary = metadata.fetch('summary',metadata['description'])
+  gemspec.description = metadata.fetch('description',metadata['summary'])
+
+  case metadata['license']
+  when Array
+    gemspec.licenses = metadata['license']
+  when String
+    gemspec.license = metadata['license']
+  end
+
+  case metadata['authors']
+  when Array
+    gemspec.authors = metadata['authors']
+  when String
+    gemspec.author = metadata['authors']
+  end
+
+  gemspec.email = metadata['email']
+  gemspec.homepage = metadata['homepage']
+
+  case metadata['require_paths']
+  when Array
+    gemspec.require_paths = metadata['require_paths']
+  when String
+    gemspec.require_path = metadata['require_paths']
+  end
+
+  gemspec.files = filter_files[metadata['files']]
+
+  gemspec.executables = metadata['executables']
+  gemspec.extensions = metadata['extensions']
+
+  if Gem::VERSION < '1.7.'
+    gemspec.default_executable = gemspec.executables.first
+  end
+
+  gemspec.test_files = filter_files[metadata['test_files']]
+
+  unless gemspec.files.include?('.document')
+    gemspec.extra_rdoc_files = metadata['extra_doc_files']
+  end
+
+  gemspec.post_install_message = metadata['post_install_message']
+  gemspec.requirements = metadata['requirements']
+
+  if gemspec.respond_to?(:required_ruby_version=)
+    gemspec.required_ruby_version = metadata['required_ruby_version']
+  end
+
+  if gemspec.respond_to?(:required_rubygems_version=)
+    gemspec.required_rubygems_version = metadata['required_ruby_version']
+  end
+
+  parse_versions = lambda { |versions|
+    case versions
+    when Array
+      versions.map { |v| v.to_s }
+    when String
+      versions.split(/,\s*/)
+    end
+  }
+
+  if metadata['dependencies']
+    metadata['dependencies'].each do |name,versions|
+      gemspec.add_dependency(name,parse_versions[versions])  
+    end
+  end
+
+  if metadata['runtime_dependencies']
+    metadata['runtime_dependencies'].each do |name,versions|
+      gemspec.add_runtime_dependency(name,parse_versions[versions])  
+    end
+  end
+
+  if metadata['development_dependencies']
+    metadata['development_dependencies'].each do |name,versions|
+      gemspec.add_development_dependency(name,parse_versions[versions])  
+    end
   end
 end

data/spec/web/helpers/rack_app.rb

@@ -1,11 +1,4 @@
-begin
-  require 'rack/test'
-rescue Gem::LoadError => e
-  raise(e)
-rescue ::LoadError
-  STDERR.puts "Error: please install the rack-test gem in order to run the spec tests"
-  exit -1
-end
+require 'rack/test'
 
 module Helpers
   module Web

data/spec/web/mechanize_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+require 'ronin/web/mechanize'
+
+describe Web::Mechanize do
+  describe "#initialize" do
+    describe ":user_agent" do
+      before(:all) do
+        Web.user_agent = 'test'
+      end
+
+      it "should default to Web.user_agent" do
+        described_class.new.user_agent.should == 'test'
+      end
+
+      it "should support using a custom User-Agent string" do
+        agent = described_class.new(:user_agent => 'test2')
+
+        agent.user_agent.should == 'test2'
+      end
+
+      it "should support using a custom User-Agent alias" do
+        agent = described_class.new(:user_agent_alias => 'iPhone')
+
+        agent.user_agent.should == "Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3"
+      end
+
+      after(:all) do
+        Web.user_agent = nil
+      end
+    end
+
+    describe ":proxy" do
+      let(:host) { '127.0.0.1' }
+      let(:port) { 8080 }
+      let(:proxy) {
+        Network::HTTP::Proxy.new(:host => host, :port => port)
+      }
+
+      before(:all) do
+        Web.proxy = {:host => 'www.example.com', :port => port}
+      end
+
+      it "should default to Web.proxy" do
+        agent = described_class.new
+
+        agent.proxy_addr.should == Web.proxy.host
+        agent.proxy_port.should == Web.proxy.port
+      end
+
+      it "should support using custom proxies" do
+        agent = described_class.new(:proxy => proxy)
+
+        agent.proxy_addr.should == host
+        agent.proxy_port.should == port
+      end
+
+      after(:all) do
+        Web.proxy = nil
+      end
+    end
+  end
+end

data/spec/web/user_agents_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+require 'ronin/web/user_agents'
+
+describe Web::UserAgents do
+  it "should list the categories of User-Agent strings" do
+    subject.categories.should_not be_empty
+  end
+
+  describe "#[]" do
+    context "with Symbol" do
+      it "should select User-Agent strings by group name" do
+        subject[:ie].should_not be_nil
+      end
+
+      it "should return nil if the group exists" do
+        subject[:foobarbaz].should be_nil
+      end
+    end
+
+    context "with String" do
+      it "should select User-Agent strings by substring" do
+        subject['MSIE'].should_not be_nil
+      end
+
+      it "should return nil if no User-Agent matches the substring" do
+        subject['FooBarBaz'].should be_nil
+      end
+    end
+
+    context "with Regexp" do
+      it "should select User-Agent strings by Regexp" do
+        subject[/AppleWebKit/i].should_not be_nil
+      end
+
+      it "should return nil if no User-Agent matches the Regexp" do
+        subject[/FooBarBaz/i].should be_nil
+      end
+    end
+  end
+
+  describe "#fetch" do
+    it "should fetch a User-Agent string" do
+      subject.fetch(:ie).should_not be_nil
+    end
+
+    it "should raise an ArgumentError if no match was found" do
+      lambda {
+        subject.fetch(:foobarbaz)
+      }.should raise_error(ArgumentError)
+    end
+
+    it "should return the default value if no match was found" do
+      subject.fetch(:foobarbaz,'default').should == 'default'
+    end
+  end
+end

data/spec/web/web_spec.rb

@@ -81,64 +81,8 @@ describe Web do
   end
 
   describe "agent" do
-    it "should provide Mechanize agents" do
-      Web.agent.class.should == Mechanize
-    end
-
-    describe ":user_agent" do
-      before(:all) do
-        Web.user_agent = 'test'
-      end
-
-      it "should default to Web.user_agent" do
-        Web.agent.user_agent.should == 'test'
-      end
-
-      it "should support using a custom User-Agent string" do
-        agent = Web.agent(:user_agent => 'test2')
-
-        agent.user_agent.should == 'test2'
-      end
-
-      it "should support using a custom User-Agent alias" do
-        agent = Web.agent(:user_agent_alias => 'iPhone')
-
-        agent.user_agent.should == "Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3"
-      end
-
-      after(:all) do
-        Web.user_agent = nil
-      end
-    end
-
-    describe ":proxy" do
-      let(:host) { '127.0.0.1' }
-      let(:port) { 8080 }
-
-      before(:all) do
-        Web.proxy = {:host => 'www.example.com', :port => port}
-      end
-
-      it "should default to Web.proxy" do
-        agent = Web.agent
-
-        agent.proxy_addr.should == Web.proxy.host
-        agent.proxy_port.should == Web.proxy.port
-      end
-
-      it "should support using custom proxies" do
-        agent = Web.agent(:proxy => Network::HTTP::Proxy.new(
-          :host => host,
-          :port => port
-        ))
-
-        agent.proxy_addr.should == host
-        agent.proxy_port.should == port
-      end
-
-      after(:all) do
-        Web.proxy = nil
-      end
+    it "should be persistent" do
+      Web.agent.object_id.should == Web.agent.object_id
     end
   end
 

metadata

@@ -2,7 +2,7 @@
 name: ronin-web
 version: !ruby/object:Gem::Version 
   prerelease: 6
-  version: 0.3.0.pre2
+  version: 0.3.0.rc1
 platform: ruby
 authors: 
 - Postmodern
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-07 00:00:00 Z
+date: 2011-07-08 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: nokogiri
@@ -74,7 +74,7 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: 0.3.0.rc1
+        version: "0.3"
   type: :runtime
   prerelease: false
   version_requirements: *id006
@@ -85,7 +85,7 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: 0.2.0.rc2
+        version: "0.2"
   type: :runtime
   prerelease: false
   version_requirements: *id007
@@ -96,7 +96,7 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: 1.1.0.rc3
+        version: "1.1"
   type: :runtime
   prerelease: false
   version_requirements: *id008
@@ -144,9 +144,11 @@ files:
 - README.md
 - Rakefile
 - bin/ronin-web
+- data/ronin/web/user_agents.yml
 - gemspec.yml
 - lib/ronin/network/mixins/web.rb
 - lib/ronin/web.rb
+- lib/ronin/web/config.rb
 - lib/ronin/web/extensions.rb
 - lib/ronin/web/extensions/nokogiri.rb
 - lib/ronin/web/extensions/nokogiri/xml.rb
@@ -155,6 +157,7 @@ files:
 - lib/ronin/web/extensions/nokogiri/xml/element.rb
 - lib/ronin/web/extensions/nokogiri/xml/node.rb
 - lib/ronin/web/extensions/nokogiri/xml/text.rb
+- lib/ronin/web/mechanize.rb
 - lib/ronin/web/middleware.rb
 - lib/ronin/web/middleware/base.rb
 - lib/ronin/web/middleware/directories.rb
@@ -182,6 +185,7 @@ files:
 - lib/ronin/web/server/base.rb
 - lib/ronin/web/server/web.rb
 - lib/ronin/web/spider.rb
+- lib/ronin/web/user_agents.rb
 - lib/ronin/web/version.rb
 - lib/ronin/web/web.rb
 - ronin-web.gemspec
@@ -197,6 +201,7 @@ files:
 - spec/web/helpers/root/test2/test2.txt
 - spec/web/helpers/root/test3.txt
 - spec/web/helpers/root/test3/test3.txt
+- spec/web/mechanize_spec.rb
 - spec/web/middleware/directories_spec.rb
 - spec/web/middleware/files_spec.rb
 - spec/web/middleware/filters/campaign_filter_spec.rb
@@ -215,6 +220,7 @@ files:
 - spec/web/server/classes/public2/static2.txt
 - spec/web/server/classes/sub_app.rb
 - spec/web/server/classes/test_app.rb
+- spec/web/user_agents_spec.rb
 - spec/web/web_spec.rb
 homepage: http://github.com/ronin-ruby/ronin-web
 licenses: 
@@ -239,7 +245,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: ronin-web
-rubygems_version: 1.8.1
+rubygems_version: 1.8.5
 signing_key: 
 specification_version: 3
 summary: A Ruby library for Ronin that provides support for web scraping and spidering functionality.
@@ -259,4 +265,6 @@ test_files:
 - spec/web/middleware/response_spec.rb
 - spec/web/middleware/proxy_spec.rb
 - spec/web/extensions/nokogiri_spec.rb
+- spec/web/user_agents_spec.rb
+- spec/web/mechanize_spec.rb
 - spec/web/web_spec.rb