strawman 0.1 → 0.2

data/README.rdoc

@@ -3,6 +3,8 @@
 A ruby gem which allows you to proxy EventMachine HTTP GET requests through
 glype proxies on the net.
 
+RDoc: http://rdoc.info/projects/MattColyer/strawman
+
 == Getting started
 
   # make sure you have gemcutter.org repos enabled
@@ -13,9 +15,8 @@ glype proxies on the net.
   require 'rubygems'
   require 'eventmachine'
   require 'em-http'
-  $LOAD_PATH << "../lib/"
   require 'strawman'
-  
+
   EventMachine.run {
     proxy_list = Strawman::ProxyList.new
     sources_set = proxy_list.set_sources([Strawman::TwitterSource.new("proxy_sites")])
@@ -32,12 +33,15 @@ glype proxies on the net.
 
 Patches happily accepted, please open a github ticket and attach the patch.
 
-  * specs need to be implemented
-  * POST doesn't work
-  * Cookies don't work
-  * SSL doesn't work
-  * Implement other sources
+* Store proxies across runs, so that the list can grow larger than the status list
+* Optionally, store proxy verification across runs
+* Undo link encoding if Glype adds it
+* Strip annoying stuff that Glype adds as a footer and header
+* POST doesn't work
+* Cookies don't work
+* SSL doesn't work
+* Implement other proxy sources
 
 == Limitations
 
-  * PUT and DELETE can't work do to the way Glype is implemented
+* PUT and DELETE can't work do to the way Glype is implemented

data/Rakefile

@@ -1,8 +1,9 @@
 require 'rubygems'
+require 'spec/rake/spectask'
 require 'rake'
 require 'echoe'
 
-Echoe.new('strawman', '0.1') do |p|
+Echoe.new('strawman', '0.2') do |p|
   p.description    = "Allows you fetch pages using glype proxies."
   p.url            = "http://github.com/mattcolyer/strawman"
   p.author         = "Matt Colyer"
@@ -11,3 +12,9 @@ Echoe.new('strawman', '0.1') do |p|
   p.development_dependencies = ["rspec"]
   p.dependencies = ["eventmachine", "em-http-request", "json"]
 end
+
+
+desc "Run all examples"
+Spec::Rake::SpecTask.new('tests') do |t|
+  t.spec_files = FileList['spec/*_spec.rb']
+end

data/examples/example.rb

@@ -4,15 +4,24 @@ require 'em-http'
 $LOAD_PATH << "../lib/"
 require 'strawman'
 
+log = Logger.new(STDOUT)
+log.level = Logger::INFO
+
 EventMachine.run {
-  proxy_list = Strawman::ProxyList.new
+  proxy_list = Strawman::ProxyList.new("http://whatismyip.org")
   sources_set = proxy_list.set_sources([Strawman::TwitterSource.new("proxy_sites")])
+
   sources_set.callback{
     http = Strawman::HttpRequest.new(proxy_list, 'http://goingtorain.com/').get
     http.callback {
-      p http.response_headers.inspect
-      p http.response
+      log.info http.response_header.inspect
+      log.info http.response
       EventMachine.stop
     }
   }
+
+  sources_set.errback{
+    log.error "Something went wrong"
+    EventMachine.stop
+  }
 }

data/lib/strawman/http_request.rb

@@ -1,11 +1,18 @@
 module Strawman
+  # = HttpRequest
+  #
+  # A simple wrapper for em-http-client's HttpRequest.
+  #
   class HttpRequest
     def initialize(proxy_list, url)
       @proxy = proxy_list.proxy
-      proxied_url = @proxy.proxy_url(url)
-      @request = EventMachine::HttpRequest.new(proxied_url)
+      @request = Transport.new(@proxy.proxy_url(url))
     end
-    
+
+    #
+    # Handles get requests. Currently it accepts no arguments (ie query
+    # parameters, http headers etc...).
+    #
     def get
       http = @request.get :head => {"referer" => @proxy.referer}
       http.callback {
@@ -14,5 +21,33 @@ module Strawman
       }
       http
     end
+
+    #
+    # TODO: Implement this.
+    #
+    def post
+      raise NotImplementedError
+    end
+
+    #
+    # Can't and won't be implemented due to Glype not proxying these requests.
+    #
+    def put
+      raise NotImplementedError
+    end
+
+    #
+    # Can't and won't be implemented due to Glype not proxying these requests.
+    #
+    def delete
+      raise NotImplementedError
+    end
+
+    #
+    # Can't and won't be implemented due to Glype not proxying these requests.
+    #
+    def head
+      raise NotImplementedError
+    end
   end
 end

data/lib/strawman/proxy.rb

@@ -1,23 +1,50 @@
 module Strawman
+  #
+  # The general proxy class, which contains functions not specific to any type
+  # of proxy.
+  #
   class Proxy
+    #
+    # Returns the the referer to use when making the proxied request.
+    #
+    def referer
+      @root_url
+    end
+
+    #
+    # Returns the url to fetch the given url through this proxy.
+    #
+    def proxy_url(url)
+      URI.join @root_url, proxy_path(url)
+    end
+
+  protected
+    def proxy_path(url)
+      raise NotImplementedError
+    end
   end
 
+  #
+  # An implementation of the Proxy class which is specific to Glype proxies.
+  # See: http://www.glype.com/ for more details.
+  #
   class GlypeProxy < Proxy
     def initialize(url)
       @root_url = url
       @valid = false
     end
 
-    def valid?
-      @valid
-    end
+    #
+    # Verifies whether this proxy is currently functional. Returns a deferable.
+    #
+    def validate(verification_url)
+      @valid = false
 
-    def validate
-      url = proxy_url("http://whatismyip.org")
       # FIXME: This only validate proxies that don't require a unique session
       # cookie which is retrieved by going to the root page and looking for the
       # s cookie.
-      http = EventMachine::HttpRequest.new(url).get :head => {'referer' => @root_url}
+      url = proxy_url(verification_url)
+      http = Transport.new(url).get :head => {'referer' => @root_url}
       http.callback {
         @valid = true if http.response_header.status == 200
       }
@@ -25,27 +52,22 @@ module Strawman
       http
     end
 
-    def to_s
-      "<GlypeProxy #{@root_url}>"
-    end
-
-    def referer
-      @root_url
+    #
+    # Used to determine whether this proxy is valid. This must be called from
+    # within the callback of the validate deferable.
+    #
+    def valid?
+      @valid
     end
 
-    def proxy_url(url)
-      URI.join @root_url, proxy_path(url)
+    def to_s
+      "<GlypeProxy #{@root_url}>"
     end
 
+protected
     def proxy_path(url)
       encoded_url = CGI.escape(Base64.encode64(url[4..-1]))
       "/browse.php?u=#{encoded_url}&f=norefer"
     end
   end
-  
-  class PhpProxy < Proxy
-    def valid?
-      false
-    end
-  end
 end

data/lib/strawman/proxy_list.rb

@@ -1,25 +1,42 @@
 module Strawman
+  #
+  # Represents a group of proxy sources
+  #
   class ProxyList
-    def initialize
+    attr_reader :proxies
+
+    #
+    # [verification_url]  The url to use to verify that the proxy is valid. All
+    #                     it needs to do is return an HTTP status of 200.
+    #
+    def initialize(verification_url)
       @proxies = []
+      @verification_url = verification_url
     end
 
+    #
+    # Takes a list of sources and returns a deferrable which will complete once
+    # all sources have been fetched and all proxies have been verified.
+    #
     def set_sources(sources)
       sources_ready = EventMachine::MultiRequest.new
+      proxies_ready = EventMachine::MultiRequest.new
 
+      # Fetch all of the sources
       sources.each do |source|
         sources_ready.add(source)
       end
 
-      proxies_ready = EventMachine::MultiRequest.new
+      # Verify all of the proxies
       sources_ready.callback do
         sources.each do |source|
           source.proxies.each do |proxy|
-            proxies_ready.add(proxy.validate)
+            proxies_ready.add(proxy.validate(@verification_url))
           end
         end
       end
 
+      # Include proxies that are verified
       proxies_ready.callback do
         sources.each do |source|
           source.proxies.each do |proxy|
@@ -31,8 +48,11 @@ module Strawman
       proxies_ready
     end
 
+    #
+    # Selects a random proxy from the list of available proxies
+    #
     def proxy
-      @proxies.choice 
+      @proxies.choice
     end
   end
 end

data/lib/strawman/source.rb

@@ -2,35 +2,68 @@ module Strawman
   class Source
   end
 
+  #
+  # A source that parses a twitter feed for urls which points to proxies, like
+  # http://twitter.com/proxy_lists. The class is deferable itself and fires
+  # its callback once the feed has been fetched and parsed.
+  #
+  # By default it caches the feed to disk (cache/twitter-username.json), and
+  # fetches a new copy after that file is an hour old. To disable this, simply
+  # pass in false to the constructor.
+  #
   class TwitterSource < Source
     include EventMachine::Deferrable
     ONE_HOUR = 60*60
     attr_reader :proxies
 
-    def initialize(twitter_username)
+    #
+    # [twitter_username]  Just the twitter user's username. Not the full url.
+    #
+    # [cache]  Whether to enable or disable caching. Defaults to enabling
+    #          caching.
+    #
+    def initialize(twitter_username, cache=true)
       @id = twitter_username
+      @cache = cache
+
+      if cache
+        fetched = update_cache
+      else
+        fetched = fetch
+      end
 
-      fetched = update_cache
       fetched.callback do
-        @proxies = JSON.parse(read_cache).map do |status|
-          match = /.*(http:\/\/.*)/.match(status["text"])
-          if match
-            GlypeProxy.new(match[1])
-          else
-            nil
-          end
-        end.compact
-        set_deferred_status :succeeded
+        if @cache
+          data = read_cache
+        else
+          data = fetched.response
+        end
+
+        @proxies = parse(data)
+
+        succeed
       end
     end
 
   private
+    def parse(data)
+      JSON.parse(data).map do |status|
+        match = /.*(http:\/\/.*)/.match(status["text"])
+
+        if match
+          GlypeProxy.new(match[1])
+        else
+          nil
+        end
+      end.compact
+    end
+
     def cache_dir
       "cache"
     end
 
     def cache_file_path
-      File.join cache_dir, "#{@id}.json"
+      File.join cache_dir, "twitter-#{@id}.json"
     end
 
     def cache_file_url
@@ -55,14 +88,16 @@ module Strawman
         fetch
       end
     end
-    
+
     def fetch
-      http = EventMachine::HttpRequest.new(cache_file_url).get
-      
-      http.callback do
-        FileUtils.mkdir(cache_dir) unless File.exist? cache_dir
-        open(cache_file_path, "w") do |f|
-          f.write(http.response)
+      http = Transport.new(cache_file_url).get
+
+      if @cache
+        http.callback do
+          FileUtils.mkdir(cache_dir) unless File.exist? cache_dir
+          open(cache_file_path, "w") do |f|
+            f.write(http.response)
+          end
         end
       end
 

data/strawman.gemspec

@@ -2,12 +2,12 @@
 
 Gem::Specification.new do |s|
   s.name = %q{strawman}
-  s.version = "0.1"
+  s.version = "0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matt Colyer"]
   s.cert_chain = ["/home/mcolyer/.ssh/gem-public_cert.pem"]
-  s.date = %q{2010-02-13}
+  s.date = %q{2010-02-19}
   s.description = %q{Allows you fetch pages using glype proxies.}
   s.email = %q{matt @nospam@ colyer.name}
   s.extra_rdoc_files = ["README.rdoc", "lib/strawman.rb", "lib/strawman/http_request.rb", "lib/strawman/proxy.rb", "lib/strawman/proxy_list.rb", "lib/strawman/source.rb"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: strawman
 version: !ruby/object:Gem::Version 
-  version: "0.1"
+  version: "0.2"
 platform: ruby
 authors: 
 - Matt Colyer
@@ -30,7 +30,7 @@ cert_chain:
   anU=
   -----END CERTIFICATE-----
 
-date: 2010-02-13 00:00:00 -08:00
+date: 2010-02-19 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency