strawman 0.2 → 0.3

data/README.rdoc

@@ -16,25 +16,42 @@ RDoc: http://rdoc.info/projects/MattColyer/strawman
   require 'eventmachine'
   require 'em-http'
   require 'strawman'
+  require 'logger'
+
+  log = Logger.new(STDOUT)
+  log.level = Logger::INFO
 
   EventMachine.run {
-    proxy_list = Strawman::ProxyList.new
-    sources_set = proxy_list.set_sources([Strawman::TwitterSource.new("proxy_sites")])
+    proxy_list = Strawman::ProxyList.new("http://whatismyip.org")
+    proxy_list.load("proxies") if File.exist?("proxies")
+    sources_set = proxy_list.add_sources([Strawman::TwitterSource.new("proxy_sites")])
+
     sources_set.callback{
-      http = Strawman::HttpRequest.new(proxy_list, 'http://goingtorain.com/').get
-      http.callback {
-        p http.response
-        EventMachine.stop
+      proxy_list.save("proxies")
+      request = Strawman::HttpRequest.new(proxy_list, 'http://goingtorain.com/')
+      request.callback {
+        http = request.get
+        http.callback {
+          log.info http.response_header.inspect
+          log.info http.response
+          EventMachine.stop
+        }
+      }
+      request.errback{
+        log.error "No available proxies"
       }
     }
+
+    sources_set.errback{
+      log.error "Something went wrong"
+      EventMachine.stop
+    }
   }
 
 == TODO
 
 Patches happily accepted, please open a github ticket and attach the patch.
 
-* Store proxies across runs, so that the list can grow larger than the status list
-* Optionally, store proxy verification across runs
 * Undo link encoding if Glype adds it
 * Strip annoying stuff that Glype adds as a footer and header
 * POST doesn't work

data/Rakefile

@@ -3,7 +3,7 @@ require 'spec/rake/spectask'
 require 'rake'
 require 'echoe'
 
-Echoe.new('strawman', '0.2') do |p|
+Echoe.new('strawman', '0.3') do |p|
   p.description    = "Allows you fetch pages using glype proxies."
   p.url            = "http://github.com/mattcolyer/strawman"
   p.author         = "Matt Colyer"

data/examples/example.rb

@@ -3,20 +3,29 @@ require 'eventmachine'
 require 'em-http'
 $LOAD_PATH << "../lib/"
 require 'strawman'
+require 'logger'
 
 log = Logger.new(STDOUT)
 log.level = Logger::INFO
 
 EventMachine.run {
   proxy_list = Strawman::ProxyList.new("http://whatismyip.org")
-  sources_set = proxy_list.set_sources([Strawman::TwitterSource.new("proxy_sites")])
+  proxy_list.load("proxies") if File.exist?("proxies")
+  sources_set = proxy_list.add_sources([Strawman::TwitterSource.new("proxy_sites")])
 
   sources_set.callback{
-    http = Strawman::HttpRequest.new(proxy_list, 'http://goingtorain.com/').get
-    http.callback {
-      log.info http.response_header.inspect
-      log.info http.response
-      EventMachine.stop
+    proxy_list.save("proxies")
+    request = Strawman::HttpRequest.new(proxy_list, 'http://goingtorain.com/')
+    request.callback {
+      http = request.get
+      http.callback {
+        log.info http.response_header.inspect
+        log.info http.response
+        EventMachine.stop
+      }
+    }
+    request.errback{
+      log.error "No available proxies"
     }
   }
 

data/lib/strawman/http_request.rb

@@ -4,9 +4,15 @@ module Strawman
   # A simple wrapper for em-http-client's HttpRequest.
   #
   class HttpRequest
+    include EventMachine::Deferrable
+
     def initialize(proxy_list, url)
-      @proxy = proxy_list.proxy
-      @request = Transport.new(@proxy.proxy_url(url))
+      proxy_response = proxy_list.proxy
+      proxy_response.callback do |proxy|
+        @proxy = proxy
+        @request = Transport.new(@proxy.proxy_url(url))
+        succeed
+      end
     end
 
     #

data/lib/strawman/proxy.rb

@@ -4,6 +4,14 @@ module Strawman
   # of proxy.
   #
   class Proxy
+    attr_reader :root_url
+    attr_writer :valid
+
+    def initialize(url)
+      @root_url = url
+      @valid = false
+    end
+
     #
     # Returns the the referer to use when making the proxied request.
     #
@@ -15,7 +23,20 @@ module Strawman
     # Returns the url to fetch the given url through this proxy.
     #
     def proxy_url(url)
-      URI.join @root_url, proxy_path(url)
+      uri = URI.join @root_url, proxy_path(url)
+      "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}?#{uri.query}"
+    end
+
+    #
+    # Used to determine whether this proxy is valid. This must be called from
+    # within the callback of the validate deferable.
+    #
+    def valid?
+      @valid
+    end
+
+    def ==(other)
+      self.class == other.class && self.root_url == other.root_url
     end
 
   protected
@@ -29,11 +50,6 @@ module Strawman
   # See: http://www.glype.com/ for more details.
   #
   class GlypeProxy < Proxy
-    def initialize(url)
-      @root_url = url
-      @valid = false
-    end
-
     #
     # Verifies whether this proxy is currently functional. Returns a deferable.
     #
@@ -52,14 +68,6 @@ module Strawman
       http
     end
 
-    #
-    # Used to determine whether this proxy is valid. This must be called from
-    # within the callback of the validate deferable.
-    #
-    def valid?
-      @valid
-    end
-
     def to_s
       "<GlypeProxy #{@root_url}>"
     end

data/lib/strawman/proxy_list.rb

@@ -3,7 +3,7 @@ module Strawman
   # Represents a group of proxy sources
   #
   class ProxyList
-    attr_reader :proxies
+    attr_accessor :proxies
 
     #
     # [verification_url]  The url to use to verify that the proxy is valid. All
@@ -11,48 +11,75 @@ module Strawman
     #
     def initialize(verification_url)
       @proxies = []
+      @dead_proxies = []
       @verification_url = verification_url
     end
 
     #
     # Takes a list of sources and returns a deferrable which will complete once
-    # all sources have been fetched and all proxies have been verified.
+    # all sources have been fetched.
     #
-    def set_sources(sources)
+    def add_sources(sources)
       sources_ready = EventMachine::MultiRequest.new
-      proxies_ready = EventMachine::MultiRequest.new
 
-      # Fetch all of the sources
       sources.each do |source|
         sources_ready.add(source)
       end
 
-      # Verify all of the proxies
       sources_ready.callback do
         sources.each do |source|
           source.proxies.each do |proxy|
-            proxies_ready.add(proxy.validate(@verification_url))
+            @proxies << proxy unless @proxies.include? proxy
           end
         end
       end
 
-      # Include proxies that are verified
-      proxies_ready.callback do
-        sources.each do |source|
-          source.proxies.each do |proxy|
-            @proxies << proxy if proxy.valid?
-          end
+      sources_ready
+    end
+
+    #
+    # Selects a random proxy from the list of available proxies and verifies
+    # it. If it isn't valid it keeps trying all available proxies before
+    # returning nil.
+    #
+    def proxy(deferrable=nil)
+      deferrable ||= EventMachine::DefaultDeferrable.new
+
+      proxy = @proxies.choice
+      deferrable.fail unless proxy
+
+      proxy_response = proxy.validate(@verification_url)
+      proxy_response.callback do
+        if proxy.valid?
+          deferrable.succeed(proxy)
+        else
+          self.proxy(deferrable)
         end
       end
 
-      proxies_ready
+      proxy_response.errback do
+        @proxies.remove(proxy)
+        @dead_proxies.add(proxy)
+      end
+
+      deferrable
+    end
+
+    #
+    # Saves all proxies that were loaded into this instance, including proxies
+    # with errors.
+    #
+    def save(filepath)
+      File.open(filepath, "w") do |f|
+        f.write((@proxies + @dead_proxies).to_yaml)
+      end
     end
 
     #
-    # Selects a random proxy from the list of available proxies
+    # Loads all proxies from the given file
     #
-    def proxy
-      @proxies.choice
+    def load(filepath)
+      @proxies = YAML.load(File.read(filepath))
     end
   end
 end

data/lib/strawman.rb

@@ -5,3 +5,4 @@ require 'strawman/http_request'
 require 'strawman/proxy'
 require 'strawman/proxy_list'
 require 'strawman/source'
+require 'strawman/transport'

data/strawman.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{strawman}
-  s.version = "0.2"
+  s.version = "0.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matt Colyer"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: strawman
 version: !ruby/object:Gem::Version 
-  version: "0.2"
+  version: "0.3"
 platform: ruby
 authors: 
 - Matt Colyer

metadata.gz.sig

@@ -1 +1,2 @@
-">�Ǥ_�{�����pa�?���8�<��l��㏹�/���}�a���I
+B�¤�a���V�@� Iv�;��ZP*��TB+E�`G`���3�Ճ�P��P��6�̮���F�Kk�uKsj���u�o�I/�UZ=qHP�}��gܪ�-2Z�mHm��<\�I�/)�#ˑ�WHs���a���K�����^�Ct�VeH�`�$��V��Dn���.�)��C�-̍�j,�C��V���,
�
�c�r&�׬�l�)�(e�Å�?)���ុ=��m��9v~c��6
+�wLt»r5C�Ii�s��BI�B@