proxtopus 0.0.3 → 0.0.31

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZDVhNGQ1MWZjZDEzNTkwYmI4YmRkMTY5ZjkxZGRiMGI0MmI5ZGIwYw==
+    NzBmMDU3N2E2MzY1YmZkM2Q3ZjI0ZWE5OGUxZTQ5MzlmNTg0NDA4Nw==
   data.tar.gz: !binary |-
-    MjkyZTJjNjUwMTdlMWUyYTdhNGJkYzFlNjI4OGJkOWRkOGRlOTJiYQ==
+    M2I3OTMxMzUxYjExMTQwNjJiMWY2MmVjOTdkZWQ2ZDdlYzZhYjBhZQ==
 SHA512:
   metadata.gz: !binary |-
-    YmNjMDJkMTcyZDMxZDE1MDM2ZjRlYmQ2ZDRiZjZhYjhlOWQ0ZTM1NzZhOWU2
-    MzkwMjJhOTU4NDAzZjRjYmIxZGQxYTFjZTVlMTA5ZWI1ZmViZDA0NzFiYmU1
-    OGRlMjUxODc2ZGNmNDdjMDViNzc2NDJmMjcyYjkwYzc5M2YzYmQ=
+    Y2Q5ZGUzZjZlYWRkMjQzNjAwODcwMTZmYTE2ZjM0MDRjMWZlZjgwZDliNDNi
+    MWIxZDY2MGI1M2JhMjdmYTQ2YjhlOWExZTAzYThlM2RlYTc5OWZkZmRlZDk0
+    NWIzOWQzMDhhYjViOGUxODgwOGVhMjViOTJjNmRkYmYyYmUxZDE=
   data.tar.gz: !binary |-
-    NDk1ODNhN2ZjZWZhMGU2NTBmZDg4Mjc5NjhhMjUyMTA5NzVlNzU4ZGE0N2Nh
-    NzljOWZhZjc1MGZlOWUwMjkwYjM3MGY0OGIwYjBiZTA3OWVmZWZjNmFmMDhh
-    MjRlYzMxMzY5ZGI3YmUzMDgxOTg0YTNhYjE1NDViMTZlZWRkZTM=
+    NDk5MTQyMGQ3M2ZhYjVmZjQ3MDZmMjcyZDJhYjkxMmU1ZGE0NzllODc3MDcx
+    ZGUwMWZlNWRkMzY4MDI4NzVjMjBlY2NhZTMzMGZjMmRkMjZiMWZiZjRjM2Jk
+    NWYzODgwOWQ1MTcyNTNlNWNmODkzZTUyNmMwMzEyYjhjYmY1NjE=

data/lib/proxtopus/client.rb

@@ -1,8 +1,7 @@
 module Proxtopus
   
   class Client
-    attr_accessor :api_url, :api_options
-    attr_reader :proxies
+    attr_accessor :resources, :proxies
   
     # called by Proxtopus.configure, returns self
     #def initialize(&block)
@@ -11,31 +10,43 @@ module Proxtopus
     #  instance_eval(&block) if block_given?
     #  self
     #end
-    def initialize(api_url, api_options)
-      @api_url = api_url
-      @api_options = api_options
-      @agent = Mechanize.new
+    def initialize(resources)
+      #@api_url = api_url
+      #@api_options = api_options
+      @resources = ResourceSet.new(resources)
       @proxies = ProxySet.new
       self
+    rescue => e
+      puts "Proxtopus::Client -- #{e.inspect}"
     end
     
     # p.collect({:cs => ['US'], etc...})
     # if no options are supplied, api_options will be used
     def collect(opts=nil)
-      use_opts = (opts.nil?)? api_options : opts
-      get(use_opts)
+      #use_opts = (opts.nil?)? api_options : opts
+      #get(use_opts)
+      @resources.each do |resource|
+        @proxies.push(resource.collect)
+      end
+    rescue => e
+      puts "Proxtopus::Client -- #{e.inspect}"
     end
     
     def random_proxy
       @proxies[rand(0..(@proxies.count-1))]
+    rescue => e
+      puts "Proxtopus::Client -- #{e.inspect}"
     end
     
     def delete_proxy(proxy)
       @proxies.delete(proxy)
+    rescue => e
+      puts "Proxtopus::Client -- #{e.inspect}"
     end
     
   private
   
+=begin
     def get(opts)
       begin
         json = @agent.get(Request.build(api_url, opts)).body
@@ -48,6 +59,7 @@ module Proxtopus
         exit
       end
     end
+=end
   end
   
 end

data/lib/proxtopus/proxy_set.rb

@@ -2,15 +2,20 @@ module Proxtopus
   class ProxySet < Array
     
     def push(proxy)
-      if !proxy.is_a?(Proxtopus::Proxy) && !proxy.is_a?(Hash)
-        raise ArgumentError, "A ProxySet may only contain Proxy or Hash objects!"
+      #puts proxy.class
+      if !proxy.is_a?(Proxtopus::Proxy) && !proxy.is_a?(Proxtopus::ProxySet) && !proxy.is_a?(Hash)
+        raise ArgumentError, "A ProxySet may only contain Proxy, ProxySet, or Hash objects!"
       end
       
-      if proxy.is_a?(Hash)
-        proxy = Proxy.new(proxy['host'], proxy['port'], proxy['protocol'], proxy['country'], proxy['anonymity'])
+      if !proxy.is_a?(Proxtopus::ProxySet)
+        if proxy.is_a?(Hash)
+          proxy = Proxy.new(proxy['host'], proxy['port'], proxy['protocol'], proxy['country'], proxy['anonymity'])
+        end
+        
+        super(proxy) if !include?(proxy)
+      else
+        proxy.each { |p| super(p) }
       end
-      
-      super(proxy) if !include?(proxy)
       self
     end
     
@@ -18,12 +23,16 @@ module Proxtopus
       if !proxy.is_a?(Proxtopus::Proxy) && !proxy.is_a?(Hash)
         raise ArgumentError, "A ProxySet may only contain Proxy or Hash objects!"
       end
-      
-      if proxy.is_a?(Hash)
-        proxy = Proxy.new(proxy['host'], proxy['port'], proxy['protocol'], proxy['country'], proxy['anonymity'])
+     
+      if !proxy.is_a?(Proxtopus::ProxySet) 
+        if proxy.is_a?(Hash)
+          proxy = Proxy.new(proxy['host'], proxy['port'], proxy['protocol'], proxy['country'], proxy['anonymity'])
+        end
+        
+        super(proxy) if !include?(proxy)
+      else
+        proxy.each { |p| super(p) }
       end
-      
-      super(proxy) if !include?(proxy)
       self
     end
     

data/lib/proxtopus/request.rb

@@ -1,14 +1,16 @@
 module Proxtopus
-  module Request
+  class Request# < Mechanize
     
-    class << self
-      def build(url, opt)
-        puts url
-        uri = URI(url.to_s)
-        uri.query = URI.encode_www_form(opt)
-        uri.to_s
-      end
-    end
+    attr_reader :resource, :response
     
+    def initialize(resource)
+      #puts "to request"
+      raise ArgumentError, "Request.new() requires resource to be of type Proxtopus::Resource!" if !resource.is_a?(Proxtopus::Resource)
+      
+      @resource = resource
+      page = Mechanize.new.get(@resource.uri.to_s)
+      @response = Response.new(page.body, @resource.format, @resource.elements)
+    end
+
   end
 end

data/lib/proxtopus/resource.rb

@@ -0,0 +1,52 @@
+module Proxtopus
+  class Resource
+    
+    attr_reader :request, :response, :url, :params, :format, :elements 
+    
+    def initialize(resource)
+      #puts resource.class
+      #puts
+      #puts resource.inspect
+      if resource.is_a?(Proxtopus::Resource)
+        @request = resource.request
+        @response = resource.response
+        @url = resource.url
+        @params = resource.params
+        @format = resource.format
+        @elements = resource.elements
+      elsif resource.is_a?(Hash)
+        @request = resource[:request]               || raise(ArgumentError, "Resource must have a specified request hash!")
+        @response = resource[:response]             || raise(ArgumentError, "Resource must have a specified resource hash!")
+        @url = resource[:request][:url]             || raise(ArgumentError, "Resource must have a specified request url!")
+        @params = resource[:request][:params]       || raise(ArgumentError, "Resource must have a specified request param!")
+        @format = resource[:response][:format]      || raise(ArgumentError, "Resource must have a specified resource format!")
+        @elements = resource[:response][:elements]  || raise(ArgumentError, "Resource must have a specified resource set of elemenra!")
+      else
+        raise ArgumentError, "Resource.new() expects resource as a Proxtopus::Resource or Hash!"
+      end
+    end
+    
+    def collect
+      Request.new(self).response.parsed
+    end
+    
+    def uri
+      uri = URI(@url.to_s)
+      uri.query = URI.encode_www_form(@params)
+      uri
+    end
+    
+    def ==(other)
+      if other.is_a?(Proxtopus::Resource)
+        if @url == other.url && @params == other.params
+          true
+        else
+          false
+        end
+      else
+        false
+      end
+    end
+    
+  end
+end

data/lib/proxtopus/resource_set.rb

@@ -0,0 +1,50 @@
+module Proxtopus
+  class ResourceSet < Array
+    
+    def initialize(resources=nil)
+      if resources
+        resources.each { |r| self.push(r) }
+      end
+    end
+    
+    def push(resource)
+      if !resource.is_a?(Proxtopus::Resource) && !resource.is_a?(Hash)
+        raise ArgumentError, "A ResourceSet may only contain Resource or Hash objects!"
+      end
+      
+      if resource.is_a?(Hash)
+        resource = Resource.new(resource)
+      end
+      
+      super(resource) if !include?(resource)
+      self
+    end
+
+=begin    
+    def shift(resource)
+      if !resource.is_a?(Proxtopus::Resource) && !resource.is_a?(Hash)
+        raise ArgumentError, "A ResourceSet may only contain Resource or Hash objects!"
+      end
+      
+      if resource.is_a?(Hash)
+        resource = Resource.new(resource['host'], resource['port'], resource['protocol'], resource['country'], resource['anonymity'])
+      end
+      
+      super(resource) if !include?(resource)
+      self
+    end
+=end
+    
+    def include?(resource)
+      if resource.is_a?(Proxtopus::Resource)
+        each do |p|
+          return true if resource == p
+        end
+        false
+      else
+        false
+      end
+    end
+    
+  end
+end

data/lib/proxtopus/response.rb

@@ -0,0 +1,28 @@
+module Proxtopus
+  class Response
+        
+    attr_reader :body, :format, :parsed
+    
+    def initialize(page, format, elements)
+      #puts "to response"
+      #raise ArgumentError, "Response.new() expects page to be of type Mechanize::Page!" if !page.is_a?(Mechanize::Page)
+      #raise ArgumentError, "Response.new() expects format to be of type Symbol!" if !format.is_a?(Symbol)
+      #raise ArgumentError, "Response.new() expects elements to be of type Symbol!" if !format.is_a?(Symbol)
+      
+      @body = page
+      @format = format
+      @elements = elements
+      
+      if format == :json
+        @parsed = Parser.json(@body, @elements)
+      elsif format == :xml
+        @parsed = Parser.xml(@body, @elements)
+      #elsif format == :rss
+      #  @parsed = Parser.rss(@body, @elements)
+      else
+        puts "Format #{format} is currenly unsupported... consider adding it =]"
+      end
+    end
+    
+  end
+end

data/lib/proxtopus/response_parser.rb

@@ -0,0 +1,42 @@
+module Proxtopus
+  class Response
+    module Parser
+      
+      class << self
+        def json(body, elements)
+          ret = ProxySet.new
+          els = JSON.parse(body)[elements[:root]]
+          els.each do |el|
+            host = el[elements[:host]]
+            port = el[elements[:port]]
+            anonymity = el[elements[:anonymity]]
+            protocol = el[elements[:protocol]]
+            country = el[elements[:country]]
+            ret.push(Proxy.new(host, port, anonymity, protocol, country))
+          end
+          ret
+        end
+        
+        def xml(body, elements)
+          ret = ProxySet.new
+          doc = Nokogiri::HTML(body)
+          els = doc.xpath(elements[:root])
+          els.each do |el|
+            host = el.xpath(elements[:host]).text
+            port = el.xpath(elements[:port]).text
+            anonymity = el.xpath(elements[:anonymity]).text
+            protocol = el.xpath(elements[:protocol]).text
+            country = el.xpath(elements[:country]).text
+            ret.push(Proxy.new(host, port, anonymity, protocol, country))
+          end
+          ret
+        end
+        
+        #def rss(body, elements)
+        #  ProxySet.new
+        #end
+      end
+      
+    end
+  end
+end

data/lib/proxtopus.rb

@@ -4,9 +4,13 @@ Bundler.require(:default)
 require 'uri'
 
 require_relative './proxtopus/client'
+require_relative './proxtopus/request'
+require_relative './proxtopus/response_parser'
+require_relative './proxtopus/response'
+require_relative './proxtopus/resource'
+require_relative './proxtopus/resource_set'
 require_relative './proxtopus/proxy'
 require_relative './proxtopus/proxy_set'
-require_relative './proxtopus/request'
 
 module Proxtopus
 
@@ -23,8 +27,9 @@ module Proxtopus
     #def new_client(&block)
     #  Client.new(&block) if block_given?
     #end
-    def client(api_url, api_options)
-      Client.new(api_url, api_options)
+    def client(resources_file)
+      resources = "[#{File.read(resources_file)}]".gsub!(/\r|\n|\s/,'')
+      Client.new(eval(resources))
     end
   end
   

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: proxtopus
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.31
 platform: ruby
 authors:
 - Ethan Barron
@@ -21,6 +21,10 @@ files:
 - lib/proxtopus/proxy.rb
 - lib/proxtopus/proxy_set.rb
 - lib/proxtopus/request.rb
+- lib/proxtopus/resource.rb
+- lib/proxtopus/resource_set.rb
+- lib/proxtopus/response.rb
+- lib/proxtopus/response_parser.rb
 homepage: 
 licenses:
 - MIT