arachnid2 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e0c363d07cd32c0cbef2130ca5bb5a340ffbb2ea41c9ada00e0e292e5058753
-  data.tar.gz: cdc27f16c8f9b0b84878b9336b6101fb8fa5961e890136b5aa891e3d7fe3cc73
+  metadata.gz: 35ba8ebca2f7ed2459bdf6db38053835a6977bb51b34627fd2f9642bb3b7abb3
+  data.tar.gz: f3ed7cd8b5cbabb1643bb2fee9121a8df97e9ba05c4d5452f53e32fa91bd8090
 SHA512:
-  metadata.gz: a299e7db051a591bd131b1f633476e70d7dd45a7216c57e02bc51705b7e216be87c8d9b98f497fb084bbae1b06b0a07188884a486b79da951202e6e582bed5f9
-  data.tar.gz: 4148aa53935e6c52c45ce5a349624366a524ac2783f2615346fed73c04b968b46596500ffe1dc7f12389820506748ce65c70ee5f38c0dacbcc3c8e94cf0701f1
+  metadata.gz: b99ae55035e1064fa2140d8d5c13e924a47ba52905611586090c0e35650e31d77067c39a00245d0be5364945f24a025821f18b45917b8584fd3aec72fd0190b3
+  data.tar.gz: ebf1e8ca73ed4a964b0b33de3ca396ba48b59df1777402ac9cc8d3ba1126cd5ea63b559c5d55a88f51595af8f58acefa592cefebaae8f590d697586b5ac27461

data/README.md

@@ -67,6 +67,12 @@ opts = {
   max_urls: 50,
   language: "en-UK",
   user_agent: "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+  proxy: {
+    ip: "1.2.3.4",
+    port: "1234",
+    username: "sam",
+    password: "coolcoolcool",
+  }
 }
 responses = []
 
@@ -100,6 +106,12 @@ This user agent is a string mapped to the HTTP header User-Agent. The
 default is
 `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15`
 
+#### `proxy`
+
+Provide your IP, port for a proxy. If required, provide credentials for
+authenticating to that proxy. Proxy options and handling are done
+by Typhoeus.
+
 ### Memory use in Docker
 
 In case you are operating the crawler within a container, Arachnid2

data/lib/arachnid2.rb

@@ -65,8 +65,16 @@ class Arachnid2
   #
   #   opts = {
   #     :time_box => 30,
-  #     :language => "es-IO",
-  #     :user_agent => "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+  #     :headers => {
+  #       'Accept-Language' => "en-UK",
+  #       'User-Agent' => "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0",
+  #     },
+  #     :proxy => {
+  #       :ip => "1.2.3.4",
+  #       :port => "1234",
+  #       :username => "sam",
+  #       :password => "coolcoolcool",
+  #     }
   #   }
   #   responses = []
   #   spider.crawl(opts) { |response|
@@ -175,25 +183,27 @@ class Arachnid2
         followlocation: true,
         cookiefile: @cookie_file.path,
         cookiejar: @cookie_file.path,
-        headers: {
-          'Accept-Language' => "#{language}",
-          'User-Agent' => "#{user_agent}"
-        }
+        headers: @options[:headers]
       }
 
+      @request_options[:headers] ||= {}
+      @request_options[:headers]['Accept-Language'] ||= DEFAULT_LANGUAGE
+      @request_options[:headers]['User-Agent']      ||= DEFAULT_USER_AGENT
+
       @request_options
     end
 
-    def language
-      @options[:language] || DEFAULT_LANGUAGE
-    end
+    def crawl_options
+      @crawl_options ||= nil
 
-    def user_agent
-      @options[:user_agent] || DEFAULT_USER_AGENT
-    end
+      if !@crawl_options
+        @crawl_options = { :max_urls => max_urls, :time_limit => time_limit }
 
-    def crawl_options
-      { :max_urls => max_urls, :time_limit => time_limit }
+        @crawl_options[:proxy] = "#{@options[:proxy][:ip]}:#{@options[:proxy][:port]}" if @options.dig(:proxy, :ip)
+        @crawl_options[:proxyuserpwd] = "#{@options[:proxy][:username]}:#{@options[:proxy][:password]}" if @options.dig(:proxy, :username)
+      end
+
+      @crawl_options
     end
 
     def max_urls

data/lib/arachnid2/version.rb

@@ -1,3 +1,3 @@
 class Arachnid2
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arachnid2
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Sam Nissen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-05-30 00:00:00.000000000 Z
+date: 2018-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler