arachnid2 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 35ba8ebca2f7ed2459bdf6db38053835a6977bb51b34627fd2f9642bb3b7abb3
-  data.tar.gz: f3ed7cd8b5cbabb1643bb2fee9121a8df97e9ba05c4d5452f53e32fa91bd8090
+  metadata.gz: 0ebb9ed9cdef3106462796f1b7fcc2483d58857bf605f04c800733358ea3f486
+  data.tar.gz: 86eaaf1bd44b85ee564b1bf3aeb08e2be326462c6bf8d3291ba2bc2f55e7c444
 SHA512:
-  metadata.gz: b99ae55035e1064fa2140d8d5c13e924a47ba52905611586090c0e35650e31d77067c39a00245d0be5364945f24a025821f18b45917b8584fd3aec72fd0190b3
-  data.tar.gz: ebf1e8ca73ed4a964b0b33de3ca396ba48b59df1777402ac9cc8d3ba1126cd5ea63b559c5d55a88f51595af8f58acefa592cefebaae8f590d697586b5ac27461
+  metadata.gz: 1b8f1d5798379c75502cf36046c3110f02ae6abbd12a8ecaa7c501e1efd0d8d86393abd64212f12e3c3f951ddf5f8f6fed5ed15a9937305cf63ff92523087c89
+  data.tar.gz: 4f02afa25d537346b2cc6daaa16ae7b19956c5478bdfb5f4e2a9188825ad3f687691ad96025570661fd97dd12954c0e367be9cbdf9267f726b34ad13ea50bff0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    arachnid2 (0.1.1)
+    arachnid2 (0.1.2)
       addressable
       adomain
       bloomfilter-rb
@@ -20,7 +20,7 @@ GEM
     diff-lcs (1.3)
     ethon (0.11.0)
       ffi (>= 1.3.0)
-    ffi (1.9.23)
+    ffi (1.9.25)
     mini_portile2 (2.3.0)
     nokogiri (1.8.2)
       mini_portile2 (~> 2.3.0)

data/README.md

@@ -63,10 +63,15 @@ require "arachnid2"
 url = "http://sixcolours.com"
 spider = Arachnid2.new(url)
 opts = {
+  followlocation: true,
+  timeout: 10000,
   time_box: 60,
   max_urls: 50,
-  language: "en-UK",
-  user_agent: "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+  :headers => {
+    'Accept-Language' => "en-UK",
+    'User-Agent' => "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0",
+  },
+  memory_limit: 89.99,
   proxy: {
     ip: "1.2.3.4",
     port: "1234",
@@ -94,16 +99,18 @@ If no valid integer is provided, it will crawl for 50 URLs before exiting.
 10000 seconds is the current maximum,
 and any value above it will be reduced to 10000.
 
-#### `language`
+#### `headers`
 
-The language is a string mapped to the HTTP header Accept-Language. The
-default is
-`en-IE, en-UK;q=0.9, en-NL;q=0.8, en-MT;q=0.7, en-LU;q=0.6, en;q=0.5, \*;0.4`
+This is a hash that represents any HTTP header key/value pairs you desire,
+and is passed directly to Typheous. Before it is sent, a default
+language and user agent are created:
+
+##### Defaults
 
-#### `user_agent`
+The HTTP header `Accept-Language` default is
+`en-IE, en-UK;q=0.9, en-NL;q=0.8, en-MT;q=0.7, en-LU;q=0.6, en;q=0.5, \*;0.4`
 
-This user agent is a string mapped to the HTTP header User-Agent. The
-default is
+The HTTP header `User-Agent` default is
 `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15`
 
 #### `proxy`
@@ -112,12 +119,13 @@ Provide your IP, port for a proxy. If required, provide credentials for
 authenticating to that proxy. Proxy options and handling are done
 by Typhoeus.
 
-### Memory use in Docker
+#### `memory_limit` and Docker
 
 In case you are operating the crawler within a container, Arachnid2
-will attempt to prevent the container from running out of memory,
-and crawls will end when the container has <= 20% of its available memory
-free.
+can attempt to prevent the container from running out of memory.
+By default, it will end the crawl when the container uses >= 80%
+of its available memory. You can override this with the
+option.
 
 ### Non-HTML links
 

data/lib/arachnid2.rb

@@ -37,7 +37,11 @@ class Arachnid2
   }
   MEMORY_USE_FILE = "/sys/fs/cgroup/memory/memory.usage_in_bytes"
   MEMORY_LIMIT_FILE = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
-  MAXIMUM_LOAD_RATE = 79.9
+  DEFAULT_MAXIMUM_LOAD_RATE = 79.9
+
+  DEFAULT_TIMEOUT = 10_000
+  MINIMUM_TIMEOUT = 1
+  MAXIMUM_TIMEOUT = 999_999
 
   #
   # Creates the object to execute the crawl
@@ -64,11 +68,14 @@ class Arachnid2
   #   spider = Arachnid2.new(url)
   #
   #   opts = {
+  #     :followlocation => true,
+  #     :timeout => 25000,
   #     :time_box => 30,
   #     :headers => {
   #       'Accept-Language' => "en-UK",
   #       'User-Agent' => "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0",
   #     },
+  #     :memory_limit => 89.99,
   #     :proxy => {
   #       :ip => "1.2.3.4",
   #       :port => "1234",
@@ -89,34 +96,35 @@ class Arachnid2
     preflight(opts)
 
     until @global_queue.empty?
-      @global_queue.size.times do
-        begin
-          q = @global_queue.shift
+      @max_concurrency.times do
+        q = @global_queue.shift
 
-          break if @global_visited.size >= @crawl_options[:max_urls]
-          break if Time.now > @crawl_options[:time_limit]
-          break if memory_danger?
+        break if @global_visited.size >= @crawl_options[:max_urls]
+        break if Time.now > @crawl_options[:time_limit]
+        break if memory_danger?
 
-          @global_visited.insert(q)
+        @global_visited.insert(q)
 
-          request = Typhoeus::Request.new(q, request_options)
+        request = Typhoeus::Request.new(q, request_options)
 
-          request.on_complete do |response|
-            links = process(response)
-            next unless links
+        request.on_complete do |response|
+          links = process(response)
+          next unless links
 
-            yield response
+          yield response
 
-            vacuum(links, response)
-          end
-
-          request.run
-        ensure
-          @cookie_file.close! if @cookie_file
+          vacuum(links, response)
         end
-      end
-    end
-  end
+
+        @hydra.queue(request)
+      end # @max_concurrency.times do
+
+      @hydra.run
+    end # until @global_queue.empty?
+
+  ensure
+    @cookie_file.close! if @cookie_file
+  end # def crawl(opts = {})
 
   private
     def process(response)
@@ -153,12 +161,24 @@ class Arachnid2
     def preflight(opts)
       @options = opts
       @crawl_options = crawl_options
-      # TODO: write looping to take advantage of Hydra
-      # @hydra = Typhoeus::Hydra.new(:max_concurrency => 1)
+      @maximum_load_rate = maximum_load_rate
+      @max_concurrency = max_concurrency
+      @hydra = Typhoeus::Hydra.new(:max_concurrency => @max_concurrency)
       @global_visited = BloomFilter::Native.new(:size => 1000000, :hashes => 5, :seed => 1, :bucket => 8, :raise => true)
       @global_queue = [@url]
     end
 
+    def max_concurrency
+      @max_concurrency ||= nil
+
+      if !@max_concurrency
+        @max_concurrency = "#{@options[:max_concurrency]}".to_i
+        @max_concurrency = 1 unless (@max_concurrency > 0)
+      end
+
+      @max_concurrency
+    end
+
     def bound_time
       boundary = "#{@options[:time_box]}".to_i
       boundary = BASE_CRAWL_TIME if boundary <= 0
@@ -175,12 +195,30 @@ class Arachnid2
       amount
     end
 
+    def followlocation
+      if @followlocation.is_a?(NilClass)
+        @followlocation = @options[:followlocation]
+        @followlocation = true unless @followlocation.is_a?(FalseClass)
+      end
+      @followlocation
+    end
+
+    def timeout
+      if !@timeout
+        @timeout = @options[:timeout]
+        @timeout = DEFAULT_TIMEOUT unless @timeout.is_a?(Integer)
+        @timeout = DEFAULT_TIMEOUT if @timeout > MAXIMUM_TIMEOUT
+        @timeout = DEFAULT_TIMEOUT if @timeout < MINIMUM_TIMEOUT
+      end
+      @timeout
+    end
+
     def request_options
       @cookie_file ||= Tempfile.new('cookies')
 
       @request_options = {
-        timeout: 10000,
-        followlocation: true,
+        timeout: timeout,
+        followlocation: followlocation,
         cookiefile: @cookie_file.path,
         cookiejar: @cookie_file.path,
         headers: @options[:headers]
@@ -236,7 +274,7 @@ class Arachnid2
 
       return false unless ( (use > 0.0) && (@limit > 0.0) )
 
-      return ( ( (use / @limit) * 100.0 ) >= MAXIMUM_LOAD_RATE )
+      return ( ( (use / @limit) * 100.0 ) >= @maximum_load_rate )
     end
 
     def in_docker?
@@ -244,4 +282,15 @@ class Arachnid2
       true
     end
 
+    def maximum_load_rate
+      @maximum_load_rate ||= nil
+
+      if !@maximum_load_rate
+        @maximum_load_rate = "#{@options[:memory_limit]}".to_f
+        @maximum_load_rate = DEFAULT_MAXIMUM_LOAD_RATE unless ((@maximum_load_rate > 0.0) && (@maximum_load_rate < 100.0))
+      end
+
+      @maximum_load_rate
+    end
+
 end

data/lib/arachnid2/version.rb

@@ -1,3 +1,3 @@
 class Arachnid2
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arachnid2
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Sam Nissen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-12 00:00:00.000000000 Z
+date: 2018-07-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler