arachnid2 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ebb9ed9cdef3106462796f1b7fcc2483d58857bf605f04c800733358ea3f486
-  data.tar.gz: 86eaaf1bd44b85ee564b1bf3aeb08e2be326462c6bf8d3291ba2bc2f55e7c444
+  metadata.gz: 420df644a588b8eac92cfda03df0ab2ca20de52e3123aa7e1990ff850fd404d5
+  data.tar.gz: 49cdd7681f110d9a1d53075563b84e2614e959dad4cb39f877a36bf28be4dbf4
 SHA512:
-  metadata.gz: 1b8f1d5798379c75502cf36046c3110f02ae6abbd12a8ecaa7c501e1efd0d8d86393abd64212f12e3c3f951ddf5f8f6fed5ed15a9937305cf63ff92523087c89
-  data.tar.gz: 4f02afa25d537346b2cc6daaa16ae7b19956c5478bdfb5f4e2a9188825ad3f687691ad96025570661fd97dd12954c0e367be9cbdf9267f726b34ad13ea50bff0
+  metadata.gz: 49229b32b3d79cb560879298d2fa54f6206c8c419bbb86e7da9e1e132f84132026689270d0a6810610921be01676b37adb494e03a4f96a094561cb55fb2f7e4b
+  data.tar.gz: 505716bfafcdb116f25f401355928a2b65573d90d6d315d6710cc29247ce4f29c5c121e9d12c2ec807fe48a047823429aedfc5fe64b8d69683b8a603a8081621

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    arachnid2 (0.1.2)
+    arachnid2 (0.1.4)
       addressable
       adomain
       bloomfilter-rb
@@ -22,11 +22,11 @@ GEM
       ffi (>= 1.3.0)
     ffi (1.9.25)
     mini_portile2 (2.3.0)
-    nokogiri (1.8.2)
+    nokogiri (1.8.4)
       mini_portile2 (~> 2.3.0)
-    public_suffix (3.0.2)
+    public_suffix (3.0.3)
     rake (10.5.0)
-    redis (4.0.1)
+    redis (4.0.2)
     rspec (3.7.0)
       rspec-core (~> 3.7.0)
       rspec-expectations (~> 3.7.0)

data/README.md

@@ -78,6 +78,12 @@ opts = {
     username: "sam",
     password: "coolcoolcool",
   }
+  :non_html_extensions => {
+    3 => [".abc", ".xyz"],
+    4 => [".abcd"],
+    6 => [".abcdef"],
+    11 => [".abcdefghijk"]
+  }
 }
 responses = []
 
@@ -119,6 +125,12 @@ Provide your IP, port for a proxy. If required, provide credentials for
 authenticating to that proxy. Proxy options and handling are done
 by Typhoeus.
 
+#### `non_html_extensions`
+
+This is the list of TLDs to ignore when collecting URLs from the page.
+The extensions are formatted as a hash of key/value pairs, where the value
+is an array of TLDs, and the keys represent the length of those TLDs. 
+
 #### `memory_limit` and Docker
 
 In case you are operating the crawler within a container, Arachnid2

data/lib/arachnid2.rb

@@ -27,7 +27,7 @@ class Arachnid2
   DEFAULT_LANGUAGE = "en-IE, en-UK;q=0.9, en-NL;q=0.8, en-MT;q=0.7, en-LU;q=0.6, en;q=0.5, *;0.4"
   DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15"
 
-  NON_HTML_EXTENSIONS = {
+  DEFAULT_NON_HTML_EXTENSIONS = {
     3 => ['.gz'],
     4 => ['.jpg', '.png', '.m4a', '.mp3', '.mp4', '.pdf', '.zip',
           '.wmv', '.gif', '.doc', '.xls', '.pps', '.ppt', '.tar',
@@ -82,6 +82,12 @@ class Arachnid2
   #       :username => "sam",
   #       :password => "coolcoolcool",
   #     }
+  #     :non_html_extensions => {
+  #       3 => [".abc", ".xyz"],
+  #       4 => [".abcd"],
+  #       6 => [".abcdef"],
+  #       11 => [".abcdefghijk"]
+  #     }
   #   }
   #   responses = []
   #   spider.crawl(opts) { |response|
@@ -163,11 +169,23 @@ class Arachnid2
       @crawl_options = crawl_options
       @maximum_load_rate = maximum_load_rate
       @max_concurrency = max_concurrency
+      @non_html_extensions = non_html_extensions
       @hydra = Typhoeus::Hydra.new(:max_concurrency => @max_concurrency)
       @global_visited = BloomFilter::Native.new(:size => 1000000, :hashes => 5, :seed => 1, :bucket => 8, :raise => true)
       @global_queue = [@url]
     end
 
+    def non_html_extensions
+      @non_html_extensions ||= nil
+
+      if !@non_html_extensions
+        @non_html_extensions   = @options[:non_html_extensions]
+        @non_html_extensions ||= DEFAULT_NON_HTML_EXTENSIONS
+      end
+
+      @non_html_extensions
+    end
+
     def max_concurrency
       @max_concurrency ||= nil
 
@@ -263,7 +281,7 @@ class Arachnid2
     def extension_ignored?(url)
       return false if url.empty?
 
-      !NON_HTML_EXTENSIONS.values.flatten.find { |e| url.downcase.end_with? e.downcase }.nil?
+      !@non_html_extensions.values.flatten.find { |e| url.downcase.end_with? e.downcase }.nil?
     end
 
     def memory_danger?

data/lib/arachnid2/version.rb

@@ -1,3 +1,3 @@
 class Arachnid2
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arachnid2
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Sam Nissen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-04 00:00:00.000000000 Z
+date: 2018-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler