arachnid2 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 420df644a588b8eac92cfda03df0ab2ca20de52e3123aa7e1990ff850fd404d5
-  data.tar.gz: 49cdd7681f110d9a1d53075563b84e2614e959dad4cb39f877a36bf28be4dbf4
+  metadata.gz: 00cef9d45ae8be8b0747d47e254737fdfdb94e3f40cfe85a99faff283653f87b
+  data.tar.gz: 24ecf163c8b2eeda25908067a0efa68aec661441c59b06c466ee22828d154f5d
 SHA512:
-  metadata.gz: 49229b32b3d79cb560879298d2fa54f6206c8c419bbb86e7da9e1e132f84132026689270d0a6810610921be01676b37adb494e03a4f96a094561cb55fb2f7e4b
-  data.tar.gz: 505716bfafcdb116f25f401355928a2b65573d90d6d315d6710cc29247ce4f29c5c121e9d12c2ec807fe48a047823429aedfc5fe64b8d69683b8a603a8081621
+  metadata.gz: 2830f48686f9c2e9a921da58cca907580c800716e856982ae5e836f6dd51ab899192456bc4090ad78aac89eb6db49496d2ad953713d4d52548f1e248a7198df2
+  data.tar.gz: d3175c6f6574dc5a6feb9955e2a7804accfdd09c705cd90dea11586bd774fb79a67fdca499afe628739eb85b8d6adf6ff13d831658b23f4a993fc210e429c8f1

data/Gemfile.lock

@@ -17,13 +17,18 @@ GEM
       addressable (~> 2.5)
     bloomfilter-rb (2.1.1)
       redis
+    coderay (1.1.2)
     diff-lcs (1.3)
     ethon (0.11.0)
       ffi (>= 1.3.0)
     ffi (1.9.25)
+    method_source (0.9.0)
     mini_portile2 (2.3.0)
     nokogiri (1.8.4)
       mini_portile2 (~> 2.3.0)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
     public_suffix (3.0.3)
     rake (10.5.0)
     redis (4.0.2)
@@ -49,8 +54,9 @@ PLATFORMS
 DEPENDENCIES
   arachnid2!
   bundler (~> 1.16)
+  pry
   rake (~> 10.0)
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.16.2
+   1.16.3

data/README.md

@@ -22,6 +22,9 @@ on that page, and visiting those to do the same.
 Hence, the simplest output would be to collect all of the responses
 while spidering from some URL.
 
+Set cached service url(optional)
+`export ARACHNID_CACHED_SERVICE_ADDRESS=http://localhost:9000`
+
 ```ruby
 require "arachnid2"
 
@@ -129,7 +132,7 @@ by Typhoeus.
 
 This is the list of TLDs to ignore when collecting URLs from the page.
 The extensions are formatted as a hash of key/value pairs, where the value
-is an array of TLDs, and the keys represent the length of those TLDs. 
+is an array of TLDs, and the keys represent the length of those TLDs.
 
 #### `memory_limit` and Docker
 

data/lib/arachnid2/cashed_arachnid_responses.rb

@@ -0,0 +1,41 @@
+require 'net/http'
+require 'json'
+module CashedArachnidResponses
+  CACHE_SERVICE_URL = ENV['ARACHNID_CACHED_SERVICE_ADDRESS'].freeze
+
+  def load_data(_url, _options)
+    return if check_config
+
+    uri = URI("#{CACHE_SERVICE_URL}/typhoeus_responses?url=#{@url}&options=#{@options}")
+    req = Net::HTTP::Get.new(uri)
+    req['Accept'] = 'json'
+    Net::HTTP.start(uri.hostname, uri.port) do |http|
+      response = http.request(req)
+      return nil if response.code != '200'
+
+      body = ::JSON.parse(response.body)
+      responses_list = Base64.decode64(body['encrypted_response'])
+      return Marshal.load responses_list # here we get array of Typhoeus::Response
+    end
+  rescue StandardError
+    nil
+  end
+
+  def put_cached_data(url, options, data)
+    return if check_config
+
+    uri = URI("#{CACHE_SERVICE_URL}/typhoeus_responses")
+
+    header = { 'Content-Type': 'application/json' }
+    req = Net::HTTP::Post.new(uri, header)
+    processed_data = Base64.encode64(Marshal.dump(data))
+    req.body = { url: url, options: options, encrypted_response: processed_data }.to_json
+    Net::HTTP.start(uri.hostname, uri.port) do |http|
+      http.request(req)
+    end
+  end
+
+  def check_config
+    CACHE_SERVICE_URL.nil?
+  end
+end

data/lib/arachnid2/version.rb

@@ -1,3 +1,3 @@
 class Arachnid2
-  VERSION = "0.1.4"
+  VERSION = "0.2.0"
 end

data/lib/arachnid2.rb

@@ -1,4 +1,5 @@
 require "arachnid2/version"
+require "arachnid2/cashed_arachnid_responses"
 
 require 'tempfile'
 require "typhoeus"
@@ -6,9 +7,10 @@ require "bloomfilter-rb"
 require "adomain"
 require "addressable/uri"
 require "nokogiri"
+require "base64"
 
 class Arachnid2
-
+  include CashedArachnidResponses
   # META:
   #   About the origins of this crawling approach
   # The Crawler is heavily borrowed from by Arachnid.
@@ -57,6 +59,7 @@ class Arachnid2
   def initialize(url)
     @url = url
     @domain = Adomain[@url]
+    @cached_data = []
   end
 
   #
@@ -113,7 +116,15 @@ class Arachnid2
 
         request = Typhoeus::Request.new(q, request_options)
 
+        data = load_data(@url, opts)
+        unless data.nil?
+          data.each do |response|
+            yield response
+          end
+          return
+        end
         request.on_complete do |response|
+          @cached_data.push(response)
           links = process(response)
           next unless links
 
@@ -126,10 +137,13 @@ class Arachnid2
       end # @max_concurrency.times do
 
       @hydra.run
-    end # until @global_queue.empty?
 
+    end # until @global_queue.empty?
+    put_cached_data(@url, opts, @cached_data) unless @cached_data.empty?
   ensure
     @cookie_file.close! if @cookie_file
+
+
   end # def crawl(opts = {})
 
   private

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arachnid2
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - Sam Nissen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-22 00:00:00.000000000 Z
+date: 2019-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -142,6 +142,7 @@ files:
 - bin/console
 - bin/setup
 - lib/arachnid2.rb
+- lib/arachnid2/cashed_arachnid_responses.rb
 - lib/arachnid2/version.rb
 homepage: https://github.com/samnissen/arachnid2
 licenses: