vessel 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44fb472d4afaf916edc97894dcc39cf8b6bfbf3f8f1f0b2e8a47f495482b1bd9
-  data.tar.gz: 36af4cd9021bd410bf1988c01f97d98df4e5646f5a56416004869fe643403672
+  metadata.gz: 291c528925e5f6990b8223762c00a996cf8435325b52048c1c0338fc873d1a42
+  data.tar.gz: 5c845e2c6b19adea7742bbad3a031fe0ad5f09a3a0f8ceff248970cc2bb74fcb
 SHA512:
-  metadata.gz: bda3863083cdce0e8011675a0e83a583d626e81ab713803a54c5056f922d4822b069dacd8d4e5f0079d4f8625a172f7f9d30d4e3586439137af088ac0911201e
-  data.tar.gz: 205b2f54fa17283daf50d0fdaa96e67f5dec4bed2c69ccc740433c90ecefaa9c4b1e13740cb703ac56cfe8c92b2df9da436fee94fc7937242465a33e91a088f5
+  metadata.gz: 77d65c60e1541c1ab1473b64bd7c451e3d57199fa19885bcf4e182d295321d0cdd36121cc673a9285a7113b482ec9143607d17a4017632d1d6826b5c5ad48b7e
+  data.tar.gz: 2a0e9bbe6de79e9ba6e7afed4d422de6b72639d91b508a921dd4bba2cf0b8e878a3de6846fca87de25e20047594b153efcf70ca3b91d373190bf976605e0bb2d

data/README.md

@@ -97,23 +97,5 @@ To be continued
 
 ## License
 
-Copyright 2018-2020 Machinio
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+The gem is available as open source under the terms of the
+[MIT License](https://opensource.org/licenses/MIT).

data/lib/vessel/cargo.rb

@@ -24,7 +24,7 @@ module Vessel
       end
 
       def start_urls(*urls)
-        settings[:start_urls] = urls
+        settings[:start_urls] = urls.flatten
       end
 
       def delay(value)
@@ -35,6 +35,10 @@ module Vessel
         settings[:timeout] = value
       end
 
+      def headers(value)
+        settings[:headers] = value
+      end
+
       def threads(min: MIN_THREADS, max: MAX_THREADS)
         settings[:min_threads] = min
         settings[:max_threads] = max
@@ -44,6 +48,14 @@ module Vessel
         settings[:middleware] = classes
       end
 
+      def ferrum(**options)
+        settings[:ferrum] = options
+      end
+
+      def intercept(&block)
+        settings[:intercept] = block
+      end
+
       def settings
         @settings ||= {
           delay: DELAY,
@@ -51,6 +63,9 @@ module Vessel
           start_urls: START_URLS,
           min_threads: MIN_THREADS,
           max_threads: MAX_THREADS,
+          ferrum: Hash.new,
+          intercept: nil,
+          headers: nil,
           domain: name&.split('::')&.last&.downcase
         }
       end
@@ -82,5 +97,9 @@ module Vessel
     def absolute_url(relative)
       Addressable::URI.join(page.current_url, relative).to_s
     end
+
+    def current_url
+      Addressable::URI.parse(page.current_url)
+    end
   end
 end

data/lib/vessel/engine.rb

@@ -21,25 +21,35 @@ module Vessel
 
       until @queue.closed?
         message = @queue.pop
+
         raise(message) if message.is_a?(Exception)
-        handle(*message)
+
+        page, request = message
+        args = [request.method, request.data].compact
+        handle(page, args)
+
         @queue.close if idle?
       end
+
+    ensure
+      scheduler.stop
     end
 
-    def handle(page, request)
+    def handle(page, args)
       crawler = @crawler_class.new(page)
-      crawler.send(request.method) do |*args|
-        if args.all? { |i| i.is_a?(Request) }
-          scheduler.post(*args)
+      crawler.send(*args) do |*result|
+        if result.flatten.all? { |i| i.is_a?(Request) }
+          scheduler.post(*result.flatten)
         else
-          @middleware&.call(*args)
+          @middleware&.call(*result)
         end
       end
     ensure
-      page.close
+      page.close if page
     end
 
+    private
+
     def start_requests
       Request.build(*settings[:start_urls])
     end

data/lib/vessel/request.rb

@@ -4,16 +4,24 @@ require "addressable/uri"
 
 module Vessel
   class Request
-    attr_reader :url, :uri, :method
+    attr_reader :url, :uri, :method, :data
 
     def self.build(*urls)
-      urls.map { |url| new(url: url) }
+      urls.empty? ? [new] : urls.map { |url| new(url: url) }
     end
 
-    def initialize(url:, method: :parse)
-      @url = url.to_s
-      @uri = Addressable::URI.parse(@url)
+    def initialize(url: nil, method: :parse, data: nil)
+      if url
+        @url = url.to_s
+        @uri = Addressable::URI.parse(@url)
+      end
+
       @method = method
+      @data = data.freeze if data
+    end
+
+    def stub?
+      !url
     end
   end
 end

data/lib/vessel/scheduler.rb

@@ -6,18 +6,23 @@ require "concurrent-ruby"
 module Vessel
   class Scheduler
     extend Forwardable
-    delegate %i[scheduled_task_count completed_task_count queue_length] => :@pool
+    delegate %i[scheduled_task_count completed_task_count queue_length] => :pool
 
-    attr_reader :browser, :queue, :delay
+    attr_reader :browser, :queue, :delay, :headers
 
     def initialize(queue, settings)
       @queue = queue
-      @min_threads, @max_threads, @delay =
-        settings.values_at(:min_threads, :max_threads, :delay)
+      @min_threads, @max_threads, @delay, @headers =
+        settings.values_at(:min_threads, :max_threads, :delay, :headers)
 
-      options = {}
+      options = settings[:ferrum]
       options.merge!(timeout: settings[:timeout]) if settings[:timeout]
       @browser = Ferrum::Browser.new(**options)
+
+      if settings[:intercept]
+        @browser.network.intercept
+        @browser.on(:request, &settings[:intercept])
+      end
     end
 
     def post(*requests)
@@ -28,6 +33,12 @@ module Vessel
       end
     end
 
+    def stop
+      pool.shutdown
+      pool.kill unless pool.wait_for_termination(30)
+      browser.quit
+    end
+
     private
 
     def pool
@@ -39,7 +50,10 @@ module Vessel
     end
 
     def goto(request)
+      return [nil, request] if request.stub?
+
       page = browser.create_page
+      page.headers.set(headers) if headers
       # Delay is set between requests when we don't want to bombard server with
       # requests so it requires crawler to be single threaded. Otherwise doesn't
       # make sense.

data/lib/vessel/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vessel
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vessel
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Dmitry Vorotilin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-06 00:00:00.000000000 Z
+date: 2021-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ferrum
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -117,7 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: High-level web crawling framework