vessel 0.2.0 → 0.3.0

data/lib/vessel/response.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "nokogiri"
+require "addressable/uri"
+
+module Vessel
+  class Response
+    extend Forwardable
+
+    attr_accessor :rejected
+    attr_reader :url, :data, :handler, :page, :attempt, :cookies
+
+    delegate %i[xpath css at_xpath at_css] => :page_or_body
+
+    def initialize(handler:, url: nil, data: nil, page: nil, attempt: 1)
+      @url = Addressable::URI.parse(url) if url
+      @data = data
+      @handler = handler
+      @attempt = attempt
+      @driven_page = page
+      @page = @driven_page&.page
+      @rejected = false
+      @cookies = @driven_page&.cookies&.map do |c|
+        {
+          name: c.name, value: c.value, domain: c.domain, path: c.path,
+          httponly: c.httponly?, secure: c.secure?, expires: c.expires
+        }
+      end
+    end
+
+    alias callback handler
+
+    def absolute_url(relative, encode: true)
+      if encode
+        relative = url_decode(relative)
+        relative = url_encode(relative)
+      end
+      join_url(url.to_s, relative)
+    end
+
+    def join_url(head, tail)
+      Addressable::URI.join(head, tail).to_s
+    end
+
+    def url_encode(uri)
+      Addressable::URI.encode(uri)
+    end
+    alias uri_encode url_encode
+
+    def url_decode(uri)
+      Addressable::URI.unencode(uri)
+    end
+    alias uri_decode url_decode
+
+    def body
+      @body ||= Nokogiri::HTML(raw)
+    end
+
+    def raw
+      @raw ||= @driven_page&.body
+    end
+
+    def sync_body(seconds = 2)
+      sleep(seconds)
+      @raw = @body = nil
+      body
+    end
+
+    def close
+      @driven_page&.close
+      @driven_page = @page = nil
+    end
+
+    def stub?
+      !url
+    end
+
+    def status
+      @driven_page&.status
+    end
+
+    def headers
+      @driven_page&.headers
+    end
+
+    def size
+      @driven_page&.size
+    end
+
+    def ==(other)
+      url == other.url &&
+        data == other.data &&
+        handler == other.handler &&
+        page == other.page &&
+        attempt == other.attempt &&
+        cookies == other.cookies
+    end
+
+    private
+
+    def page_or_body
+      Vessel.page_snapshot ? body : page
+    end
+  end
+end

data/lib/vessel/scheduler.rb

@@ -6,37 +6,37 @@ require "concurrent-ruby"
 module Vessel
   class Scheduler
     extend Forwardable
+
     delegate %i[scheduled_task_count completed_task_count queue_length] => :pool
 
-    attr_reader :browser, :queue, :delay, :headers
+    attr_reader :driver, :queue, :settings
 
     def initialize(queue, settings)
       @queue = queue
-      @min_threads, @max_threads, @delay, @headers =
-        settings.values_at(:min_threads, :max_threads, :delay, :headers)
-
-      options = settings[:ferrum]
-      options.merge!(timeout: settings[:timeout]) if settings[:timeout]
-      @browser = Ferrum::Browser.new(**options)
-
-      if settings[:intercept]
-        @browser.network.intercept
-        @browser.on(:request, &settings[:intercept])
-      end
+      @settings = settings
+      @driver = Driver::Registry.instance.build(settings)
     end
 
     def post(*requests)
-      requests.map do |request|
-        Concurrent::Promises.future_on(pool, queue, request) do |queue, request|
-          queue << goto(request)
-        end
+      requests.flatten.map do |request|
+        Concurrent::Future.execute(executor: pool,
+                                   args: [request]) { |r| queue << driver.go_to(r) }
       end
     end
 
     def stop
       pool.shutdown
       pool.kill unless pool.wait_for_termination(30)
-      browser.quit
+      driver.stop
+    end
+
+    def force_kill
+      pool.send(:ns_kill_execution)
+      driver.stop
+    end
+
+    def idle?
+      queue_length.zero? && scheduled_task_count == completed_task_count
     end
 
     private
@@ -44,24 +44,9 @@ module Vessel
     def pool
       @pool ||= Concurrent::ThreadPoolExecutor.new(
         max_queue: 0,
-        min_threads: @min_threads,
-        max_threads: @max_threads
+        min_threads: settings[:min_threads],
+        max_threads: settings[:max_threads]
       )
     end
-
-    def goto(request)
-      return [nil, request] if request.stub?
-
-      page = browser.create_page
-      page.headers.set(headers) if headers
-      # Delay is set between requests when we don't want to bombard server with
-      # requests so it requires crawler to be single threaded. Otherwise doesn't
-      # make sense.
-      sleep(delay) if @max_threads == 1 && delay > 0
-      page.goto(request.url)
-      [page, request]
-    rescue => e
-      e
-    end
   end
 end

data/lib/vessel/skeleton/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "vessel"

data/lib/vessel/skeleton/config/boot.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(Dir.pwd) unless $LOAD_PATH.include?(Dir.pwd)
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
+
+require "bundler/setup"
+Bundler.require(:default)
+Bundler.require(Vessel.env.to_sym)
+
+require "lib/loader"
+Vessel.loader = ApplicationLoader.new
+
+Dir["config/environments/*.rb"].each { |f| require f }
+Dir["config/environments/#{Vessel.env}/*.rb"].each { |f| require f }
+Dir["config/middleware/*.rb"].each { |f| require f }
+Dir["lib/helpers/*.rb"].each { |f| require f }
+Dir["config/fields/*.rb"].each { |f| require f }

data/lib/vessel/skeleton/config/environments/dev/dev.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class ApplicationCrawler < Vessel::Cargo
+  middleware "Debug"
+end

data/lib/vessel/skeleton/config/environments/prod/prod.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class ApplicationCrawler < Vessel::Cargo
+  delay 4..6
+  threads max: 1
+  middleware "Debug"
+end

data/lib/vessel/skeleton/config/middleware/debug.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class Debug < Vessel::Middleware
+  def call(hash, _fields)
+    puts deep_sort(hash)
+    hash
+  end
+
+  private
+
+  def deep_sort(hash)
+    sorted = hash.sort.to_h
+    sorted.transform_values { |v| v.is_a?(Hash) ? deep_sort(v) : v }
+  end
+end

data/lib/vessel/skeleton/lib/loader.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class ApplicationLoader < Vessel::Loader
+end

data/lib/vessel/templates/template.erb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class <%= @class_name %> < ApplicationCrawler
+  domain "<%= @domain %>"
+  start_urls "https://<%= @domain %>"
+
+  def parse
+  end
+end

data/lib/vessel/templator.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Vessel
+  class Templator
+    TEMPLATES = "templates/*.erb"
+
+    def initialize(domain, options)
+      @domain = domain
+      @options = options
+      @class_name = @domain.gsub("-", ".dash.").split(".").map(&:capitalize).join
+    end
+
+    def build
+      abort "Template #{template_path} not found" unless File.file?(template_path)
+
+      file = File.read(template_path)
+      evaluated = ERB.new(file).result(binding)
+      Logger.debug("Templator: creating file #{store_path}")
+      File.write(store_path, evaluated)
+    end
+
+    private
+
+    def template_path
+      File.expand_path("templates/template.erb", __dir__)
+    end
+
+    def store_path
+      "crawlers/#{@domain}.rb"
+    end
+  end
+end

data/lib/vessel/util.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Vessel
+  module Util
+    def deep_clone(value)
+      case value
+      when Hash
+        value.inject({}) { |b, (k, v)| b.merge(k => deep_clone(v)) }
+      when Array
+        value.inject([]) { |b, v| b << deep_clone(v) }
+      else
+        value
+      end
+    end
+  end
+end

data/lib/vessel/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vessel
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/vessel.rb

@@ -1,14 +1,44 @@
 # frozen_string_literal: true
 
+module Vessel
+  class Error < StandardError; end
+
+  class NotImplementedError < Error; end
+
+  class << self
+    attr_writer :loader, :page_snapshot
+
+    def boot
+      require File.expand_path("config/boot")
+    end
+
+    def env
+      ENV.fetch("VESSEL_ENV", "dev")
+    end
+
+    def page_snapshot
+      !!@page_snapshot
+    end
+
+    def loader
+      @loader ||= Loader.new
+    end
+  end
+end
+
 require "concurrent-ruby"
+require "vessel/cli"
 require "vessel/engine"
 require "vessel/middleware"
+require "vessel/middleware_scheduler"
 require "vessel/scheduler"
 require "vessel/request"
+require "vessel/response"
 require "vessel/version"
 require "vessel/cargo"
-
-module Vessel
-  class Error < StandardError; end
-  class NotImplementedError < Error; end
-end
+require "vessel/driver"
+require "vessel/driver/ferrum/driver"
+require "vessel/driver/mechanize/driver"
+require "vessel/templator"
+require "vessel/loader"
+require "vessel/logger"

metadata

@@ -1,108 +1,176 @@
 --- !ruby/object:Gem::Specification
 name: vessel
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Dmitry Vorotilin
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-09 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: ferrum
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.18'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.18'
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
+        version: '2.14'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.14'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
-  type: :development
+        version: '1.18'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '1.18'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
-  type: :development
+        version: '1.5'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.8'
+        version: '1.5'
 description: Vessel is a high-level web crawling framework, used to crawl websites
   and extract structured data from their pages
 email:
 - d.vorotilin@gmail.com
-executables: []
+executables:
+- vessel
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
 - LICENSE
 - README.md
+- bin/console
+- bin/vessel
 - lib/vessel.rb
 - lib/vessel/cargo.rb
+- lib/vessel/cargo/callbacks.rb
+- lib/vessel/cargo/field.rb
+- lib/vessel/cargo/field_type.rb
+- lib/vessel/cargo/fieldable.rb
+- lib/vessel/cargo/fields.rb
+- lib/vessel/cargo/settings.rb
 - lib/vessel/cli.rb
+- lib/vessel/driver.rb
+- lib/vessel/driver/ferrum/driver.rb
+- lib/vessel/driver/ferrum/page.rb
+- lib/vessel/driver/mechanize/driver.rb
+- lib/vessel/driver/mechanize/page.rb
+- lib/vessel/driver/page.rb
+- lib/vessel/driver/registry.rb
 - lib/vessel/engine.rb
+- lib/vessel/engine/stats.rb
+- lib/vessel/loader.rb
+- lib/vessel/logger.rb
 - lib/vessel/middleware.rb
+- lib/vessel/middleware_scheduler.rb
+- lib/vessel/proxy.rb
 - lib/vessel/request.rb
+- lib/vessel/response.rb
 - lib/vessel/scheduler.rb
+- lib/vessel/skeleton/Gemfile
+- lib/vessel/skeleton/config/boot.rb
+- lib/vessel/skeleton/config/environments/dev/dev.rb
+- lib/vessel/skeleton/config/environments/prod/prod.rb
+- lib/vessel/skeleton/config/fields/.keep
+- lib/vessel/skeleton/config/middleware/debug.rb
+- lib/vessel/skeleton/crawlers/.keep
+- lib/vessel/skeleton/lib/helpers/.keep
+- lib/vessel/skeleton/lib/loader.rb
+- lib/vessel/skeleton/log/.keep
+- lib/vessel/templates/template.erb
+- lib/vessel/templator.rb
+- lib/vessel/util.rb
 - lib/vessel/version.rb
-homepage: https://github.com/route/vessel
+homepage: https://github.com/rubycdp/vessel
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  homepage_uri: https://vessel.rubycdp.com/
+  bug_tracker_uri: https://github.com/rubycdp/vessel/issues
+  changelog_uri: https://github.com/rubycdp/vessel/blob/main/CHANGELOG.md
+  documentation_uri: https://github.com/rubycdp/vessel/blob/main/README.md
+  source_code_uri: https://github.com/rubycdp/vessel
+  rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
 - lib
@@ -110,15 +178,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 4.0.6
 specification_version: 4
 summary: High-level web crawling framework
 test_files: []