vessel 0.2.0 → 0.3.0

data/lib/vessel/cargo/field_type.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Vessel
+  class Cargo
+    class FieldType
+      class << self
+        def add(*names, &block)
+          names.each { |n| types.merge!(n.to_sym => block) }
+        end
+
+        def types
+          @types ||= {}
+        end
+      end
+    end
+  end
+end

data/lib/vessel/cargo/fieldable.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "vessel/cargo/field"
+require "vessel/cargo/field_type"
+require "vessel/cargo/fields"
+
+module Vessel
+  class Cargo
+    module Fieldable
+      def fields
+        @fields ||= Fields.new
+      end
+
+      def field(name, *rest, &)
+        options = rest.last.is_a?(Hash) ? rest.pop : {}
+        service = options.delete(:service)
+        field = Field.new(name: name, value: options[:value], context: self, **options, &)
+        value = field.apply
+        if service
+          fields.service[name.to_sym] = value
+        else
+          fields[field.name] = value
+        end
+      end
+    end
+  end
+end

data/lib/vessel/cargo/fields.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Vessel
+  class Cargo
+    class Fields
+      attr_reader :service
+
+      def initialize(data = nil)
+        @service = {}
+        @fields = data.to_h
+      end
+
+      def []=(key, value)
+        @fields[key.to_sym] = value
+      end
+
+      def [](key)
+        @fields[key.to_sym]
+      end
+
+      def values_at(*args)
+        @fields.values_at(*args)
+      end
+
+      def count
+        @fields.count
+      end
+
+      def select(&)
+        @fields.send(:select, &)
+      end
+
+      def zip(value)
+        @fields.zip(value)
+      end
+
+      def key?(value)
+        @fields.key?(value)
+      end
+
+      def keys
+        @fields.keys
+      end
+
+      def to_h
+        @fields
+      end
+
+      def each(&)
+        @fields.each(&)
+      end
+    end
+  end
+end

data/lib/vessel/cargo/settings.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "vessel/proxy"
+require "vessel/util"
+
+module Vessel
+  class Cargo
+    module Settings
+      include Vessel::Util
+
+      DELAY = 0
+      START_URLS = {}.freeze
+      MIDDLEWARE = [].freeze
+      MIN_THREADS = 1
+      MAX_THREADS = Concurrent.processor_count
+      NETWORK_ERROR_ATTEMPTS = 5
+
+      def name
+        settings[:domain]
+      end
+
+      def domain(value)
+        settings[:domain] = value
+        Vessel.loader.register(name, self)
+      end
+
+      def start_urls(*url_handlers)
+        settings[:start_urls] = url_handlers.compact.flatten.inject({}) do |options, url_handler|
+          options.merge(url_handler.is_a?(Hash) ? url_handler : { url_handler => Request::DEFAULT_HANDLER })
+        end
+      end
+
+      def driver(name, **options)
+        settings[:driver_name] = name.to_sym
+        settings[:driver_options] = options
+      end
+
+      def delay(value)
+        settings[:delay] = value
+      end
+
+      def headers(value)
+        settings[:headers] = value
+      end
+
+      def cookie(name:, value:, domain: nil, path: nil, httponly: nil, secure: nil, expires: nil)
+        settings[:cookies] ||= []
+        domain ||= settings[:domain]
+        cookie = { name: name, value: value, domain: domain, path: path,
+                   httponly: httponly, secure: secure, expires: expires }.compact
+        settings[:cookies] << cookie
+      end
+
+      def cookies(*value)
+        value.flatten.each { |c| cookie(**c) }
+      end
+
+      def threads(min: MIN_THREADS, max: MAX_THREADS)
+        settings[:min_threads] = min
+        settings[:max_threads] = max
+      end
+
+      def middleware(*classes)
+        settings[:middleware] = classes
+      end
+
+      def proxy(klass)
+        settings[:proxy] = klass
+      end
+
+      def blacklist(patterns)
+        settings[:blacklist] = patterns
+      end
+
+      def whitelist(patterns)
+        settings[:whitelist] = patterns
+      end
+
+      def network_error_attempts(value)
+        settings[:network_error_attempts] = value.to_i
+      end
+
+      def allow_cookies(value)
+        settings[:allow_cookies] = value
+      end
+
+      def settings
+        @settings ||= if superclass.respond_to?(:settings)
+                        deep_clone(superclass.settings)
+                      else
+                        {
+                          delay: DELAY, start_urls: START_URLS, middleware: MIDDLEWARE,
+                          min_threads: MIN_THREADS, max_threads: MAX_THREADS,
+                          driver_name: :ferrum, driver_options: {}, headers: nil,
+                          cookies: nil, allow_cookies: true, proxy: nil,
+                          blacklist: nil, whitelist: nil, domain: nil,
+                          network_error_attempts: NETWORK_ERROR_ATTEMPTS
+                        }
+                      end
+      end
+
+      def to_domain_name(klass)
+        # name&.split("::")&.last&.downcase
+        klass.name.gsub(/[a-zA-Z](?=[A-Z])/, '\0 ').downcase.split.map do |value|
+          next "." if value == "dot"
+          next "-" if value == "dash"
+
+          value
+        end.join
+      end
+    end
+  end
+end

data/lib/vessel/cargo.rb

@@ -1,87 +1,52 @@
 # frozen_string_literal: true
 
-require "ferrum"
 require "forwardable"
+require "vessel/cargo/fieldable"
+require "vessel/cargo/settings"
+require "vessel/cargo/callbacks"
 
 module Vessel
   class Cargo
-    DELAY = 0
-    START_URLS = [].freeze
-    MIDDLEWARE = [].freeze
-    MIN_THREADS = 1
-    MAX_THREADS = Concurrent.processor_count
-
-    class << self
-      attr_reader :settings
-
-      def run(settings = nil, &block)
-        self.settings.merge!(Hash(settings))
-        Engine.run(self, &block)
-      end
-
-      def domain(name)
-        settings[:domain] = name
-      end
-
-      def start_urls(*urls)
-        settings[:start_urls] = urls.flatten
-      end
-
-      def delay(value)
-        settings[:delay] = value
-      end
-
-      def timeout(value)
-        settings[:timeout] = value
-      end
-
-      def headers(value)
-        settings[:headers] = value
-      end
-
-      def threads(min: MIN_THREADS, max: MAX_THREADS)
-        settings[:min_threads] = min
-        settings[:max_threads] = max
-      end
+    def self.run(settings = nil, &)
+      self.settings.merge!(Hash(settings))
+      Engine.run(self, &)
+    end
 
-      def middleware(*classes)
-        settings[:middleware] = classes
-      end
+    def self.build_request(url:, **options)
+      Request.new(url: url,
+                  delay: settings[:delay],
+                  cookies: settings[:cookies],
+                  headers: settings[:headers],
+                  **options)
+    end
 
-      def ferrum(**options)
-        settings[:ferrum] = options
-      end
+    def self.start_requests
+      return build_request(url: nil) if settings[:start_urls].empty?
 
-      def intercept(&block)
-        settings[:intercept] = block
-      end
+      settings[:start_urls].map { |u, h| build_request(url: u, handler: h) }
+    end
 
-      def settings
-        @settings ||= {
-          delay: DELAY,
-          middleware: MIDDLEWARE,
-          start_urls: START_URLS,
-          min_threads: MIN_THREADS,
-          max_threads: MAX_THREADS,
-          ferrum: Hash.new,
-          intercept: nil,
-          headers: nil,
-          domain: name&.split('::')&.last&.downcase
-        }
-      end
+    def self.store_cookies(cookies)
+      settings[:cookies] = cookies
     end
 
+    include Callbacks
+    include Fieldable
+    extend Settings
     extend Forwardable
-    delegate %i[at_css css at_xpath xpath] => :page
 
-    attr_reader :page
+    delegate %i[at_css css at_xpath xpath absolute_url join_url
+                url url_encode url_decode uri_encode uri_decode
+                data attempt body raw] => :response
 
-    def initialize(page = nil)
-      @page = page
-    end
+    attr_reader :settings, :domain, :response, :page, :size
 
-    def domain
-      self.class.settings[:domain]
+    def initialize(response = nil)
+      @settings = self.class.settings
+      @domain = settings[:domain]
+      @response = response
+      @page = response&.page
+      @size = response&.size
     end
 
     def parse
@@ -90,16 +55,11 @@ module Vessel
 
     private
 
-    def request(**options)
-      Request.new(**options)
-    end
-
-    def absolute_url(relative)
-      Addressable::URI.join(page.current_url, relative).to_s
-    end
-
-    def current_url
-      Addressable::URI.parse(page.current_url)
+    def request(url:, encode: true, **options)
+      url = absolute_url(url, encode: encode)
+      self.class.build_request(url: url, **options)
     end
   end
+
+  Crawler = Cargo
 end

data/lib/vessel/cli.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "pp"
 require "thor"
 
 module Vessel
@@ -7,9 +11,64 @@ module Vessel
       puts Vessel::VERSION
     end
 
-    desc "start NAME", "Run given crawler."
+    desc "list", "List all available crawlers of current project."
+    def list
+      Vessel.boot
+      Vessel.loader.load_crawlers
+      puts Vessel.loader.list_crawlers
+    end
+
+    desc "settings NAME", "Get the settings for given name."
+    def settings(name)
+      crawler = find_crawler(name)
+      settings = crawler.settings
+      puts "Delay: #{settings[:delay]}"
+      puts "Allow cookies: #{settings[:allow_cookies]}"
+      puts "Cookies: #{settings[:cookies]}"
+      puts "Middleware: #{settings[:middleware]}"
+      puts "Start urls:\n"
+      PP.pp(settings[:start_urls])
+    end
+
+    desc "start NAME", "Start crawling given name."
     def start(name)
-      raise NotImplementedError
+      crawler = find_crawler(name)
+      engine = Engine.new(crawler)
+      engine.run
+    end
+
+    desc "parse NAME URL CALLBACK --data=key:value", "Parse given url and call a callback with response."
+    method_option :data, type: :hash, default: {}, required: false
+    def parse(name, url, callback)
+      crawler = find_crawler(name)
+      engine = Engine.new(crawler)
+      data = options[:data].transform_keys(&:to_sym)
+      engine.parse(url, callback, data)
+    end
+
+    desc "new NAME", "Generate new project."
+    def new(name)
+      templates = File.expand_path("skeleton", __dir__)
+      FileUtils.cp_r(templates, "./#{name}")
+      puts "Project #{name} generated"
+    end
+
+    desc "generate NAME OPTIONS", "Generate crawler skeleton with given name and parameters."
+    def generate(name, *options)
+      require "erb"
+      Vessel.boot
+      templator = Vessel.loader.templator
+      templator.new(name, options).build
+    end
+
+    private
+
+    def find_crawler(name)
+      Vessel.boot
+      Vessel.loader.load_crawler(name)
+      crawler = Vessel.loader.find_crawler(name)
+      abort "Crawler not found" unless crawler
+      crawler
     end
   end
 end

data/lib/vessel/driver/ferrum/driver.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "ferrum"
+require "vessel/driver/ferrum/page"
+
+module Vessel
+  class Driver
+    module Ferrum
+      class Driver < ::Vessel::Driver
+        DEFAULT_OPTIONS = {
+          timeout: 60,
+          js_errors: false,
+          process_timeout: 30,
+          pending_connection_errors: false,
+          browser_options: { "ignore-certificate-errors" => nil }
+        }.freeze
+
+        def self.direct_network_errors
+          @direct_network_errors ||= (super + [::Ferrum::TimeoutError, ::Ferrum::StatusError]).freeze
+        end
+
+        driver_name :ferrum
+
+        def start
+          driver_options = settings.fetch(:driver_options, {})
+          @browser = ::Ferrum::Browser.new(**DEFAULT_OPTIONS, **driver_options)
+        end
+
+        def stop
+          browser&.quit
+          @browser = nil
+        end
+
+        def create_page(proxy: nil)
+          options = {}
+          options.merge!(proxy: proxy) if proxy
+          page = browser.create_page(**options)
+          Page.new(page, context: browser.contexts[page.context_id])
+        end
+      end
+    end
+  end
+end

data/lib/vessel/driver/ferrum/page.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Vessel
+  class Driver
+    module Ferrum
+      class Page < Page
+        extend Forwardable
+
+        delegate %i[css at_css xpath at_xpath body current_url go_to] => :page
+
+        attr_reader :page
+
+        def initialize(page, context:)
+          super()
+          @page = page
+          @context = context
+        end
+
+        def close
+          @context.dispose if page.use_proxy?
+          page.close
+        end
+
+        def headers=(headers)
+          return unless headers
+
+          page.headers.set(headers)
+        end
+
+        def headers
+          page.headers.get
+        end
+
+        def cookies=(cookies)
+          cookies.each { |c| page.cookies.set(c) }
+        end
+
+        def cookies
+          page.cookies.all.values
+        end
+
+        def status
+          page.network.status
+        end
+
+        def size
+          page.network.traffic.map do |req|
+            [req.response&.body_size, req.response&.headers_size]
+          end.flatten.compact.inject(&:+)
+        end
+
+        def blacklist=(patterns)
+          page.network.blacklist = patterns
+        end
+
+        def whitelist=(patterns)
+          page.network.whitelist = patterns
+        end
+      end
+    end
+  end
+end

data/lib/vessel/driver/mechanize/driver.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "mechanize"
+require "vessel/driver/mechanize/page"
+
+module Vessel
+  class Driver
+    module Mechanize
+      class Driver < ::Vessel::Driver
+        def self.indirect_network_errors
+          @indirect_network_errors ||= (super + [::Mechanize::ResponseCodeError]).freeze
+        end
+
+        driver_name :mechanize
+
+        def start
+          # Nothing to start here
+        end
+
+        def stop
+          # Nothing to stop here
+        end
+
+        def create_page(proxy: nil)
+          mechanize = ::Mechanize.new do |m|
+            m.user_agent = settings.dig(:headers, "User-Agent")
+            m.set_proxy(proxy[:host], proxy[:port], proxy[:user], proxy[:password]) if proxy
+          end
+
+          Page.new(mechanize)
+        end
+      end
+    end
+  end
+end

data/lib/vessel/driver/mechanize/page.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Vessel
+  class Driver
+    module Mechanize
+      class Page < Page
+        extend Forwardable
+
+        delegate %i[css at_css xpath at_xpath body] => :page
+
+        attr_reader :mechanize
+
+        def initialize(mechanize)
+          super()
+          @mechanize = mechanize
+        end
+
+        def close
+          mechanize.shutdown
+          @mechanize = nil
+        end
+
+        def go_to(url)
+          mechanize.get(url)
+        end
+
+        def headers=(headers)
+          mechanize.request_headers = headers
+        end
+
+        def headers
+          mechanize.request_headers
+        end
+
+        def cookies=(cookies)
+          cookies.each { |c| mechanize.cookie_jar << ::Mechanize::Cookie.new(c) }
+        end
+
+        def cookies
+          mechanize.cookies
+        end
+
+        def status
+          page.code.to_i
+        end
+
+        def size
+          page.content.bytesize
+        end
+
+        def current_url
+          page&.uri.to_s
+        end
+
+        # rubocop:disable all
+        def blacklist=(*)
+          @@_blacklist_warning ||= begin
+            warn "blacklist is not supported by mechanize driver"
+            true
+          end
+        end
+
+        def whitelist=(*)
+          @@_whitelist_warning ||= begin
+            warn "whitelist is not supported by mechanize driver"
+            true
+          end
+        end
+        # rubocop:enable all
+
+        def page
+          mechanize.current_page
+        end
+      end
+    end
+  end
+end