browser_crawler 0.4.0

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/browser_crawler.gemspec

@@ -0,0 +1,47 @@
+# coding: utf-8
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'browser_crawler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'browser_crawler'
+  spec.version       = BrowserCrawler::VERSION
+  spec.required_ruby_version = '>= 2.5.0'
+  spec.authors       = ['Dmytro Samodurov',
+                        'Artem Rumiantcev',
+                        'Denys Ivanchuk',
+                        'Sergiy Tyatin']
+  spec.email         = ['dimasamodurov@gmail.com', 'tema.place@gmail.com']
+  spec.licenses      = ['MIT']
+
+  spec.summary       = 'Simple site crawler using Capybara'
+  spec.description   = ''
+  spec.homepage      = 'https://github.com/DimaSamodurov/browser_crawler'
+
+  # Prevent pushing this gem to RubyGems.org.
+  # To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host
+  # or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['homepage_uri'] = spec.homepage
+    spec.metadata['source_code_uri'] = spec.homepage
+  else
+    raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.'
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport', '~> 5.2', '>= 5.2.2'
+  spec.add_dependency 'capybara', '~> 3.24', '>= 3.24.0'
+  spec.add_dependency 'chromedriver-helper', '~> 2.1', '>= 2.1.0'
+  spec.add_dependency 'cuprite', '~> 0.6.0'
+
+  spec.add_development_dependency 'bundler', '~> 1.17.2', '>= 1.17.2'
+  spec.add_development_dependency 'pry-byebug', '~> 3.6', '>= 3.6'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.66'
+end

data/lib/browser_crawler.rb

@@ -0,0 +1,12 @@
+require 'browser_crawler/version'
+require 'browser_crawler/options'
+require 'browser_crawler/engine'
+
+require 'browser_crawler/followups/screenshots_indexer'
+require 'browser_crawler/followups/wraith_integrator'
+
+# Crawls web site and extracts links available.
+
+module BrowserCrawler
+  # Your code goes here...
+end

data/lib/browser_crawler/dsl/js_helpers.rb

@@ -0,0 +1,13 @@
+module BrowserCrawler
+  module DSL
+    module JsHelpers
+      def wait_for_page_to_load
+        10.times do
+          return if page.evaluate_script('document.readyState') == 'complete'
+
+          sleep(0.5)
+        end
+      end
+    end
+  end
+end

data/lib/browser_crawler/dsl/sign_in.rb

@@ -0,0 +1,37 @@
+module BrowserCrawler
+  module DSL
+    module SignIn
+      def sign_in
+        visit '/'
+        pingfed_o365_login
+      end
+
+      def pingfed_login(force: true)
+        if force || page.has_content?('Enter your credentials')
+          fill_in 'input_username', with: ENV.fetch('username')
+          fill_in 'input_password', with: ENV.fetch('password')
+          click_on 'Login'
+        end
+      end
+
+      def o365_login(force: true)
+        if force || page.has_content?('Stay signed in?')
+          check 'DontShowAgain'
+          click_on 'Yes'
+        end
+      end
+
+      def o365_stay_signed_in(force: true)
+        if force || page.has_content?('Stay signed in?')
+          check 'DontShowAgain'
+          click_on 'Yes'
+        end
+      end
+
+      def pingfed_o365_login(force: true)
+        pingfed_login(force: force)
+        o365_stay_signed_in(force: force)
+      end
+    end
+  end
+end

data/lib/browser_crawler/engine.rb

@@ -0,0 +1,156 @@
+require 'capybara'
+require 'capybara/dsl'
+require 'logger'
+
+require_relative 'dsl/sign_in'
+require_relative 'dsl/js_helpers'
+require_relative 'report_factory'
+require_relative 'reports/store'
+require_relative 'support/capybara'
+require_relative 'screenshot_operator'
+require_relative 'url_tools'
+require_relative 'engine_utilities/crawl_manager'
+require_relative 'hooks_operator'
+require_relative 'hooks_container'
+
+module BrowserCrawler
+  class Engine
+    include Capybara::DSL
+    include HooksOperator
+    include DSL::SignIn
+    include DSL::JsHelpers
+
+    class UnavailableCallBackMethod < StandardError
+    end
+
+    REPORT_SAVE_FOLDER_PATH    = 'tmp'.freeze
+
+    CUPRITE_OPTIONS            = {
+      window_size: [1280, 1600]
+    }.freeze
+
+    SCREENSHOT_OPERATOR_OPTIONS = {
+      save_screenshots: false,
+      save_screenshots_to: nil,
+      format: 'png',
+      filename: nil
+    }.freeze
+
+    attr_reader :report_store,
+                :screenshot_operator,
+                :crawl_manager,
+                :logger
+
+    def initialize(browser_options: {},
+                   screenshots_options: {},
+                   max_pages: nil,
+                   deep_visit: false,
+                   logger: nil)
+      screenshots_operator_options = SCREENSHOT_OPERATOR_OPTIONS
+                                     .merge(screenshots_options)
+      @screenshot_operator = ScreenshotOperator.new(screenshots_operator_options)
+
+      cuprite_options = CUPRITE_OPTIONS.merge(browser_options)
+
+      @logger = logger || Logger.new(STDOUT)
+
+      register_chrome_driver(cuprite_options)
+      initialize_report_store(cuprite_options)
+      initialize_crawl_manager(max_pages, deep_visit)
+    end
+
+    def js_before_run(javascript: '')
+      return if javascript.empty?
+
+      @javascript_before_run = javascript
+    end
+
+    def extract_links(url:)
+      initialize_crawler(url)
+
+      begin
+        with_hooks_for(type: :all) do
+          crawl_manager.crawl(
+            target_url: url,
+            capybara_session: Capybara.current_session,
+            screenshot_operator: screenshot_operator
+          )
+        end
+      rescue StandardError => error
+        logger
+          .fatal("#{error.message} \n #{error.backtrace.join("\n")}")
+      ensure
+        @report_store.finish
+      end
+      self
+    end
+
+    def report_save(folder_path: '', type: :yaml)
+      save_folder_path = folder_path.empty? ? REPORT_SAVE_FOLDER_PATH : folder_path
+      ReportFactory.save(store: @report_store,
+                         type: type.to_sym,
+                         save_folder_path: save_folder_path)
+    end
+
+    def before(type: :all, &hook)
+      HooksContainer.instance.add_hook(method: :before, type: type, hook: hook)
+    end
+
+    def after(type: :all, &hook)
+      HooksContainer.instance.add_hook(method: :after, type: type, hook: hook)
+    end
+
+    def unvisited_links(&hook)
+      HooksContainer.instance.add_hook(type: :unvisited_links, hook: hook)
+    end
+
+    def change_page_scan_rules(&hook)
+      HooksContainer.instance.add_hook(type: :scan_rules, hook: hook)
+    end
+
+    private
+
+    def initialize_crawler(url)
+      Capybara.current_session.quit
+
+      uri               = UrlTools.uri!(url: url)
+      Capybara.app_host = "#{uri.scheme}://#{uri.host}:#{uri.port}"
+
+      @report_store.start(url: url)
+
+      return if @javascript_before_run.nil?
+
+      Capybara.current_session
+        .driver
+        .browser
+        .page
+        .command('Page.addScriptToEvaluateOnNewDocument',
+                 source: @javascript_before_run)
+    end
+
+    def initialize_report_store(cuprite_options)
+      @report_store = Reports::Store.new
+      @report_store.metadata[:screenshots_path] = screenshot_operator
+                                                  .screenshots_folder
+      @report_store.metadata[:window_width] = cuprite_options[:window_size][0]
+      @report_store.metadata[:window_height] = cuprite_options[:window_size][1]
+    end
+
+    def register_chrome_driver(cuprite_options)
+      Capybara.register_chrome_driver(:cuprite_chrome, options: cuprite_options)
+      Capybara.run_server             = false
+      Capybara.default_driver         = :cuprite_chrome
+      # a workaround to extracting data from inactive tabs, dialogs, etc.
+      Capybara.ignore_hidden_elements = false
+    end
+
+    def initialize_crawl_manager(max_pages, deep_visit)
+      @crawl_manager = EngineUtilities::CrawlManager.new(
+        report_store: report_store,
+        max_pages: max_pages.to_i,
+        deep_visit: deep_visit,
+        logger: @logger
+      )
+    end
+  end
+end

data/lib/browser_crawler/engine_utilities/crawl_manager.rb

@@ -0,0 +1,100 @@
+require_relative '../url_tools'
+require_relative 'link_inspector'
+require_relative 'page_inspector'
+require_relative 'inspect_page_process'
+
+module BrowserCrawler
+  module EngineUtilities
+    # This main operated class which controls queue of unvisisted links.
+    class CrawlManager
+
+      attr_reader :target_url,
+                  :unvisited_links_queue,
+                  :report_store,
+                  :host_name,
+                  :deep_visit,
+                  :max_pages,
+                  :logger,
+                  :page_inspector
+
+      def initialize(report_store:,
+                     max_pages: 0,
+                     deep_visit: false,
+                     logger: nil)
+        @report_store     = report_store
+        @max_pages        = max_pages
+        @deep_visit       = deep_visit
+        @logger           = logger || Logger.new(STDOUT)
+      end
+
+      def crawl(target_url:, capybara_session:, screenshot_operator: nil)
+        @host_name             = UrlTools.uri!(url: target_url).host
+        @unvisited_links_queue = [target_url]
+
+        loop do
+          break if unvisited_links_queue.empty? || limit_reached?
+
+          unvisited_link = unvisited_links_queue.shift
+
+          link_inspector = LinkInspector.new(raw_link: unvisited_link,
+                                             host_name: host_name)
+
+          unless link_valid?(link_inspector)
+            @logger.info("Skipped visited #{unvisited_link}")
+            report_store.record_unrecognized_link(unvisited_link)
+            next
+          end
+
+          inspect_page(link_inspector: link_inspector,
+                       capybara_session: capybara_session,
+                       screenshot_operator: screenshot_operator)
+        end
+      end
+
+      def link_valid?(link_inspector)
+        link_inspector.link_valid? &&
+          internal_resource?(link_inspector) &&
+          page_unvisited?(link_inspector)
+      end
+
+      private
+
+      def inspect_page(link_inspector:, capybara_session:, screenshot_operator:)
+        InspectPageProcess.new(link_inspector: link_inspector,
+                               capybara_session: capybara_session,
+                               screenshot_operator: screenshot_operator,
+                               report_store: report_store,
+                               logger: logger)
+                          .call(unvisited_links_queue: unvisited_links_queue)
+      rescue StandardError => error
+        error_handler(link: link_inspector.raw_link, error: error)
+      end
+
+      def internal_resource?(link_inspector)
+        link_inspector.internal_url? || deep_visit
+      end
+
+      def page_unvisited?(link_inspector)
+        !visited_pages.include?(link_inspector.full_url)
+      end
+
+      def limit_reached?
+        return false if max_pages.zero?
+
+        visited_pages.count >= max_pages
+      end
+
+      def visited_pages
+        report_store.visited_pages
+      end
+
+      def error_handler(link:, error:)
+        error_link = "visiting link - #{link};\n"
+        error_message = "error message: #{error.message};\n"
+        error_backtrace = "error backtrace: #{error.backtrace.join("\n")};\n"
+        logger.error("Error: #{error_link} #{error_message} #{error_backtrace}")
+        report_store.record_crawler_error(link: link, error: error)
+      end
+    end
+  end
+end

data/lib/browser_crawler/engine_utilities/inspect_page_process.rb

@@ -0,0 +1,74 @@
+require_relative '../hooks_operator'
+
+module BrowserCrawler
+  module EngineUtilities
+    # Inspect a passed link and update the loop queue if it is necessary.
+    class InspectPageProcess
+      include Capybara::DSL
+      include HooksOperator
+
+      attr_reader :page_inspector,
+                  :screenshot_operator,
+                  :link_inspector,
+                  :logger
+
+      def initialize(link_inspector:,
+                     capybara_session:,
+                     report_store:,
+                     screenshot_operator: nil,
+                     logger:)
+        @page_inspector = PageInspector.new(
+          link_inspector: link_inspector,
+          capybara_session: capybara_session,
+          report_store: report_store
+        )
+        @link_inspector = link_inspector
+        @screenshot_operator = screenshot_operator
+        @logger = logger
+      end
+
+      def call(unvisited_links_queue:)
+        visit_page
+        update_queue(unvisited_links_queue: unvisited_links_queue)
+      end
+
+      private
+
+      def add_to_queue?(links:)
+        links && !links.empty?
+      end
+
+      # returns array consists of unvisited_links
+      # if some hooks is existed to execute hooks instead of base behavior
+      def find_unvisited_links
+        exchange_on_hooks(type: :unvisited_links) do
+          @page_inspector.scan_result
+        end
+      end
+
+      def visit_page
+        logger.info("Visiting #{link_inspector.raw_link}")
+
+        @page_inspector.visit_page
+
+        @page_inspector.save_to_report(screenshot_operator: screenshot_operator)
+
+        logger
+          .info("#{@page_inspector.scan_result.size} links found on the page.")
+      end
+
+      def update_queue(unvisited_links_queue:)
+        unvisited_links = find_unvisited_links
+
+        logger
+          .info("#{unvisited_links.size} will add to unvisited links queue.")
+
+        return unless add_to_queue?(links: unvisited_links)
+
+        unvisited_links_queue.push(*unvisited_links).uniq!
+
+        logger.info("#{unvisited_links_queue.size} - current state the queue.")
+      end
+    end
+  end
+end