browser_crawler 0.4.0

data/lib/browser_crawler/engine_utilities/link_inspector.rb

@@ -0,0 +1,31 @@
+require_relative '../url_tools'
+
+module BrowserCrawler
+  module EngineUtilities
+    class LinkInspector
+      attr_reader :raw_link, :host_name, :uri
+
+      def initialize(raw_link:, host_name:)
+        @raw_link = raw_link
+        @host_name = host_name
+        @uri = UrlTools.uri(url: raw_link)
+      end
+
+      def external_url?
+        !internal_url?
+      end
+
+      def link_valid?
+        @link_valid ||= !uri.nil? && uri.host && uri.scheme
+      end
+
+      def internal_url?
+        @internal_url ||= !uri.nil? && uri.host == host_name
+      end
+
+      def full_url
+        @full_url ||= UrlTools.full_url(uri: uri)
+      end
+    end
+  end
+end

data/lib/browser_crawler/engine_utilities/link_scanner.rb

@@ -0,0 +1,38 @@
+module BrowserCrawler
+  module EngineUtilities
+    class LinkScanner
+      include Capybara::DSL
+      include HooksOperator
+
+      attr_reader :link_inspector
+
+      def initialize(link_inspector:)
+        @link_inspector = link_inspector
+      end
+
+      def scan(page:)
+        link_inspector.internal_url? ? get_page_links(page: page) : []
+      end
+
+      private
+
+      def get_page_links(page:)
+        remove_blank_links(link_matcher(page: page))
+      end
+
+      def remove_blank_links(links)
+        links.reject do |link|
+          link.nil? || link.empty?
+        end
+      end
+
+      # return Array consists of links from page
+      # if hooks exists when execute them instead of basic behavior
+      def link_matcher(page:)
+        exchange_on_hooks(type: :scan_rules) do
+          page.all('a').map { |a| a['href'] }
+        end
+      end
+    end
+  end
+end

data/lib/browser_crawler/engine_utilities/page_inspector.rb

@@ -0,0 +1,65 @@
+require_relative '../url_tools'
+require_relative '../hooks_operator'
+require_relative 'link_scanner'
+
+module BrowserCrawler
+  module EngineUtilities
+    class PageInspector
+      include Capybara::DSL
+      include HooksOperator
+
+      attr_accessor :link_inspector,
+                    :link_scanner,
+                    :capybara_session,
+                    :scan_result,
+                    :report_store
+
+      def initialize(link_inspector:, capybara_session:, report_store:)
+        @link_inspector = link_inspector
+        @capybara_session = capybara_session
+        @report_store = report_store
+        @scan_result = []
+        @link_scanner = LinkScanner.new(link_inspector: link_inspector)
+      end
+
+      def visit_page
+        uri = link_inspector.uri
+        Capybara.app_host = "#{uri.scheme}://#{uri.host}:#{uri.port}"
+
+        visit link_inspector.full_url
+
+        with_hooks_for(type: :each) do
+          @scan_result = scanning
+        end
+      end
+
+      def save_to_report(screenshot_operator: nil)
+        screenshot_path = save_screenshot(screenshot_operator)
+
+        report_store.record_page_visit(
+          page: link_inspector.full_url,
+          extracted_links: scan_result,
+          screenshot_filename: screenshot_path,
+          external: link_inspector.external_url?,
+          code: capybara_session.status_code
+        )
+      end
+
+      def before_page_scan; end
+
+      private
+
+      def scanning
+        link_scanner.scan(page: capybara_session) || []
+      end
+
+      def save_screenshot(screenshot_operator)
+        return unless screenshot_operator&.save_screenshots?
+
+        capybara_session.save_screenshot(
+          screenshot_operator.file_path(url: capybara_session.current_url)
+        )
+      end
+    end
+  end
+end

data/lib/browser_crawler/errors/invalid_hooks_type.rb

@@ -0,0 +1,12 @@
+module BrowserCrawler
+  module Errors
+    class InvalidHooksType < StandardError
+      def initialize(invalid_type:)
+        message = "Passed hooks type `#{invalid_type}` is invalid." \
+                  ' A type has to apply one of the follow values:' \
+                  " #{HooksContainer::VALID_TYPES.join(', ')}"
+        super(message)
+      end
+    end
+  end
+end

data/lib/browser_crawler/followups/screenshots_indexer.rb

@@ -0,0 +1,40 @@
+require 'erb'
+module BrowserCrawler
+  module Followups
+    # Indexes screenshots captured by the crawler, creates index.html from the captured screenshots.
+    # ERB Template can be provided that will receive the list of files.
+    class ScreenshotsIndexer
+      def initialize(template:)
+        @template = template || File.read(default_template_file)
+      end
+
+      # Produce index.html with links to screenshots found in the `path` specified.
+      # Optionally file_mask can be provided to filter out files to be indexed.
+      def index_directory(path, file_mask: '*.png')
+        files = Dir[File.join(path, file_mask)].map { |file| File.basename(file) }
+        html = render_index(files: files)
+        index_path = File.join(path, 'index.html')
+        File.write(index_path, html)
+        index_path
+      end
+
+      def index_report(report)
+        sorted_pages = Hash[report.pages.sort_by { |(k, _v)| k }]
+        files = Hash[sorted_pages.map do |(k, _v)|
+          k
+        end]
+      end
+
+      private
+
+      def default_template_file
+        File.join(__dir__, 'templates/index.html.erb')
+      end
+
+      def render_index(files:)
+        renderer = ERB.new(@template)
+        renderer.result(binding)
+      end
+    end
+  end
+end

data/lib/browser_crawler/followups/templates/index.html.erb

@@ -0,0 +1,69 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+<head>
+  <title>Screenshots</title>
+
+  <style type="text/css">
+    body {
+      color: #eee;
+      margin-top: 20px;
+      font-family: Arial, "Helvetica Neue", Helvetica, sans-serif;
+    }
+
+    a {
+      color: #FFF;
+    }
+
+    a:hover {
+      color: yellow;
+      text-decoration: underline;
+    }
+
+    .thumbnails {
+      overflow: scroll;
+      height: 150px;
+    }
+
+    .thumbnails img {
+      height: 80px;
+      border: 4px solid #555;
+      padding: 1px;
+      margin: 0 10px 10px 0;
+    }
+
+    .thumbnails img:hover {
+      border: 4px solid #00ccff;
+      cursor: pointer;
+    }
+
+    .preview img {
+      border: 4px solid #444;
+      padding: 1px;
+      width: 800px;
+    }
+  </style>
+
+</head>
+<body>
+
+<div class="gallery" align="center">
+  <h2>Screenshots captured</h2>
+
+  <br/>
+
+  <div class="thumbnails">
+    <% files.each_with_index do |img, index| %>
+        <img onmouseover="preview.src=img<%= index %>.src" name="img<%= index %>" src="<%= img %>" alt=""/>
+    <% end %>
+  </div>
+  <br/>
+
+  <div class="preview" align="center">
+    <img name="preview" src="<%= files.first %>" alt=""/>
+  </div>
+
+</div>
+
+
+</body>
+</html>

data/lib/browser_crawler/followups/wraith_integrator.rb

@@ -0,0 +1,41 @@
+require 'yaml'
+require 'active_support/core_ext/string'
+
+module BrowserCrawler
+  module Followups
+    # Updates the :paths section of the Wraith's config file.
+    class WraithIntegrator
+      def initialize(report:)
+        @report = if report.respond_to?(:pages)
+                    report
+                  else
+                    YAML.safe_load(report, [Symbol]).symbolize_keys
+                  end
+      end
+
+      def update_config(wraith_config_file, path_suffix: nil)
+        config = YAML.safe_load(File.read(wraith_config_file))
+        config['paths'] = paths(with_suffix: path_suffix)
+        File.write(wraith_config_file, config.to_yaml)
+      end
+
+      # @return [Hash] sorted hash of page_name => path pair values appended with optional suffix.
+      #     Page name equals to path which makes it easy to navigate the page from the Wraith gallery.
+      def paths(with_suffix: nil)
+        Hash[sorted_pages.map { |(k, v)| [k, "#{v}#{with_suffix}"] }]
+      end
+
+      def named_pages
+        @report[:pages].each_with_object({}) do |(page_url, _links), h|
+          page_path = URI(page_url.to_s).path
+          page_name = page_path.parameterize
+          h[page_name] = page_path
+        end
+      end
+
+      def sorted_pages
+        Hash[named_pages.sort_by { |(k, _v)| k }]
+      end
+    end
+  end
+end

data/lib/browser_crawler/hooks_container.rb

@@ -0,0 +1,31 @@
+require 'singleton'
+require_relative 'errors/invalid_hooks_type'
+
+module BrowserCrawler
+  class HooksContainer
+    include Singleton
+
+    VALID_TYPES = %i[each all unvisited_links scan_rules].freeze
+
+    def initialize
+      reset
+    end
+
+    def reset
+      @hooks_container = Hash.new { |h, k| h[k] = { each: [],
+                                                    all: [],
+                                                    unvisited_links: [],
+                                                    scan_rules: []} }
+    end
+
+    attr_reader :hooks_container
+
+    def add_hook(method: :run_only_one, type:, hook: nil)
+      unless VALID_TYPES.include?(type)
+        raise Errors::InvalidHooksType.new(invalid_type: type)
+      end
+
+      @hooks_container[method][type.to_sym] << hook
+    end
+  end
+end

data/lib/browser_crawler/hooks_operator.rb

@@ -0,0 +1,44 @@
+module BrowserCrawler
+  module HooksOperator
+    def with_hooks_for(type:)
+      run_before_hooks(type: type)
+      yield
+      run_after_hooks(type: type)
+    end
+
+    def exchange_on_hooks(type:, &default_block)
+      hooks_array = BrowserCrawler::HooksContainer
+              .instance.hooks_container[:run_only_one][type]
+
+      if hooks_array && !hooks_array.empty?
+        instance_exec(&hooks_array[0])
+      elsif block_given?
+        instance_exec(&default_block)
+      end
+    end
+
+    private
+
+    def run_before_hooks(type:)
+      before_hook = BrowserCrawler::HooksContainer.instance
+                      .hooks_container[:before][type]
+      return unless before_hook
+
+      run_hooks(before_hook)
+    end
+
+    def run_after_hooks(type:)
+      after_hook = BrowserCrawler::HooksContainer.instance
+                     .hooks_container[:after][type]
+      return unless after_hook
+
+      run_hooks(after_hook)
+    end
+
+    def run_hooks(hooks)
+      hooks.each do |hook|
+        instance_exec(&hook)
+      end
+    end
+  end
+end

data/lib/browser_crawler/options.rb

@@ -0,0 +1,86 @@
+require 'optionparser'
+
+module BrowserCrawler
+  module Options
+    module_function
+
+    def default_options
+      {
+        report_folder: 'tmp',
+        report_format: 'yaml',
+        window_width: 1024,
+        window_height: 768
+      }
+    end
+
+    def parse_args
+      options = {}
+      p = OptionParser.new do |opts|
+        opts.on_tail
+
+        opts.banner = 'Site crawler. Usage example: crawl http://localhost:3000'
+
+        opts.on('-U', '[--url] URL', 'Crawls the site starting from the url specified. E.g. http://localhost:3000/welcome.') do |v|
+          options[:url] = v
+        end
+
+        opts.on('-u', '--user USERNAME', 'The authentication user name (optional).') do |v|
+          options[:username] = v
+        end
+
+        opts.on('-p', '--password PASSWORD', 'The authentication password (optional).') do |v|
+          options[:password] = v
+        end
+
+        opts.on('-n', '--max_pages NUM', 'The maximum number of pages to visit.') do |v|
+          options[:max_pages] = v.to_i
+        end
+
+        opts.on('-w', '--window_size WxH', 'Browser window size. Default 1024x768') do |v|
+          options[:window_width], options[:window_height] = v.split('x')
+        end
+
+        opts.on('-r', '--report FOLDER', 'The folder path to save report to. '\
+                                         'Default: tmp') do |v|
+          options[:report_folder] = v
+        end
+
+        opts.on('-f', '--report_format TYPE', 'The report type to save result  '\
+                                         'Default: yaml') do |v|
+          options[:report_format] = v
+        end
+
+        opts.on('-s', '--screenshots_path PATH',
+                'If specified along with the url, screenshots are captured visiting each page.'\
+                ' Otherwise used to generate a screenshots index based on files caprured previously. ') do |v|
+          options[:screenshots_path] = v
+        end
+
+        opts.on('-t', '--template FILENAME',
+                'Specify the template used for indexing.'\
+                '  Default: followups/templates/index.html.erb') do |v|
+          options[:index_template] = v
+        end
+
+        opts.on('-c', '--wraith_config FILENAME',
+                'Update config "paths" section with the pages extracted.') do |v|
+          options[:wraith_config] = v
+        end
+
+        opts.on('-h', '--help', 'Show this help message and exit.') do
+          puts opts
+        end
+      end
+      p.parse!
+
+      options[:url] = ARGV.pop unless ARGV.empty?
+
+      if options.empty?
+        puts p
+        exit
+      end
+
+      default_options.merge(options)
+    end
+  end
+end