browser_crawler 0.4.0

data/lib/browser_crawler/report_factory.rb

@@ -0,0 +1,22 @@
+require 'fileutils'
+require_relative 'reports/csv_report'
+require_relative 'reports/yaml_report'
+
+module BrowserCrawler
+  # It saves store data to yaml or csv report file.
+  module ReportFactory
+    module_function
+
+    REPORT_MATCHER = {
+      yaml: Reports::YamlReport,
+      csv: Reports::CsvReport
+    }.freeze
+
+    def save(store:, type:, save_folder_path:)
+      FileUtils.mkdir_p(save_folder_path)
+      REPORT_MATCHER[type]
+        .new(store: store)
+        .export(save_folder_path: save_folder_path)
+    end
+  end
+end

data/lib/browser_crawler/reports/csv_report.rb

@@ -0,0 +1,75 @@
+require 'csv'
+
+module BrowserCrawler
+  module Reports
+    # It involves methods which allow to save a store to a csv file
+    class CsvReport
+      def initialize(store:)
+        @store = store
+      end
+
+      def export(save_folder_path:)
+        CSV.open("#{save_folder_path}/crawler_report.csv", 'wb') do |csv|
+          csv << ['pages',
+                  'extracted links',
+                  'is external',
+                  'http status',
+                  'http code']
+
+          @store.pages.each do |page, crawler_result|
+            save_to_csv(csv, page, crawler_result)
+          end
+        end
+      end
+
+      private
+
+      def filter_links(links)
+        return nil if links.nil?
+
+        links.select do |link|
+          link =~ /\A#{URI.regexp(%w[http https])}\z/
+        end
+      end
+
+      def save_to_row(page, crawler_result, link = nil)
+        [page,
+         link,
+         crawler_result[:external],
+         humanize_code(crawler_result[:code]),
+         crawler_result[:code]]
+      end
+
+      def save_to_csv(csv, page, crawler_result)
+        extracted_links = filter_links(crawler_result[:extracted_links])
+
+        if extracted_links.nil? || extracted_links.empty?
+          csv << save_to_row(page, crawler_result)
+          return
+        end
+
+        extracted_links.each do |link|
+          csv << save_to_row(page, crawler_result, link)
+        end
+      end
+
+      def csv_header
+        ['pages',
+         'extracted links',
+         'external?',
+         'http status',
+         'http code']
+      end
+
+      def humanize_code(code)
+        case code.to_i
+        when 200..225 then :active
+        when 401 then :unauthorized
+        when 301..308 then :redirect
+        else
+          :broken
+        end
+      end
+    end
+  end
+end

data/lib/browser_crawler/reports/store.rb

@@ -0,0 +1,114 @@
+module BrowserCrawler
+  module Reports
+    # Value object that contains crawling results.
+    # Example:
+    # {
+    #   pages: {
+    #     '/':
+    #       {
+    #         screenshot: 'file1.png',
+    #         error: nil,
+    #         extracted_links: ['http://welcome[pdf]', 'http://support']
+    #       },
+    #     'welcome':
+    #       {
+    #         screenshot: 'file2.png',
+    #         error: 'Invalid URI',
+    #         extracted_links: nil
+    #       }
+    #   },
+    #   metadata: {
+    #     custom_attribute: 'Sample report title'
+    #   },
+    #   unrecognized_links: ['mailto://', 'javascript://'],
+    #   crawler_error: {
+    #     'http://welcome.page' => {
+    #       message: 'Something has a wrong type',
+    #       backtrace: ['/call:10', '/sum: 11']
+    #     }
+    #   },
+    #   started_at: 12345,
+    #   finished_at: 123456
+    # }
+
+    # It involves methods which allow to save data to a store structure
+    class Store
+      attr_reader :pages, :metadata, :unrecognized_links, :crawler_error
+      attr_accessor :error
+
+      def initialize(pages: {},
+                     metadata: {},
+                     started_at: nil,
+                     finished_at: nil)
+        @pages = pages
+        @metadata = metadata
+        @started_at = started_at
+        @finished_at = finished_at
+        @crawler_error = {}
+        @unrecognized_links = []
+      end
+
+      def start(url:)
+        @pages.clear
+        @started_at = Time.now
+        @metadata[:url] = url
+      end
+
+      def finish
+        @finished_at = Time.now
+      end
+
+      def to_h
+        {}.merge(pages: @pages)
+          .merge(@metadata)
+          .merge(
+            unrecognized_links: @unrecognized_links,
+            crawler_error: @crawler_error,
+            started_at: @started_at,
+            finished_at: @finished_at,
+            links_count: count_all_links
+          )
+      end
+
+      def record_unrecognized_link(link)
+        return if @unrecognized_links.include?(link)
+
+        @unrecognized_links << link unless @unrecognized_links.include?(link)
+      end
+
+      def record_page_visit(page:,
+                            extracted_links: nil,
+                            screenshot_filename: nil,
+                            error: nil,
+                            external: false,
+                            code: nil)
+        @pages[page] = {
+          screenshot: screenshot_filename,
+          error: error,
+          extracted_links: extracted_links,
+          code: code,
+          external: external
+        }
+      end
+
+      def record_crawler_error(link:, error:)
+        @crawler_error[link] = {
+          message: error.message,
+          backtrace: error.backtrace
+        }
+      end
+
+      def visited_pages
+        @pages.keys
+      end
+
+      private
+
+      def count_all_links
+        @pages.inject(0) do |sum, (_, data)|
+          sum + data[:extracted_links]&.size.to_i if data && data[:extracted_links]
+        end
+      end
+    end
+  end
+end

data/lib/browser_crawler/reports/yaml_report.rb

@@ -0,0 +1,15 @@
+module BrowserCrawler
+  module Reports
+    # It involves methods which allow to save a store to an yaml file
+    class YamlReport
+      def initialize(store:)
+        @store = store
+      end
+
+      def export(save_folder_path:)
+        File.write("#{save_folder_path}/crawler_report.yaml",
+                   @store.to_h.to_yaml)
+      end
+    end
+  end
+end

data/lib/browser_crawler/screenshot_operator.rb

@@ -0,0 +1,47 @@
+module BrowserCrawler
+  # Control operations on screenshots
+  class ScreenshotOperator
+    attr_reader :format, :save_screenshots, :filename_base, :screenshots_folder
+
+    def initialize(save_screenshots: false,
+                   save_screenshots_to: nil,
+                   format: 'png',
+                   filename: nil)
+      @screenshots_folder = save_screenshots_to
+      @format = format
+      @save_screenshots = save_screenshots
+      @filename_base = filename || 'screenshot'
+    end
+
+    def save_screenshots?
+      [screenshots_folder, save_screenshots].any?
+    end
+
+    def file_path(url: nil)
+      "#{save_path}/#{filename(url: url)}"
+    end
+
+    def filename(url: nil)
+      if !filename_base_default? || url.nil?
+        "#{filename_prefix}_#{filename_base}.#{format}"
+      else
+        path = UrlTools.uri(url: url)&.path&.gsub('/', '%')&.gsub('.', '')
+        "#{filename_prefix}_#{path}.#{format}"
+      end
+    end
+
+    private
+
+    def filename_base_default?
+      filename_base == 'screenshot'
+    end
+
+    def save_path
+      screenshots_folder || File.join(Dir.pwd, 'tmp', 'screenshots')
+    end
+
+    def filename_prefix
+      Time.now.getutc.to_s.tr(' ', '_')
+    end
+  end
+end

data/lib/browser_crawler/support/capybara.rb

@@ -0,0 +1,20 @@
+require 'capybara'
+require 'capybara/cuprite'
+
+# Register new driver for capybara
+module Capybara
+  module_function
+
+  def register_chrome_driver(name, options: {})
+    unless options[:browser_options]
+      options[:browser_options] = {
+        '--headless' => nil, '--disable-gpu' => nil,
+        '--disable-extensions' => nil, '--no-sandbox' => nil
+      }
+    end
+
+    Capybara.register_driver name do |app|
+      ::Capybara::Cuprite::Driver.new(app, options)
+    end
+  end
+end

data/lib/browser_crawler/url_tools.rb

@@ -0,0 +1,32 @@
+module BrowserCrawler
+  module UrlTools
+    def uri(url:)
+      uri!(url: url)
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def uri!(url:)
+      string_url = url.to_s
+      raise URI::InvalidURIError unless string_url =~ /\A#{URI.regexp(%w[http https])}\z/
+
+      URI(string_url)
+    end
+
+    def full_url(uri:)
+      path_query = get_path_query(uri: uri)
+      if uri.port == 80 || uri.port == 443
+        "#{uri.scheme}://#{uri.host}#{uri.path}#{path_query}"
+      else
+        "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}#{path_query}"
+      end.sub(%r{(/)+$}, '')
+    end
+
+    def get_path_query(uri:)
+      uri_fragment = uri.query
+      uri_fragment.nil? || (uri_fragment == '') ? nil : "?#{uri.query}"
+    end
+
+    module_function :uri, :uri!, :full_url, :get_path_query
+  end
+end

data/lib/browser_crawler/version.rb

@@ -0,0 +1,3 @@
+module BrowserCrawler
+  VERSION = '0.4.0'.freeze
+end

metadata

@@ -0,0 +1,244 @@
+--- !ruby/object:Gem::Specification
+name: browser_crawler
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- Dmytro Samodurov
+- Artem Rumiantcev
+- Denys Ivanchuk
+- Sergiy Tyatin
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-08-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.2
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.24.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.24'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.24.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.24'
+- !ruby/object:Gem::Dependency
+  name: chromedriver-helper
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: cuprite
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.17.2
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.17.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.17.2
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.17.2
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.6'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.6'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.66'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.66'
+description: ''
+email:
+- dimasamodurov@gmail.com
+- tema.place@gmail.com
+executables:
+- console
+- crawl
+- setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/crawl
+- bin/setup
+- browser_crawler.gemspec
+- lib/browser_crawler.rb
+- lib/browser_crawler/dsl/js_helpers.rb
+- lib/browser_crawler/dsl/sign_in.rb
+- lib/browser_crawler/engine.rb
+- lib/browser_crawler/engine_utilities/crawl_manager.rb
+- lib/browser_crawler/engine_utilities/inspect_page_process.rb
+- lib/browser_crawler/engine_utilities/link_inspector.rb
+- lib/browser_crawler/engine_utilities/link_scanner.rb
+- lib/browser_crawler/engine_utilities/page_inspector.rb
+- lib/browser_crawler/errors/invalid_hooks_type.rb
+- lib/browser_crawler/followups/screenshots_indexer.rb
+- lib/browser_crawler/followups/templates/index.html.erb
+- lib/browser_crawler/followups/wraith_integrator.rb
+- lib/browser_crawler/hooks_container.rb
+- lib/browser_crawler/hooks_operator.rb
+- lib/browser_crawler/options.rb
+- lib/browser_crawler/report_factory.rb
+- lib/browser_crawler/reports/csv_report.rb
+- lib/browser_crawler/reports/store.rb
+- lib/browser_crawler/reports/yaml_report.rb
+- lib/browser_crawler/screenshot_operator.rb
+- lib/browser_crawler/support/capybara.rb
+- lib/browser_crawler/url_tools.rb
+- lib/browser_crawler/version.rb
+homepage: https://github.com/DimaSamodurov/browser_crawler
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/DimaSamodurov/browser_crawler
+  source_code_uri: https://github.com/DimaSamodurov/browser_crawler
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.1
+signing_key: 
+specification_version: 4
+summary: Simple site crawler using Capybara
+test_files: []