aranha 0.10.1 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77b72be05123a10faead5d82b596c579668f00716754447a025feb6635c76916
-  data.tar.gz: 5d5f6ccf53c73becdc225060175303259ec11c257dacc6c28eaff1426a0946c5
+  metadata.gz: dea7f9194a220cad9e2f168f4fa9a32b17353f45233eef8d0698788535317228
+  data.tar.gz: 40fa4399386bd6b1f2e52fc914cbb8fb436099f29c5493f404aa219b40cfb586
 SHA512:
-  metadata.gz: bcec515206067ecb5a7a1d749c76d594cce289dc33df4dcb57b8065b390ffa763c4c1838ef1bb9e587f318208256f9ba18c667de64a66415029b841ec965adb7
-  data.tar.gz: 0fdf294b6ee26c63a039c77cfa79ec5d16136b61ca89159817c885a31bb2bd36f8fa5a9e25c47a0c33dbd2a2cee6db8302087d671701ffea55c5a1fd32cc3399
+  metadata.gz: 1955d0bbaa9fffbe63c24814bad2e7f97a1c960e1b4e5041ca1775fc10a12e8eb3653c9d73c706011805de7af4608cad3a2f5a10548ae3434d91a199eb0b9255
+  data.tar.gz: 532b2cd21a97ae55873aa996815947cf3759212b0279081dac374a23d2f9f390b059b422dd9ee5e2a2627e0e66de524702ffd817c0910063b8c376b22e47b1f3

data/lib/aranha/selenium/driver_factory/base.rb

@@ -49,6 +49,10 @@ module Aranha
           option_value(:headless)
         end
 
+        def user_agent
+          options.fetch(:user_agent)
+        end
+
         private
 
         def option_value(key)

data/lib/aranha/selenium/driver_factory/chrome.rb

@@ -19,6 +19,7 @@ module Aranha
           r.add_argument('--headless') if headless?
           r.add_argument('--disable-popup-blocking')
           r.add_argument('--disable-translate')
+          r.add_argument("user-agent=#{user_agent}") if user_agent.present?
           r.add_preference(:download, prompt_for_download: false, default_directory: downloads_dir)
           r
         end

data/lib/aranha/selenium/driver_factory/firefox.rb

@@ -40,6 +40,7 @@ module Aranha
           r['browser.download.folderList'] = 2
           r['browser.helperApps.neverAsk.saveToDisk'] = auto_download_mime_types.join(';')
           r['pdfjs.disabled'] = true
+          r['general.useragent.override'] = user_agent if user_agent.present?
           r
         end
 

data/lib/aranha/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Aranha
-  VERSION = '0.10.1'
+  VERSION = '0.11.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.11.0
 platform: ruby
 authors:
 - Eduardo H. Bogoni
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-17 00:00:00.000000000 Z
+date: 2019-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: active_scaffold
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.4.41.1
+- !ruby/object:Gem::Dependency
+  name: aranha-parsers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: eac_ruby_utils
   requirement: !ruby/object:Gem::Requirement
@@ -149,23 +163,6 @@ files:
 - lib/aranha/engine.rb
 - lib/aranha/fixtures.rb
 - lib/aranha/fixtures/download.rb
-- lib/aranha/parsers.rb
-- lib/aranha/parsers/base.rb
-- lib/aranha/parsers/html.rb
-- lib/aranha/parsers/html/base.rb
-- lib/aranha/parsers/html/item.rb
-- lib/aranha/parsers/html/item_list.rb
-- lib/aranha/parsers/html/node.rb
-- lib/aranha/parsers/html/node/base.rb
-- lib/aranha/parsers/html/node/default.rb
-- lib/aranha/parsers/invalid_state_exception.rb
-- lib/aranha/parsers/source_address.rb
-- lib/aranha/parsers/source_address/file.rb
-- lib/aranha/parsers/source_address/hash_http_get.rb
-- lib/aranha/parsers/source_address/hash_http_post.rb
-- lib/aranha/parsers/source_address/http_get.rb
-- lib/aranha/parsers/spec/source_target_fixtures.rb
-- lib/aranha/parsers/spec/source_target_fixtures_example.rb
 - lib/aranha/processor.rb
 - lib/aranha/selenium.rb
 - lib/aranha/selenium/driver_factory.rb
@@ -205,5 +202,5 @@ specification_version: 4
 summary: Rails utilities for web crawling.
 test_files:
 - test/aranha_test.rb
-- test/test_helper.rb
 - test/integration/navigation_test.rb
+- test/test_helper.rb

data/lib/aranha/parsers.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-module Aranha
-  module Parsers
-    require 'aranha/parsers/base'
-    require 'aranha/parsers/html'
-    require 'aranha/parsers/invalid_state_exception'
-    require 'aranha/parsers/source_address'
-  end
-end

data/lib/aranha/parsers/base.rb

@@ -1,58 +0,0 @@
-# frozen_string_literal: true
-
-require 'open-uri'
-require 'fileutils'
-require 'aranha/parsers/source_address'
-
-module Aranha
-  module Parsers
-    class Base
-      LOG_DIR_ENVVAR = 'ARANHA_PARSERS_LOG_DIR'
-
-      attr_reader :source_address
-
-      def initialize(url)
-        @source_address = ::Aranha::Parsers::SourceAddress.new(url)
-        log_content(source_address.serialize, '-source-address')
-      end
-
-      delegate :url, to: :source_address
-
-      def content
-        s = source_address.content
-        log_content(s)
-        s
-      end
-
-      private
-
-      def log_content(content, suffix = '')
-        path = log_file(suffix)
-
-        return unless path
-        File.open(path, 'wb') { |file| file.write(content) }
-      end
-
-      def log_file(suffix)
-        dir = log_parsers_dir
-        return nil unless dir
-        f = ::File.join(dir, "#{self.class.name.parameterize}#{suffix}.log")
-        FileUtils.mkdir_p(File.dirname(f))
-        f
-      end
-
-      def log_parsers_dir
-        return ENV[LOG_DIR_ENVVAR] if ENV[LOG_DIR_ENVVAR]
-        return ::Rails.root.join('log', 'parsers') if rails_root_exist?
-        nil
-      end
-
-      def rails_root_exist?
-        ::Rails.root
-        true
-      rescue NameError
-        return false
-      end
-    end
-  end
-end

data/lib/aranha/parsers/html.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-module Aranha
-  module Parsers
-    module Html
-      require 'aranha/parsers/html/base'
-      require 'aranha/parsers/html/item'
-      require 'aranha/parsers/html/item_list'
-      require 'aranha/parsers/html/node'
-    end
-  end
-end

data/lib/aranha/parsers/html/base.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-require 'nokogiri'
-require 'aranha/parsers/base'
-require 'aranha/parsers/html/node/default'
-
-module Aranha
-  module Parsers
-    module Html
-      class Base < ::Aranha::Parsers::Base
-        class << self
-          def fields
-            @fields ||= []
-            @fields.dup
-          end
-
-          def field(name, type, xpath)
-            @fields ||= []
-            @fields << Field.new(name, type, xpath)
-          end
-
-          Field = Struct.new(:name, :type, :xpath)
-        end
-
-        def nokogiri
-          @nokogiri ||= Nokogiri::HTML(content, &:noblanks)
-        end
-
-        protected
-
-        def node_parser_class
-          ::Aranha::Parsers::Html::Node::Default
-        end
-
-        private
-
-        def node_parser
-          @node_parser ||= node_parser_class.new(fields)
-        end
-
-        def fields
-          self.class.fields.map { |f| [f.name, f.type, f.xpath] }
-        end
-      end
-    end
-  end
-end

data/lib/aranha/parsers/html/item.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-require 'aranha/parsers/html/base'
-
-module Aranha
-  module Parsers
-    module Html
-      class Item < Base
-        def data
-          @data ||= node_parser.parse(item_node)
-        end
-
-        def item_node
-          @item_node ||= begin
-            r = item_xpath ? nokogiri.at_xpath(item_xpath) : nokogiri
-            raise "Item node not found (Item xpath: #{item_xpath})" unless r
-            r
-          end
-        end
-      end
-    end
-  end
-end

data/lib/aranha/parsers/html/item_list.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-require 'aranha/parsers/html/base'
-
-module Aranha
-  module Parsers
-    module Html
-      class ItemList < Base
-        def data
-          count = 0
-          @data ||= nokogiri.xpath(items_xpath).map do |m|
-            count += 1
-            node_parser.parse(m)
-          end
-        rescue StandardError => e
-          raise StandardError, "#{e.message} (Count: #{count})"
-        end
-
-        def items_xpath
-          raise "Class #{self.class} has no method \"item_xpath\". Implement it"
-        end
-      end
-    end
-  end
-end

data/lib/aranha/parsers/html/node.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-module Aranha
-  module Parsers
-    module Html
-      module Node
-        require 'aranha/parsers/html/node/base'
-        require 'aranha/parsers/html/node/default'
-      end
-    end
-  end
-end

data/lib/aranha/parsers/html/node/base.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  module Parsers
-    module Html
-      module Node
-        class Base
-          attr_reader :fields
-
-          def initialize(fields)
-            @fields = fields
-          end
-
-          def parse(node)
-            Hash[fields.map { |f| [f[0], parse_field(node, f[2], f[1])] }]
-          end
-
-          private
-
-          def parse_field(node, xpath, parser_method)
-            value_method = "#{parser_method}_value"
-            return send(value_method, node, xpath) if respond_to?(value_method)
-
-            raise "Method \"#{value_method}\" not found in #{self.class}"
-          end
-        end
-      end
-    end
-  end
-end

data/lib/aranha/parsers/html/node/default.rb

@@ -1,93 +0,0 @@
-# frozen_string_literal: true
-
-require 'aranha/parsers/html/node/base'
-
-module Aranha
-  module Parsers
-    module Html
-      module Node
-        class Default < ::Aranha::Parsers::Html::Node::Base
-          def string_value(node, xpath)
-            if node.at_xpath(xpath)
-              node.at_xpath(xpath).text.to_s.tr("\u00A0", ' ').strip
-            else
-              ''
-            end
-          end
-
-          def quoted_value(node, xpath)
-            s = string_value(node, xpath)
-            return '' unless s
-
-            m = /\"([^\"]+)\"/.match(s)
-            return m[1] if m
-
-            ''
-          end
-
-          def integer_value(node, xpath)
-            r = string_value(node, xpath)
-            return nil if r.blank?
-
-            m = /\d+/.match(r)
-            raise "Integer not found in \"#{r}\"" unless m
-
-            m[0].to_i
-          end
-
-          def integer_optional_value(node, xpath)
-            r = string_value(node, xpath)
-            m = /\d+/.match(r)
-            m ? m[0].to_i : nil
-          end
-
-          def float_value(node, xpath)
-            parse_float(node, xpath, true)
-          end
-
-          def float_optional_value(node, xpath)
-            parse_float(node, xpath, false)
-          end
-
-          def array_value(node, xpath)
-            r = node.xpath(xpath).map { |n| n.text.strip }
-            r.join('|')
-          end
-
-          def join_value(node, xpath)
-            m = ''
-            node.xpath(xpath).each do |n|
-              m << n.text.strip
-            end
-            m
-          end
-
-          def duration_value(node, xpath)
-            m = /(\d+) m/.match(join_value(node, xpath))
-            m ? m[1].to_i : nil
-          end
-
-          def regxep(node, xpath, pattern)
-            s = string_value(node, xpath)
-            m = pattern.match(s)
-            return m if m
-
-            raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
-          end
-
-          private
-
-          def parse_float(node, xpath, required)
-            s = string_value(node, xpath)
-            m = /\d+(?:[\.\,](\d+))?/.match(s)
-            if m
-              m[0].sub(',', '.').to_f
-            elsif required
-              raise "Float value not found in \"#{s}\""
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/aranha/parsers/invalid_state_exception.rb

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  module Parsers
-    class InvalidStateException < StandardError
-    end
-  end
-end

data/lib/aranha/parsers/source_address.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-require 'yaml'
-require 'active_support/core_ext/module/delegation'
-require 'aranha/parsers/source_address/hash_http_get'
-require 'aranha/parsers/source_address/hash_http_post'
-require 'aranha/parsers/source_address/http_get'
-require 'aranha/parsers/source_address/file'
-
-module Aranha
-  module Parsers
-    class SourceAddress
-      class << self
-        SUBS = [
-          ::Aranha::Parsers::SourceAddress::HashHttpGet,
-          ::Aranha::Parsers::SourceAddress::HashHttpPost,
-          ::Aranha::Parsers::SourceAddress::HttpGet,
-          ::Aranha::Parsers::SourceAddress::File
-        ].freeze
-
-        def detect_sub(source)
-          return source.sub if source.is_a?(self)
-          SUBS.each do |sub|
-            return sub.new(source) if sub.valid_source?(source)
-          end
-          raise "No content fetcher found for source \"#{source}\""
-        end
-
-        def deserialize(string)
-          new(string =~ %r{\A[a-z]+://} ? string.strip : ::YAML.load(string))
-        end
-
-        def from_file(path)
-          deserialize(::File.read(path))
-        end
-      end
-
-      attr_reader :sub
-
-      def initialize(source)
-        @sub = self.class.detect_sub(source)
-      end
-
-      delegate :content, :url, to: :sub
-
-      def to_s
-        sub.url
-      end
-
-      def serialize
-        sub.serialize.strip + "\n"
-      end
-    end
-  end
-end

data/lib/aranha/parsers/source_address/file.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-require 'aranha/parsers/source_address/http_get'
-
-module Aranha
-  module Parsers
-    class SourceAddress
-      class File < ::Aranha::Parsers::SourceAddress::HttpGet
-        SCHEME = 'file://'
-
-        class << self
-          def valid_source?(source)
-            source.to_s.start_with?(SCHEME + '/', '/')
-          end
-        end
-
-        def initialize(source)
-          super source.to_s.gsub(/\A#{Regexp.quote(SCHEME)}/, '')
-        end
-
-        def url
-          "#{SCHEME}#{source}"
-        end
-
-        def content
-          ::File.open(source, &:read)
-        end
-      end
-    end
-  end
-end

data/lib/aranha/parsers/source_address/hash_http_get.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-require 'aranha/parsers/source_address/hash_http_post'
-
-module Aranha
-  module Parsers
-    class SourceAddress
-      class HashHttpGet < ::Aranha::Parsers::SourceAddress::HashHttpPost
-        class << self
-          def valid_source?(source)
-            source.is_a?(::Hash) &&
-              source.with_indifferent_access[:method].to_s.downcase.strip == 'get'
-          end
-        end
-
-        def content
-          HTTPClient.new.get_content(
-            source[:url],
-            source[:params]
-          )
-        end
-      end
-    end
-  end
-end

data/lib/aranha/parsers/source_address/hash_http_post.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-require 'active_support/core_ext/hash/indifferent_access'
-require 'httpclient'
-require 'yaml'
-
-module Aranha
-  module Parsers
-    class SourceAddress
-      class HashHttpPost
-        class << self
-          def valid_source?(source)
-            source.is_a?(::Hash) &&
-              source.with_indifferent_access[:method].to_s.downcase.strip == 'post'
-          end
-        end
-
-        attr_reader :source
-
-        def initialize(source)
-          @source = source.with_indifferent_access
-        end
-
-        def ==(other)
-          self.class == other.class && source == other.source
-        end
-
-        def url
-          source.fetch(:url)
-        end
-
-        def serialize
-          source.to_yaml
-        end
-
-        def content
-          HTTPClient.new.post_content(
-            source[:url],
-            source[:params].merge(follow_redirect: true)
-          )
-        end
-      end
-    end
-  end
-end

data/lib/aranha/parsers/source_address/http_get.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-require 'addressable'
-require 'net/http'
-
-module Aranha
-  module Parsers
-    class SourceAddress
-      class HttpGet
-        class << self
-          def location_uri(source_uri, location)
-            ::Addressable::URI.join(source_uri, location).to_s
-          end
-
-          def valid_source?(source)
-            source.to_s =~ %r{\Ahttps?://}
-          end
-        end
-
-        attr_reader :source
-
-        def initialize(source)
-          @source = source.to_s
-        end
-
-        def ==(other)
-          self.class == other.class && source == other.source
-        end
-
-        def url
-          source
-        end
-
-        def content
-          content_fetch(url)
-        end
-
-        def serialize
-          url
-        end
-
-        private
-
-        def content_fetch(uri, limit = 10)
-          raise 'too many HTTP redirects' if limit.zero?
-
-          response = Net::HTTP.get_response(URI(uri))
-
-          case response
-          when Net::HTTPSuccess then
-            response.body
-          when Net::HTTPRedirection then
-            content_fetch(self.class.location_uri(uri, response['location']), limit - 1)
-          else
-            response.value
-          end
-        end
-      end
-    end
-  end
-end

data/lib/aranha/parsers/spec/source_target_fixtures.rb

@@ -1,67 +0,0 @@
-# encoding: UTF-8
-# frozen_string_literal: true
-
-require 'yaml'
-
-module Aranha
-  module Spec
-    # Lists pairs of source/target files in a directory.
-    class SourceTargetFixtures
-      class << self
-        def source_target_basename(file)
-          m = /^(.+)\.(?:source|target)(?:\..+)?$/.match(File.basename(file))
-          m ? m[1] : nil
-        end
-      end
-
-      attr_reader :fixtures_directory
-
-      def initialize(fixtures_directory)
-        @fixtures_directory = fixtures_directory
-      end
-
-      def source_target_files
-        sources_targets_basenames.map do |basename|
-          OpenStruct.new(source: source_file(basename), target: target_file(basename))
-        end
-      end
-
-      def source_files
-        r = []
-        source_target_files.each do |st|
-          r << st.source if st.source
-        end
-        r
-      end
-
-      def target_file(basename)
-        fixture_file(basename, 'target')
-      end
-
-      def source_file(basename)
-        fixture_file(basename, 'source')
-      end
-
-      private
-
-      def fixture_file(basename, suffix)
-        prefix = "#{basename}.#{suffix}"
-        Dir.foreach(fixtures_directory) do |item|
-          next if item == '.' || item == '..'
-          return File.expand_path(item, fixtures_directory) if item.starts_with?(prefix)
-        end
-        nil
-      end
-
-      def sources_targets_basenames
-        basenames = Set.new
-        Dir.foreach(fixtures_directory) do |item|
-          next if item == '.' || item == '..'
-          b = self.class.source_target_basename(item)
-          basenames << b if b.present?
-        end
-        basenames
-      end
-    end
-  end
-end

data/lib/aranha/parsers/spec/source_target_fixtures_example.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'source_target_fixtures'
-
-RSpec.shared_examples 'source_target_fixtures' do |spec_file| # rubocop:disable Metrics/BlockLength
-  let(:spec_file) { spec_file }
-
-  it 'fixtures directory should exist' do
-    expect(::File.directory?(fixtures_dir)).to be true
-  end
-
-  context 'in fixtures directory' do
-    it 'should have at least one file' do
-      expect(source_target_fixtures.source_target_files.count).to be > 0
-    end
-
-    if ENV['WRITE_TARGET_FIXTURES']
-      it 'should write target data for all files' do
-        source_target_fixtures.source_files.each do |source_file|
-          sd = sort_results(source_data(source_file))
-          basename = ::Aranha::Spec::SourceTargetFixtures.source_target_basename(source_file)
-          target_file = File.expand_path("../#{basename}.target.yaml", source_file)
-          File.write(target_file, sd.to_yaml)
-        end
-      end
-    else
-      it 'should parse data for all files' do
-        source_target_fixtures.source_target_files.each do |st|
-          assert_source_target_complete(st)
-          sd = source_data(st.source)
-          td = YAML.load_file(st.target)
-          expect(sort_results(sd)).to eq(sort_results(td))
-        end
-      end
-    end
-  end
-
-  def source_target_fixtures
-    @source_target_fixtures ||= ::Aranha::Spec::SourceTargetFixtures.new(fixtures_dir)
-  end
-
-  def assert_source_target_complete(st)
-    expect(st.source).to(be_truthy, "Source not found (Target: #{st.target})")
-    expect(st.target).to(be_truthy, "Target not found (Source: #{st.source})")
-  end
-
-  def source_data(source_file)
-    described_class.new(source_file).data
-  end
-
-  def fixtures_dir
-    ::File.join(
-      ::File.dirname(spec_file),
-      ::File.basename(spec_file, '.*') + '_files'
-    )
-  end
-
-  def sort_results(r)
-    r
-  end
-end