aranha 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db4987f0c4c725a35cddd376afbd950d541491d700e3f392bb4def95d2885a24
-  data.tar.gz: f5d84a48f26780e4a9c84c623a150ccd1528afdd0c3b39c23580bbe22dbe4896
+  metadata.gz: e3050cda5e754315e7b80518851f3ce91aa167e8ca8359084203ed61b20c7aa7
+  data.tar.gz: 80525648edf9ba10f99f7ab2bd684a0916ce91ea20892550b8bb6f19d092b7d7
 SHA512:
-  metadata.gz: 9ec0aa19303a62e0f260da849f55524d14f539b522b4944f28fb399698a60bc6cb4f2f3c10aaf5782c76ba1d5c5102d1350a6aaf7fd050a2a3969d3ced028ba9
-  data.tar.gz: f098aeb43bce8afe3a9330667ef0e34805852737857bf54f066c4ad976b48b2b4ed05d8a36d3c560f41640596889a30604e239a0c30e8a4d9bb8b0a165e9b589
+  metadata.gz: 1b2036fa6ebc24937ac23fda40a8aacbf0a89c3f233d17ddf82c15ecdf1c26bbe36b39b04eeb2f4fa8bfcaa0170477a46338bb74f8ba356d84c591e59944c17c
+  data.tar.gz: 919372af358638d177b31f61dfe65b41a7d7cb81cb50d235b818e63c8de306d102fc0508143dfa6b4da6fffa23ad02d1889f4e364ffc3f5ddb56082552a629a1

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 begin
   require 'bundler/setup'
 rescue LoadError
@@ -14,14 +16,11 @@ RDoc::Task.new(:rdoc) do |rdoc|
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 
-APP_RAKEFILE = File.expand_path("../test/dummy/Rakefile", __FILE__)
+APP_RAKEFILE = File.expand_path('test/dummy/Rakefile', __dir__)
 load 'rails/tasks/engine.rake'
 
-
 load 'rails/tasks/statistics.rake'
 
-
-
 Bundler::GemHelper.install_tasks
 
 require 'rake/testtask'
@@ -33,5 +32,4 @@ Rake::TestTask.new(:test) do |t|
   t.verbose = false
 end
 
-
 task default: :test

data/app/controllers/aranha/addresses_controller.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require_dependency 'aranha/application_controller'
 
 module Aranha

data/app/controllers/aranha/application_controller.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aranha
   class ApplicationController < ActionController::Base
     protect_from_forgery with: :exception

data/app/helpers/aranha/application_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aranha
   module ApplicationHelper
   end

data/app/models/aranha/address.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Aranha
   class Address < ActiveRecord::Base
     include ::Eac::InequalityQueries

data/config/routes.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Aranha::Engine.routes.draw do
   resources(:addresses) { as_routes }
 end

data/db/migrate/20171201021251_create_aranha_addresses.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 class CreateAranhaAddresses < ActiveRecord::Migration
   def change
     create_table :aranha_addresses do |t|

data/lib/aranha.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'httpclient'
 require 'active_support/dependencies'
 require_dependency 'aranha/engine'
@@ -7,4 +8,9 @@ require_dependency 'active_scaffold'
 module Aranha
 end
 
+require_dependency 'aranha/default_processor'
 require_dependency 'aranha/processor'
+require_dependency 'aranha/parsers/base'
+require_dependency 'aranha/parsers/html/base'
+require_dependency 'aranha/parsers/html/item_list'
+require_dependency 'aranha/dom_elements_traverser'

data/lib/aranha/default_processor.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Aranha
+  class DefaultProcessor
+    attr_reader :source_uri
+
+    def initialize(source_uri)
+      unless source_uri.is_a?(Addressable::URI)
+        source_uri = source_uri.to_s.gsub(%r{\A/}, 'file:///')
+      end
+      @source_uri = Addressable::URI.parse(source_uri)
+    end
+
+    def process
+      raise 'Implement method process'
+    end
+
+    protected
+
+    def target_uri
+      source_uri
+    end
+
+    def data
+      @data ||= parser_class.new(target_uri).data
+    end
+
+    def parser_class
+      r = self.class.name.gsub('::Processors::', '::Parsers::').constantize
+      return r unless is_a?(r)
+
+      raise "Parser can be not the process class: #{r}"
+    end
+  end
+end

data/lib/aranha/dom_elements_traverser.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_dependency 'aranha/dom_elements_traverser/conditions'
+require_dependency 'aranha/dom_elements_traverser/data'
+require_dependency 'aranha/dom_elements_traverser/cursor'
+
+module Aranha
+  class DomElementsTraverser
+    include ::Aranha::DomElementsTraverser::Conditions
+    include ::Aranha::DomElementsTraverser::Cursor
+    include ::Aranha::DomElementsTraverser::Data
+
+    class << self
+      def traverse(options, &block)
+        new(elements_from_options(options), &block)
+      end
+
+      def empty
+        new([])
+      end
+
+      private
+
+      def elements_from_options(options)
+        options = ::EacRubyUtils::OptionsConsumer.new(options)
+        elements = nil
+        options.consume(:children_of) { |v| elements = v.children.to_a }
+        raise 'None option of [:children_of] defined' unless elements
+
+        options.validate
+        elements
+      end
+    end
+
+    private
+
+    def initialize(elements, &block)
+      @elements = elements
+      @index = 0
+      @data = {}
+      instance_eval(&block) if block
+    end
+  end
+end

data/lib/aranha/dom_elements_traverser/conditions.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Aranha
+  class DomElementsTraverser
+    module Conditions
+      private
+
+      def match_conditions?(conditions)
+        raise "No element (Conditions: #{conditions})" unless current
+
+        conditions.all? { |key, value| match_condition?(key, value) }
+      end
+
+      def match_condition?(key, value)
+        case key.to_sym
+        when :text then match_text_condition?(value)
+        when :name then match_name_condition?(value)
+        else raise "Unknown key condition: (#{key})"
+        end
+      end
+
+      def match_name_condition?(tag_name)
+        current.name.casecmp(tag_name.to_s).zero?
+      end
+
+      def match_text_condition?(texts)
+        texts = [texts.to_s] unless texts.is_a?(Array)
+        texts.all? { |t| current.text.downcase.include?(t.downcase) }
+      end
+    end
+  end
+end

data/lib/aranha/dom_elements_traverser/cursor.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Aranha
+  class DomElementsTraverser
+    module Cursor
+      private
+
+      def current
+        @elements[@index]
+      end
+
+      def skip
+        @index += 1
+      end
+
+      def skip_until(options)
+        oc = ::EacRubyUtils::OptionsConsumer.new(options)
+        optional = oc.consume(:optional, false)
+        while current
+          break if match_conditions?(oc.left_data)
+
+          skip
+        end
+        raise "No element found for conditions #{oc.left_data}" unless current || optional
+
+        current
+      end
+
+      def skip_until_after(conditions)
+        skip_until(conditions)
+        skip
+        current
+      end
+
+      def if_found(conditions, &block)
+        marked = @index
+        skip_until({ optional: true }.merge(conditions))
+        if current
+          instance_eval(&block) if block
+        else
+          @index = marked
+        end
+      end
+    end
+  end
+end

data/lib/aranha/dom_elements_traverser/data.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Aranha
+  class DomElementsTraverser
+    module Data
+      def data
+        @data.dup
+      end
+
+      private
+
+      def store(key, options = {}, &converter)
+        validate(options)
+        value = store_value(options, converter)
+        @data[key] = value
+        r = current
+        skip
+        r
+      end
+
+      def store_value(options, converter)
+        value = if options.key?(:attribute)
+                  current.attribute(options[:attribute]).value
+                else
+                  current.text.strip
+                end
+        converter ? converter.call(value) : value
+      end
+
+      def validate(options)
+        return unless options.key?(:validate)
+        return if match_conditions?(options[:validate])
+
+        raise "Element does not match conditions #{options[:validate]}" \
+          " (Element: |#{current}|#{current.name}|)"
+      end
+    end
+  end
+end

data/lib/aranha/engine.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aranha
   class Engine < ::Rails::Engine
     isolate_namespace Aranha

data/lib/aranha/parsers/base.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'open-uri'
+require 'fileutils'
+
+module Aranha
+  module Parsers
+    class Base
+      def initialize(url)
+        @url = url
+      end
+
+      def url
+        @url.gsub(%r{/+$}, '')
+      end
+
+      def content
+        s = content_by_url_type
+        log_content(s)
+        s
+      end
+
+      private
+
+      def content_by_url_type
+        if @url.is_a?(Hash)
+          content_hash
+        elsif /^http/ =~ @url
+          content_get
+        else
+          content_file
+        end
+      end
+
+      def content_file
+        ::File.open(@url.gsub(%r{\Afile://}, ''), &:read)
+      end
+
+      def content_get
+        content_get_fetch(@url)
+      end
+
+      def content_get_fetch(uri, limit = 10)
+        raise 'too many HTTP redirects' if limit.zero?
+
+        response = Net::HTTP.get_response(URI(uri))
+
+        case response
+        when Net::HTTPSuccess then
+          response.body
+        when Net::HTTPRedirection then
+          content_get_fetch(response['location'], limit - 1)
+        else
+          response.value
+        end
+      end
+
+      def content_hash
+        return content_post if @url[:method] == :post
+
+        raise "Unknown URL format: #{@url}"
+      end
+
+      def content_post
+        HTTPClient.new.post_content(@url[:url], @url[:params].merge(follow_redirect: true))
+      end
+
+      def log_content(content)
+        File.open(log_file, 'wb') { |file| file.write(content) }
+      end
+
+      def log_file
+        f = Rails.root.join('log', 'parsers', "#{self.class.name.parameterize}.log")
+        FileUtils.mkdir_p(File.dirname(f))
+        f
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/base.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_dependency 'aranha/parsers/base'
+require_dependency 'aranha/parsers/html/node/default'
+
+module Aranha
+  module Parsers
+    module Html
+      class Base < ::Aranha::Parsers::Base
+        def nokogiri
+          @nokogiri ||= Nokogiri::HTML(content, &:noblanks)
+        end
+
+        protected
+
+        def node_parser_class
+          ::Aranha::Parsers::Html::Node::Default
+        end
+
+        private
+
+        def node_parser
+          @node_parser ||= node_parser_class.new(fields)
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/item_list.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Aranha
+  module Parsers
+    module Html
+      class ItemList < Base
+        def data
+          count = 0
+          @data ||= nokogiri.xpath(items_xpath).map do |m|
+            count += 1
+            node_parser.parse(m)
+          end
+        rescue StandardError => e
+          e.message << " / Count: #{count}"
+          raise e
+        end
+
+        def items_xpath
+          raise "Class #{self.class} has no method \"item_xpath\". Implement it"
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node/base.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Base
+          attr_reader :fields
+
+          def initialize(fields)
+            @fields = fields
+          end
+
+          def parse(node)
+            Hash[fields.map { |f| [f[0], parse_field(node, f[2], f[1])] }]
+          end
+
+          private
+
+          def parse_field(node, xpath, parser_method)
+            value_method = "#{parser_method}_value"
+            return send(value_method, node, xpath) if respond_to?(value_method)
+
+            raise "Method \"#{value_method}\" not found in #{self.class}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node/default.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require_dependency 'aranha/parsers/html/node/base'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          def string_value(node, xpath)
+            if node.at_xpath(xpath)
+              node.at_xpath(xpath).text.to_s.tr("\u00A0", ' ').strip
+            else
+              ''
+            end
+          end
+
+          def quoted_value(node, xpath)
+            s = string_value(node, xpath)
+            return '' unless s
+
+            m = /\"([^\"]+)\"/.match(s)
+            return m[1] if m
+
+            ''
+          end
+
+          def integer_value(node, xpath)
+            r = string_value(node, xpath)
+            return nil if r.blank?
+
+            m = /\d+/.match(r)
+            raise "Integer not found in \"#{r}\"" unless m
+
+            m[0].to_i
+          end
+
+          def integer_optional_value(node, xpath)
+            r = string_value(node, xpath)
+            m = /\d+/.match(r)
+            m ? m[0].to_i : nil
+          end
+
+          def float_value(node, xpath)
+            parse_float(node, xpath, true)
+          end
+
+          def float_optional_value(node, xpath)
+            parse_float(node, xpath, false)
+          end
+
+          def array_value(node, xpath)
+            r = node.xpath(xpath).map { |n| n.text.strip }
+            r.join('|')
+          end
+
+          def join_value(node, xpath)
+            m = ''
+            node.xpath(xpath).each do |n|
+              m << n.text.strip
+            end
+            m
+          end
+
+          def duration_value(node, xpath)
+            m = /(\d+) m/.match(join_value(node, xpath))
+            m ? m[1].to_i : nil
+          end
+
+          def regxep(node, xpath, pattern)
+            s = string_value(node, xpath)
+            m = pattern.match(s)
+            return m if m
+
+            raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
+          end
+
+          private
+
+          def parse_float(node, xpath, required)
+            s = string_value(node, xpath)
+            m = /\d+(?:[\.\,](\d+))?/.match(s)
+            if m
+              m[0].sub(',', '.').to_f
+            elsif required
+              raise "Float value not found in \"#{s}\""
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/processor.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
+
 require 'net/http'
 
 module Aranha
   class Processor
     NETWORK_EXCEPTIONS = [::HTTPClient::BadResponseError, Errno::ECONNRESET,
-                          ::Net::HTTPFatalError].freeze
+                          ::Net::HTTPFatalError, ::HTTPClient::ReceiveTimeoutError].freeze
     DEFAULT_MAX_TRIES = 3
 
     def initialize
@@ -32,28 +33,29 @@ module Aranha
         false
       elsif @failed.any?
         @try += 1
-        max_tries > 0 && @try >= max_tries
+        max_tries.positive? && @try >= max_tries
       else
         true
       end
     end
 
-    def process_address(a)
-      Rails.logger.info("Processing #{a} (Try: #{@try}/#{max_tries_s}," \
+    def process_address(address)
+      Rails.logger.info("Processing #{address} (Try: #{@try}/#{max_tries_s}," \
           " Unprocessed: #{unprocessed.count}/#{Aranha::Address.count})")
       begin
-        a.process
-        @failed.delete(a.id)
+        address.process
+        @failed.delete(address.id)
       rescue StandardError => ex
-        process_exception(a, ex)
+        process_exception(address, ex)
       end
     end
 
-    def process_exception(a, ex)
-      raise ex unless network_exception?(ex)
-      @failed[a.id] ||= 0
-      @failed[a.id] += 1
-      Rails.logger.warn(ex)
+    def process_exception(address, exception)
+      raise exception unless network_exception?(exception)
+
+      @failed[address.id] ||= 0
+      @failed[address.id] += 1
+      Rails.logger.warn(exception)
     end
 
     def next_address
@@ -64,8 +66,8 @@ module Aranha
       ::Aranha::Address.unprocessed
     end
 
-    def network_exception?(ex)
-      NETWORK_EXCEPTIONS.any? { |klass| ex.is_a?(klass) }
+    def network_exception?(exception)
+      NETWORK_EXCEPTIONS.any? { |klass| exception.is_a?(klass) }
     end
 
     def not_try_ids

data/lib/aranha/version.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Aranha
-  VERSION = '0.0.4'
+  VERSION = '0.1.0'
 end

data/lib/tasks/aranha_tasks.rake

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 namespace(:aranha) do
   task process: :environment do
     ::Aranha::Processor.new

data/test/aranha_test.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class AranhaTest < ActiveSupport::TestCase
-  test "truth" do
+  test 'truth' do
     assert_kind_of Module, Aranha
   end
 end

data/test/dummy/Rakefile

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 # Add your own tasks in files placed in lib/tasks ending in .rake,
 # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
 
-require File.expand_path('../config/application', __FILE__)
+require File.expand_path('config/application', __dir__)
 
 Rails.application.load_tasks

data/test/dummy/app/controllers/application_controller.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ApplicationController < ActionController::Base
   # Prevent CSRF attacks by raising an exception.
   # For APIs, you may want to use :null_session instead.

data/test/dummy/app/helpers/application_helper.rb

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 module ApplicationHelper
 end

data/test/dummy/bin/bundle

@@ -1,3 +1,5 @@
 #!/usr/bin/env ruby
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+# frozen_string_literal: true
+
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
 load Gem.bin_path('bundler', 'bundle')

data/test/dummy/bin/rails

@@ -1,4 +1,6 @@
 #!/usr/bin/env ruby
-APP_PATH = File.expand_path('../../config/application', __FILE__)
+# frozen_string_literal: true
+
+APP_PATH = File.expand_path('../config/application', __dir__)
 require_relative '../config/boot'
 require 'rails/commands'

data/test/dummy/bin/rake

@@ -1,4 +1,6 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
+
 require_relative '../config/boot'
 require 'rake'
 Rake.application.run

data/test/dummy/bin/setup

@@ -1,16 +1,18 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
+
 require 'pathname'
 
 # path to your application root.
-APP_ROOT = Pathname.new File.expand_path('../../',  __FILE__)
+APP_ROOT = Pathname.new File.expand_path('..', __dir__)
 
 Dir.chdir APP_ROOT do
   # This script is a starting point to setup your application.
   # Add necessary setup steps to this file:
 
-  puts "== Installing dependencies =="
-  system "gem install bundler --conservative"
-  system "bundle check || bundle install"
+  puts '== Installing dependencies =='
+  system 'gem install bundler --conservative'
+  system 'bundle check || bundle install'
 
   # puts "\n== Copying sample files =="
   # unless File.exist?("config/database.yml")
@@ -18,12 +20,12 @@ Dir.chdir APP_ROOT do
   # end
 
   puts "\n== Preparing database =="
-  system "bin/rake db:setup"
+  system 'bin/rake db:setup'
 
   puts "\n== Removing old logs and tempfiles =="
-  system "rm -f log/*"
-  system "rm -rf tmp/cache"
+  system 'rm -f log/*'
+  system 'rm -rf tmp/cache'
 
   puts "\n== Restarting application server =="
-  system "touch tmp/restart.txt"
+  system 'touch tmp/restart.txt'
 end

data/test/dummy/config.ru

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # This file is used by Rack-based servers to start the application.
 
 require ::File.expand_path('../config/environment', __FILE__)

data/test/dummy/config/application.rb

@@ -1,9 +1,11 @@
-require File.expand_path('../boot', __FILE__)
+# frozen_string_literal: true
+
+require File.expand_path('boot', __dir__)
 
 require 'rails/all'
 
 Bundler.require(*Rails.groups)
-require "aranha"
+require 'aranha'
 
 module Dummy
   class Application < Rails::Application

data/test/dummy/config/boot.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 # Set up gems listed in the Gemfile.
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../../Gemfile', __FILE__)
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../Gemfile', __dir__)
 
 require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
-$LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__)
+$LOAD_PATH.unshift File.expand_path('../../../lib', __dir__)

data/test/dummy/config/environment.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 # Load the Rails application.
-require File.expand_path('../application', __FILE__)
+require File.expand_path('application', __dir__)
 
 # Initialize the Rails application.
 Rails.application.initialize!

data/test/dummy/config/environments/development.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Rails.application.configure do
   # Settings specified here will take precedence over those in config/application.rb.
 

data/test/dummy/config/environments/production.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Rails.application.configure do
   # Settings specified here will take precedence over those in config/application.rb.
 
@@ -35,7 +37,8 @@ Rails.application.configure do
   # yet still be able to expire them through the digest params.
   config.assets.digest = true
 
-  # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb
+  # `config.assets.precompile` and `config.assets.version` have moved to
+  # config/initializers/assets.rb
 
   # Specifies the header that your server uses for sending files.
   # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache

data/test/dummy/config/environments/test.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Rails.application.configure do
   # Settings specified here will take precedence over those in config/application.rb.
 

data/test/dummy/config/initializers/assets.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Be sure to restart your server when you modify this file.
 
 # Version of your assets, change this if you want to expire all your assets.

data/test/dummy/config/initializers/backtrace_silencers.rb

@@ -1,7 +1,11 @@
+# frozen_string_literal: true
+
 # Be sure to restart your server when you modify this file.
 
-# You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces.
+# You can add backtrace silencers for libraries that you're using but don't wish to see
+# in your backtraces.
 # Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ }
 
-# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code.
+# You can also remove all the silencers if you're trying to debug a problem that might
+# stem from framework code.
 # Rails.backtrace_cleaner.remove_silencers!

data/test/dummy/config/initializers/cookies_serializer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Be sure to restart your server when you modify this file.
 
 Rails.application.config.action_dispatch.cookies_serializer = :json

data/test/dummy/config/initializers/filter_parameter_logging.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Be sure to restart your server when you modify this file.
 
 # Configure sensitive parameters which will be filtered from the log file.

data/test/dummy/config/initializers/inflections.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Be sure to restart your server when you modify this file.
 
 # Add new inflection rules using the following format. Inflections

data/test/dummy/config/initializers/mime_types.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Be sure to restart your server when you modify this file.
 
 # Add new mime types for use in respond_to blocks:

data/test/dummy/config/initializers/session_store.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Be sure to restart your server when you modify this file.
 
 Rails.application.config.session_store :cookie_store, key: '_dummy_session'

data/test/dummy/config/initializers/to_time_preserves_timezone.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Be sure to restart your server when you modify this file.
 
 # Preserve the timezone of the receiver when calling to `to_time`.

data/test/dummy/config/initializers/wrap_parameters.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Be sure to restart your server when you modify this file.
 
 # This file contains settings for ActionController::ParamsWrapper which

data/test/dummy/config/routes.rb

@@ -1,4 +1,5 @@
-Rails.application.routes.draw do
+# frozen_string_literal: true
 
-  mount Aranha::Engine => "/aranha"
+Rails.application.routes.draw do
+  mount Aranha::Engine => '/aranha'
 end

data/test/integration/navigation_test.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class NavigationTest < ActionDispatch::IntegrationTest

data/test/test_helper.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
+
 # Configure Rails Environment
 ENV['RAILS_ENV'] = 'test'
 
-require File.expand_path('../../test/dummy/config/environment.rb', __FILE__)
+require File.expand_path('../test/dummy/config/environment.rb', __dir__)
 ActiveRecord::Migrator.migrations_paths = [
-  File.expand_path('../../test/dummy/db/migrate', __FILE__)
+  File.expand_path('../test/dummy/db/migrate', __dir__)
 ]
 require 'rails/test_help'
 
@@ -17,6 +18,6 @@ Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
 
 # Load fixtures from the engine
 if ActiveSupport::TestCase.respond_to?(:fixture_path=)
-  ActiveSupport::TestCase.fixture_path = File.expand_path('../fixtures', __FILE__)
+  ActiveSupport::TestCase.fixture_path = File.expand_path('fixtures', __dir__)
   ActiveSupport::TestCase.fixtures :all
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors:
 - Eduardo H. Bogoni
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-10 00:00:00.000000000 Z
+date: 2018-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: active_scaffold
@@ -25,19 +25,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.4.41.1
 - !ruby/object:Gem::Dependency
-  name: rails
+  name: eac_ruby_utils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.10
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.10
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.10
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -86,7 +100,17 @@ files:
 - config/routes.rb
 - db/migrate/20171201021251_create_aranha_addresses.rb
 - lib/aranha.rb
+- lib/aranha/default_processor.rb
+- lib/aranha/dom_elements_traverser.rb
+- lib/aranha/dom_elements_traverser/conditions.rb
+- lib/aranha/dom_elements_traverser/cursor.rb
+- lib/aranha/dom_elements_traverser/data.rb
 - lib/aranha/engine.rb
+- lib/aranha/parsers/base.rb
+- lib/aranha/parsers/html/base.rb
+- lib/aranha/parsers/html/item_list.rb
+- lib/aranha/parsers/html/node/base.rb
+- lib/aranha/parsers/html/node/default.rb
 - lib/aranha/processor.rb
 - lib/aranha/version.rb
 - lib/tasks/aranha_tasks.rake
@@ -156,40 +180,40 @@ summary: Rails utilities for web crawling.
 test_files:
 - test/dummy/Rakefile
 - test/dummy/README.rdoc
-- test/dummy/bin/rails
-- test/dummy/bin/rake
-- test/dummy/bin/bundle
-- test/dummy/bin/setup
-- test/dummy/config/initializers/assets.rb
-- test/dummy/config/initializers/to_time_preserves_timezone.rb
-- test/dummy/config/initializers/wrap_parameters.rb
-- test/dummy/config/initializers/session_store.rb
-- test/dummy/config/initializers/cookies_serializer.rb
-- test/dummy/config/initializers/inflections.rb
-- test/dummy/config/initializers/mime_types.rb
-- test/dummy/config/initializers/backtrace_silencers.rb
-- test/dummy/config/initializers/filter_parameter_logging.rb
+- test/dummy/config.ru
 - test/dummy/config/boot.rb
-- test/dummy/config/locales/en.yml
-- test/dummy/config/secrets.yml
-- test/dummy/config/environment.rb
 - test/dummy/config/database.yml
-- test/dummy/config/routes.rb
+- test/dummy/config/secrets.yml
+- test/dummy/config/locales/en.yml
+- test/dummy/config/application.rb
 - test/dummy/config/environments/development.rb
 - test/dummy/config/environments/test.rb
 - test/dummy/config/environments/production.rb
-- test/dummy/config/application.rb
-- test/dummy/app/assets/stylesheets/application.css
-- test/dummy/app/assets/javascripts/application.js
+- test/dummy/config/environment.rb
+- test/dummy/config/routes.rb
+- test/dummy/config/initializers/assets.rb
+- test/dummy/config/initializers/cookies_serializer.rb
+- test/dummy/config/initializers/inflections.rb
+- test/dummy/config/initializers/session_store.rb
+- test/dummy/config/initializers/wrap_parameters.rb
+- test/dummy/config/initializers/to_time_preserves_timezone.rb
+- test/dummy/config/initializers/filter_parameter_logging.rb
+- test/dummy/config/initializers/backtrace_silencers.rb
+- test/dummy/config/initializers/mime_types.rb
+- test/dummy/db/schema.rb
 - test/dummy/app/views/layouts/application.html.erb
-- test/dummy/app/helpers/application_helper.rb
 - test/dummy/app/controllers/application_controller.rb
-- test/dummy/db/schema.rb
-- test/dummy/public/favicon.ico
-- test/dummy/public/500.html
+- test/dummy/app/helpers/application_helper.rb
+- test/dummy/app/assets/stylesheets/application.css
+- test/dummy/app/assets/javascripts/application.js
 - test/dummy/public/422.html
 - test/dummy/public/404.html
-- test/dummy/config.ru
-- test/test_helper.rb
+- test/dummy/public/favicon.ico
+- test/dummy/public/500.html
+- test/dummy/bin/bundle
+- test/dummy/bin/setup
+- test/dummy/bin/rails
+- test/dummy/bin/rake
 - test/aranha_test.rb
+- test/test_helper.rb
 - test/integration/navigation_test.rb